diff --git a/.clj-kondo/config.edn b/.clj-kondo/config.edn index af4647fb4..88d4649bc 100644 --- a/.clj-kondo/config.edn +++ b/.clj-kondo/config.edn @@ -107,6 +107,11 @@ (orcpub.routes-test/with-conn) (orcpub.routes.folder-test/with-conn) (orcpub.email-change-test/with-conn) + (orcpub.routes.public-reads-test/with-conn) + (orcpub.routes.item-save-test/with-conn) + (orcpub.routes.share-test/with-conn) + (orcpub.routes.party-test/with-conn) + (orcpub.heartbeat-test/with-conn) (user/with-db)]}} ;; native/cljs and web/cljs are separate source roots; kondo doesn't know ;; about them so ns names appear to mismatch their file paths. diff --git a/.githooks/pre-push b/.githooks/pre-push new file mode 100755 index 000000000..85d08bcb4 --- /dev/null +++ b/.githooks/pre-push @@ -0,0 +1,56 @@ +#!/usr/bin/env bash +# pre-push guard (reminder, not an auto-fold): refuse to push a commit that still +# carries docs/branch-changelog.md onto integration / develop / main. Mirrors +# .github/workflows/changelog-guard.yml so a forgotten fold is caught locally +# instead of on CI. The fold itself stays a deliberate manual step: +# scripts/fold-branch-changelog.sh "" (then commit, then push) +# +# Activate once per clone: scripts/setup-hooks.sh (or: git config core.hooksPath .githooks) + +status=0 +zero="0000000000000000000000000000000000000000" + +# Current in-progress release = the first "## [Name]" in CHANGELOG.md, so the +# suggested fold command is copy-pasteable. Falls back to a placeholder. +release="$(grep -m1 -oE '^## \[[^]]+\]' CHANGELOG.md 2>/dev/null | sed -E 's/^## \[(.+)\]$/\1/')" +release="${release:-}" + +while read -r _local_ref local_sha remote_ref _remote_sha; do + [ "$local_sha" = "$zero" ] && continue # branch deletion — skip + case "$remote_ref" in + refs/heads/integration|refs/heads/develop|refs/heads/main) + guarded=1 + if git cat-file -e "${local_sha}:docs/branch-changelog.md" 2>/dev/null; then + target="${remote_ref#refs/heads/}" + echo "" >&2 + echo " pre-push blocked: docs/branch-changelog.md is still un-folded on the commit" >&2 + echo " bound for '${target}'. Fold it into CHANGELOG.md, then commit and re-push:" >&2 + echo "" >&2 + echo " scripts/fold-branch-changelog.sh \"${release}\"" >&2 + echo "" >&2 + echo " (Decide whether this branch earned a '## Highlights' before folding —" >&2 + echo " see docs/branch-changelog.template.md. To bypass once: git push --no-verify.)" >&2 + echo "" >&2 + status=1 + fi + ;; + esac +done + +# Same rules CI applies, run here so a style problem is a local message rather than +# a red build after the merge. Skipped when python3 is absent; CI still catches it. +if [ "${guarded:-0}" = "1" ] && [ -x scripts/lint-changelog.py ] && command -v python3 >/dev/null 2>&1; then + if ! scripts/lint-changelog.py >/dev/null 2>&1; then + echo "" >&2 + echo " pre-push blocked: CHANGELOG.md does not follow the house style." >&2 + echo "" >&2 + scripts/lint-changelog.py 2>&1 | sed -E 's/^::error [^:]*::/ /; s/^/ /' >&2 + echo "" >&2 + echo " House style: docs/branch-changelog.template.md" >&2 + echo " (To bypass once: git push --no-verify.)" >&2 + echo "" >&2 + status=1 + fi +fi + +exit $status diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md index 5baec0250..9906f4aa7 100644 --- a/.github/PULL_REQUEST_TEMPLATE.md +++ b/.github/PULL_REQUEST_TEMPLATE.md @@ -7,4 +7,5 @@ - [ ] I have commented my code, particularly in hard-to-understand areas - [ ] I have made corresponding changes to the documentation if necessary - [ ] There is no commented out code in this PR. - - [ ] My changes generate no new warnings (check the console) \ No newline at end of file + - [ ] My changes generate no new warnings (check the console) + - [ ] `docs/branch-changelog.md` is updated in the house style (`docs/branch-changelog.template.md`), and I decided whether this branch earns a `## Highlights`. \ No newline at end of file diff --git a/.github/workflows/changelog-guard.yml b/.github/workflows/changelog-guard.yml new file mode 100644 index 000000000..5a37acda1 --- /dev/null +++ b/.github/workflows/changelog-guard.yml @@ -0,0 +1,48 @@ +# Guard: a branch changelog must be FOLDED into CHANGELOG.md (and removed) before +# landing on integration or develop. See docs/CONTRIBUTING.md. +# +# Runs on push (the post-merge backstop): if a merge landed on integration/develop +# without folding its docs/branch-changelog.md, this goes red — run +# scripts/fold-branch-changelog.sh, commit, and push again. Not run on +# pull_request, since the branch changelog legitimately lives on the feature branch +# until it is folded at merge time. + +name: Changelog guard + +# The fold guard is a post-merge backstop and only makes sense on push. The style +# lint is the opposite: it is feedback, so it also runs on pull_request, where the +# author can still act on it. Both are in one workflow because they are one topic. +on: + push: + branches: [integration, develop, main] + pull_request: + paths: ["CHANGELOG.md", "scripts/lint-changelog.py", ".github/workflows/changelog-guard.yml"] + workflow_dispatch: + +permissions: + contents: read + +jobs: + branch-changelog-folded: + name: Branch changelog folded + if: github.event_name != 'pull_request' + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - name: A branch changelog must be folded before landing here + run: | + if [ -f docs/branch-changelog.md ]; then + echo "::error file=docs/branch-changelog.md::An un-folded branch changelog is present on $GITHUB_REF_NAME. Fold it into CHANGELOG.md and remove it before landing — run: scripts/fold-branch-changelog.sh \"\"" + exit 1 + fi + echo "OK — no un-folded branch changelog on $GITHUB_REF_NAME." + + changelog-style: + name: Changelog house style + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + # No setup step: ubuntu-latest ships python3, and the linter imports nothing + # outside the standard library on purpose, so this stays a checkout and a run. + - name: CHANGELOG.md follows docs/branch-changelog.template.md + run: scripts/lint-changelog.py --stats diff --git a/.gitignore b/.gitignore index e595950d0..260f7b62b 100644 --- a/.gitignore +++ b/.gitignore @@ -22,8 +22,12 @@ pom.xml /classes/ /out/ /target/ -/node_modules/ -.clj-kondo +# no trailing slash: the e2e tooling is run by symlinking this to a sibling checkout's copy, +# and a trailing slash matches only a real directory +/node_modules +# the linter's cache, not its config: config.edn is tracked and must stay addable +.clj-kondo/* +!.clj-kondo/config.edn .calva .lein-deps-sum .lein-repl-history @@ -50,7 +54,10 @@ deploy/homebrew/* branch-sync-plan.md dev/test-accounts.edn -# E2E test artifacts +# E2E test artifacts. Unanchored, because a leading slash pins the pattern to the +# repo root: /node_modules/ and /e2e/node_modules/ both missed the real one at +# scripts/e2e/node_modules, and 179 files of it went into a commit. +node_modules/ /e2e/node_modules/ /e2e/playwright-report/ /e2e/test-results/ @@ -71,12 +78,19 @@ cljs-test-runner-out # Cursor IDE local config (contains user-specific paths) .cursor/ -# Claude Code local data (conversation history, credentials) +# Local assistant tool state (conversation history, credentials) — belongs in dotfiles, +# not in a code branch .claude-data/ - -# Agentic/AI tool files — belong in dotfiles or agents/ branch, not code branches .claude/ +# The agent entry point. Versioned on agents/develop, the agentic silo, and copied into a +# working tree per environment by scripts/agent-setup.sh. Ignored here because this branch +# ships to the public repo: an untracked copy is one `git add -A` away from being committed, +# which is how it drifted onto code branches before, and why the restructure deleted it. +# Note `.claude/` above does NOT cover this — that is a directory, and CLAUDE.md is a file +# at the repo root. +/CLAUDE.md + # NewRelic (DMV-specific, contains license key) newrelic* @@ -101,8 +115,30 @@ cljsjs/ # Local devcontainer setup (superseded by dotfiles) .devcontainer/local-setup.sh -# Design handoff reference material (preserved on agents/develop, not committed to code branches) +# Design handoff reference material (kept out of the code branches) design_handoff_character_menus # Generated clojure-CLI cljs build artifact (regenerated from project.clj; lein/figwheel-main is canonical) /deps.edn + +# Deploy-injected static vendor assets (font-awesome, etc.). index.clj references +# /assets/... at runtime, but these binaries are provided at deploy time and are +# not part of the source tree. A local copy may exist to render the screenshot +# harness faithfully; never commit the vendored fonts/CSS. +/resources/public/assets/ +/package-lock.json + +# Perf-investigation scratch: throwaway probe scripts and the multi-megabyte .orcbrew +# packs generated by dev/scale_orcbrew_pack.clj. The harnesses worth keeping live in +# test/browser/; the packs are regenerable and must not be committed. +/dev-scratch/ +dev-scratch/shots/ + +# Throwaway probe scripts at repo root (_pop.js, _mob.js …). Real probes live in +# test/browser/ — see docs/kb/fast-browser-probes.md. This stops `git add -A` sweeping +# a hardcoded-path scratch file into a commit, which happened twice in one session. +_*.js + +# Python bytecode from scripts/ (scripts/lint-changelog.py is run, not imported) +__pycache__/ +*.pyc diff --git a/CHANGELOG.md b/CHANGELOG.md index 0de007f2c..bfa895c4a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,10 +1,17 @@ # Changelog -## [Summer Patch] — 2026 (character-load resilience, homebrew salvage, PDF printing) +All notable changes are documented here, newest release first. Format: [Keep a Changelog](https://keepachangelog.com). +House style and the branch → release fold: [`docs/branch-changelog.template.md`](docs/branch-changelog.template.md). -Consolidates the June bug-patch bundle, the homebrew conflict-resolution/salvage -work, and the summer additions (PDF printing, character-load recovery, spell-key -reconciliation) into one release. + + +## [Summer Patch] — 2026 (character-load resilience, homebrew salvage & library management, PDF printing) + +### Highlights + +You can now move and copy homebrew between sources, turn content off without deleting it, and see in one place what needs fixing, with imports and exports that no longer spawn silent duplicates or false warnings. Characters that used to blank-screen on a bad load now recover in place, and printable spell cards and card backs read cleanly in black and white. ### Fixed @@ -14,7 +21,9 @@ Character loading & display - **Character sheets no longer go blank** — an unrenderable section shows a recovery message; the rest of the sheet stays usable (`565c33c0`). - **The Features tab loads for every character**, a nameless trait shows "[Unnamed feature]" instead of crashing the name sort, and Hunter's Evasion is named (`2a6fde93`, `5c3b073f`, `dd65d66a`). - **Homebrew class source no longer poisons spell-selection keys** — the source label was folded into the class `:name`, so saved spells/cantrips vanished when it changed; keys now come from a stable class identity, and orphaned saves repair on load (`9a709c0d`, `fe549631`, `a3e26155`). -- **Boolean toggles no longer corrupt data** and self-heal old damage (`1e9f27ec`); **non-ASCII name detection works in the browser** (`d9b23021`). +- **Boolean toggles no longer corrupt data** and self-heal old damage (`1e9f27ec`). +- **Non-ASCII name detection works in the browser** (`d9b23021`). +- **Inline "Custom" content isn't flagged as missing** — a character built with the built-in Custom option no longer triggers a false "Missing Content" warning; the `:custom`/`:none` inline sentinels are recognized as present, not homebrew keys to resolve (`124faa9a`). Homebrew import / export / salvage - **"Rename all" resolves duplicate keys in one pass** instead of the 20 → 3 → 1 → 0 re-import crawl (`c037de78`). @@ -23,7 +32,9 @@ Homebrew import / export / salvage - **Import can't report success it won't keep** — problems surface at import time against the loader's floor, and success fires only after the write persists (`c037de78`, `7782e831`). - **Dangling spell references render** with a key-derived name + edit link instead of a blank card, reported once per class (`a4dbfe19`, `73e75c9d`). - **Old-name spells in imported paks resolve** — 17 pre-2024 wizard-possessive keys (Leomund's, Tasha's, Bigby's…) map to their current SRD keys; loaded homebrew is never overridden (`cf1f4f1c`). -- **Keyword-trap imports are caught and routed to repair** instead of silently vanishing (`d9b23021`); **unreadable storage is preserved** for recovery, not deleted (`eedffc08`); **quota-failed saves warn and offer a backup**. +- **Keyword-trap imports are caught and routed to repair** instead of silently vanishing (`d9b23021`). +- **Unreadable storage is preserved** for recovery, not deleted (`eedffc08`). +- **Quota-failed saves warn and offer a backup**. - **Readable import/export errors** — plain-English console messages, dedup shown as a log line (`eba28a9c`, `e512dc45`); **post-save export** and **autosave-on-empty-template** crashes fixed (`e3c9a9ee`). ### Added @@ -31,8 +42,11 @@ Homebrew import / export / salvage Homebrew resilience & repair - **Per-entry salvage** — one bad entry no longer quarantines its whole source; valid items stay, broken ones are set aside for repair (`957e09ab`, `7782e831`, `c037de78`). - **Entry-level repair panel** in My Content — editable Name + Option-source per set-aside entry, Fix & Restore, and Discard (`d34007ff`, `c037de78`). -- **Export runs the same checks as import** — duplicates/cleanups caught on the way out; raw/pretty export stays an unchecked escape hatch (`9df1b4ae`, `c037de78`). -- **Resilient homebrew loading** with a My Content repair panel (`eedffc08`); **builder escape hatches** — draft export, refresh-safe WIP restore, "Save anyway" with placeholders, emergency raw export, Export & Auto-Fix (`eac350d0`, `e3c9a9ee`); **"Show homebrew source on class names" toggle** (`8f94a94c`); **fill-in dialog on export** with live field guidance (`1547cd69`, `22172adb`). +- **Export runs the same checks as import** — duplicates/cleanups caught on the way out, on every export: one source or the whole library, plain or pretty-printed. The unchecked hatches stay: the footer safety valve, emergency raw export, per-builder draft export, quarantined-source export, and "export as-is" (`9df1b4ae`, `c037de78`). +- **Resilient homebrew loading** with a My Content repair panel (`eedffc08`). +- **Builder escape hatches** — draft export, refresh-safe WIP restore, "Save anyway" with placeholders, emergency raw export, and Export & Auto-Fix (`eac350d0`, `e3c9a9ee`). +- **"Show homebrew source on class names" toggle** (`8f94a94c`). +- **Fill-in dialog on export** with live field guidance (`1547cd69`, `22172adb`). PDF & printing - **Card-back logo** — "Print logo on card backs" under a new Appearance section; the mark redrawn to print legibly (solid black, filled letters), with a faded-color option for color printers (`e8e560a3`, `99e20389`, `d0f2bfd8`). @@ -45,7 +59,1009 @@ Support - **Import and export share one correction gate**, and the exported library is canonicalized so import → export → re-import is idempotent (`9df1b4ae`, `c037de78`). - **Quarantine granularity is per-entry**, backward-compatible with whole-source entries, with precise per-entry diagnostics (`957e09ab`, `7782e831`, `c037de78`). - **Source-less imported content lands in the real "Default Option Source"** instead of a phantom placeholder (`54f4e87d`). -- **Save validation covers every required field** — dropdowns and multi-selects too (`e512dc45`); **save and load share one spec registry** so they can't drift (`ca977e0a`); normal exports strip meaningless blank flags; invalid-key errors are element-specific. +- **Save validation covers every required field** — dropdowns and multi-selects too (`e512dc45`). +- **Save and load share one spec registry** so they can't drift (`ca977e0a`). +- **Normal exports strip meaningless blank flags.** +- **Invalid-key errors are element-specific.** +- **PDF form appearances are baked on generation** — filled fields render consistently across all PDF viewers instead of only in Acrobat, and spell-card generation is more efficient (`45d106b4`). + +### Homebrew library management (My Content) + +**Added** +- **Move / copy content between sources** — one select-mode mechanism for single or bulk, + with a clobber-free key policy: a move keeps the key unless it is taken, a copy always + mints a fresh one (`903f44cb`). +- **Four-level disable hierarchy** — global / source / section / item, checked as an OR. The two new levels (global "all homebrew" + per-section) live in a local overlay store, so they're a per-device view preference that never mutates `.orcbrew` data or travels with an export (`95426d8c`). +- **Passive library health-status card** — one line per problem type with a count, covering + unresolved key conflicts, missing required fields, and export blockers. Warning-yellow for + attention, red for broken. Always on in the My Content hub, dismissable and remembered + elsewhere (`b58fe80b`, `79982e03`, `d0338049`, `e5372fed`, `e7040f4a`). +- **Opinionated, summary-first import** — safe defaults resolve conflicts up front with a one-click Import; the full per-conflict panel becomes "Review" (`e90466c1`). +- **Richer duplicate-key resolution** — severity split with honest labeling for the collapse-risk types, "keep both, turn one off" for a deterministic winner, rename the *existing* item, and an internal keeper-picker (`87512e47`, `052e6e55`, `0c30a022`, `862d9b26`). +- **Mutual-exclusion legibility** — per-row twin notes, a library banner, disabled-content badges colored by reason, and swap-on-enable keeping ≤1 enabled twin (`8543d8f6`, `d94973a6`). +- **My Content toolbar redesign** — two-zone (content vs library actions), select mode, and a 3-step delete guard (`49f2aafe`, `8fc497d9`). +- **Disabled-item visibility** — a count, a show/hide toggle, and search within a source (`47758423`). +- **Share a character with its homebrew embedded** — view-only, with a keep-in-library option and collision notice; custom magic items included (`4cae54e7`, `7bf4516a`, `35539c4c`). +- **Source-name-choice modal on import** — when a single-source file's name meaningfully differs from the source its content declares, ask whether to rename or keep, instead of silently guessing (`fa5909cf`). +- **Number→word name repair** for keyword-trap recovery ("9 Lives" → "Nine Lives") (`4c128a66`). + +**Fixed** +- **Single-source export/import no longer spawns a duplicate source** — the source is recovered from the content's `:option-pack`, not the browser-mangled filename; a last-resort dedup-suffix strip covers files with no declared source (`40413f17`, `e53a8b71`). +- **"Skip this one" in the conflict modal actually skips** — it was a no-op that imported the colliding item anyway (`47b57793`). +- **The `:route` handler no longer crashes on an unmatched (nil) URL** (`dab319a0`). +- **Dark-on-dark text in the conflict-modal body** (`b100b927`). +- **Custom item save persists the shown type** instead of blanking it (`52c0e40a`). +- **Stale `:key` after a rename** — the item's own `:key` is rewritten so a double rename is a no-op (`0c30a022`). +- **Recovery panel "Fix & Restore" auto-names invalid entries** in one click (`5e196348`, `898478b0`). +- **One home for source-less content** — folded the stray "Unsorted Homebrew" default into "Default Option Source"; "Unnamed Content" stays separate on purpose (nameless sources, for findability) (`a5d18e2f`, `b5ba38d0`). +- **Shared-character links render on first load** — decoded homebrew overlays now force a rebuild so the sheet paints immediately instead of only after a manual refresh (`9db84754`). + +**Changed / internal** +- **Health detectors are memoized subscriptions** — one library walk per plugins change instead of dozens per render (`47b57793`). +- **Conflict/export modals aligned to the health-card severity vocabulary** (`6cbd890f`). +- **Dead-code sweep** — removed verified-dead helpers; pre-existing dead code restored with dated investigation markers (`47b57793`, `874d57d5`). +- **Data-driven library list** — empty content-type categories hide; the list is derived (`e3023cd3`). +- **Gitignore deploy-injected static assets** (font-awesome) (`d8331619`). +- **Share buttons and the character-list filter sit flush with their toolbars** — plain form-button styling, header/list variants, and an aligned name-filter input (`4ef95b74`, `e5dbf8da`, `a48db2b5`). + +### starting-equipment + +**Highlights** + +Homebrew classes can define their starting equipment from the builder UI, in the full SRD +form: fixed items, choice groups, bundles, and nested weapon sub-choices. It applies on the +character sheet and round-trips through save, export, and import. You can also start from an +SRD class and change only what you want, and the export stores just those changes against +the base class. + +**Added** + +- **Starting Equipment section in the class builder** — a homebrew class can grant fixed + items and choice groups, including bundles and nested weapon picks, so the full SRD + equipment form is buildable without hand-editing `.orcbrew`. It round-trips through save, + export and import, and legacy simple choices convert to the editable form in one click + (`a4f13086`, `5a5f65a8`). +- **"Start from an SRD class"** — a dropdown fills the builder with any SRD class's starting + equipment. It's read from the live class (by applying the class's own modifier functions), + so it always matches what the class actually grants. All 12 classes (`2470e1d2`, `3fc5585d`). +- **Save only the changes** — a class filled from an SRD class stores a small "based on + plus these changes" form instead of a full copy. This lives only in the exported + file; everything in the running app stays the full form the existing functions already use. + A "Based on " banner shows the link, with a **Detach** button to save a full copy + instead (`6f494788`, `8172e915`, `be43690b`, `bcc05aa1`). + +**Fixed** + +- **Filling from a class keeps the SRD's own names** — a grouped focus pick stayed "Arcane + Focus" instead of being renamed "Starting Equipment: Arcane Focus" (the rename would have + changed the key it maps to) (`d6213b6e`). +- **A choice we don't recognise is never silently dropped** — an unrecognised sub-choice is + listed out option-by-option instead of vanishing; a genuinely empty one raises an error + with context (`2950117c`). + +**Changed** + +- **"Start from a class" reads the live class, not a hand-written table** — nothing to drift + out of sync; verified by a round-trip test against all 12 classes (`3fc5585d`, `f44324b3`). +- **Notification view components collected into `orcpub.dnd.e5.views.notifications`** — the + message banner, a reusable callout box, and the shared-content banner now live in one + namespace; the health/legacy banners render through the shared callout instead of + hand-rolled boxes (`4621b4c2`, `25f5d9a1`). +- **`lein e2e-server`** boots the full app (Pedestal + in-memory Datomic, no transactor) on + :8890 for browser tests (`47c413b0`). + +### feature/one-template-per-style + +**Highlights** + +Character sheets are generated from one template per style, and a multiclass +caster's spells can be packed one class to a column so a party of four casters +prints on one page instead of four — with a Warlock's Pact Magic kept as its own +pool. Spell rows mark concentration, casting time and costly materials; magic +item cards print alongside spell cards; and every page carries the site name. + +**Added** + +- `pdf/sheet-masters` names the file each style grows from and where that style's + artwork carries its attribution, and `pdf/grow-spell-sections!` reshapes an + opened master to the number of spellcasting sections a character needs + (`a78aaaf`). + +**Fixed** + +- A character with more casting classes than its template held got the features + and traits page wedged between its spell pages: generated pages were appended + to the end of the document rather than placed after the last spell page. The + eight-class fixture shipped as spell pages 3–8, features at 9, then spell pages + 10 and 11 (`de9a746`). +- Spells vanished off printed sheets because three templates numbered their spell-row fields + with gaps: **Glyph of Warding** was dropped from every wizard's style 1 and 3 sheet, and + style 4 lost **Continual Flame** and **Darkness** mid-list. Style 1's PREPARED ticks + carried the same numbering, so a prepared spell printed unticked. + `dev/fix_spell_row_fields.clj` renumbers them. +- Style 3 printed an empty HIT DICE box on every sheet — the box is drawn, the + `hd` field was never there. Style 4's second-page name box was likewise always + empty: it calls the field `character-name-p2` where the export writes + `character-name-2`. `dev/fix_missing_char_fields.clj` adds the one and renames + the other. +- `spell-packing/sheet-geometry` claimed capacity the templates did not have — + 13 rows at style 4's level 2 where only 11 could be filled — and undercounted + its level 1 at 12 where it holds 13. It is the field count now, with a test + tying it to the templates. +- Styles 3 and 4 threw `StackOverflowError` for any character with two or more casting + classes, so those sheets could not be exported at all. Clones are now inserted with + `insertAfter`, avoiding the whole-object-graph cycle check that exhausted the stack. + Styles 1 and 2 were never affected, which is why this survived. + +**Changed** + +- The 28 templates are now 8: for each style, one to grow from and one with no + spell page for a character who casts nothing. 44.3 MB to 9.7 MB. +- Style 4 grows from a one-spell-page master like every other style: its licence footer + turned out not to be baked into the artwork, so the marked page alone yields clones that + all carry the footer. The retired file leaves `resources/` 4.5 MB lighter, and the + surviving page renders byte-identically. +- That page's footer block is four operators rather than six: `0 i` sets flatness + tolerance, which applies to path curves and not to glyph fills, and `/GS2 gs` + is the page default, differing from GS0 only in stroke adjustment. +- Style 4's structure tree is removed along with the page it referenced, taking the file + from 2622 objects to 1301. The cost is style 4's accessibility tagging — styles 1 and 2 + keep theirs, style 3 never had any — and restoring it means writing the pruner properly. +- Exports are smaller at every caster count above one, by 49 KB to 671 KB + depending on style, and a character with no spellcasting gets a file the same + size as before. +- Generating a sheet repeats less work: form values are looked up once instead of twice, + prose fields are measured only when they hold something, and a cloned spell page no longer + re-reads its source per clone. A six-caster sheet allocates 162 MB rather than 607, and a + character who casts nothing no longer scans the pages for spell sections at all. + +**Added (spell row annotations)** + +- A spell row can carry its concentration, casting-time and costly-material marks + beside the name, behind `print-spell-annotations?`. Of 319 spells concentration + touches 126, a costly material 52, a bonus action 14 and a reaction 4. +- FIXED columns, not appended to the name. A `C` among letters is the same visual + class as the letters — single capital, same weight — so finding it is a serial + search and the eye has to read every row; a column turns that into one vertical + sweep. More spacing does not fix a serial search, alignment does. +- Drawn, not written into fields: 11 bytes a row against 671 as form fields + (6.6 KB against 389 KB over 594 rows), on a branch whose point was smaller files. +- The rows are narrowed by the reserved zone BEFORE the values are written, so a + long name shrinks to clear the columns rather than running under them. Verified: + the longest real spell names fit the narrowed row on all four styles. +- Ritual is deliberately not marked — its `R` would sit beside the `RE` of + reaction, and plain V S M is on nearly every spell, so it is the widest to print + and the least worth reading. + +**Added (packing, server half)** + +- The export accepts `:spell-relabels`, the small instruction list the browser + sends alongside the field map when it has packed a character's spells into + boxes other than their own numeral. The server applies `relabel-spell-level!` + and `reuse-cantrips-box!` per instruction and needs nothing else — it never has + to know what a spell level is. +- Bounds-checked, because it comes from the client and reaches field names and a + drawn label: section must name a page the document actually grew, box must be + one of the ten, and label a single digit or nil. Malformed instructions are + refused and counted rather than thrown on, so a client sending something this + server does not understand cannot cost the character their sheet. The list is + capped at ten boxes a section. +- `relabel-instructions` counted sections from ZERO off `map-indexed`, while + every field name carries a 1-based suffix — so the instructions named a section + no template has. + +- `pdf_spec` split a character's spells by a hardcoded copy of style 1's row + counts, whatever style was being exported: a style 4 sheet was handed 8 cantrips + for a box with 7 fields and lost one, and 12 first-level spells for a box that + holds 13. It reads `spell-packing/sheet-geometry` now, so the counts have one + home and a test ties them to the templates. + +- `spell-packing/packed-fields` turns a packing into the field map the export + writes: each class holds its own contiguous run of boxes in one column, so + **four short lists fit one page** where today they take four. Rendered proof in + `target/packed-demo.pdf`. +- This is also what separates a Warlock's Pact Magic. Every level box carries its + own `spell-slots` field, so a class holding its own column carries its own slot + counts — Warlock 2, Sorcerer 4/3, Paladin 4/3, Bard 4 on one page. Grouping by + ability, as today, merges a Warlock and a Sorcerer into one CHA section and + writes every box the character-wide total. +- A pact caster is given the first column outright: cantrips in box 0, the one + level it casts at in box 1 — renumbered as the character levels rather than + taking a new box — spilling into box 2 only because a level 20 Warlock knows 15 + spells against box 1's 12 rows. That is both simpler than fitting it like any + other class and what keeps its slot pool off the classes beside it. +- Each spell column is headed with the class that holds it, so a party of casters can be + told apart at a glance. A class with no cantrips is headed in its first level box + instead of going unnamed, and headings are only placed on bars with no slot inputs the + player writes in. +- The compartments are read off the live fields rather than written down, so they + follow the artwork: 51.9–91.1 and 103–195.8 on style 1. A style with no + `slots-expended` field (2 and 4) has the wide one taken from the spell row's + right edge instead. Box 0 has no slots fields at all and borrows level 1's. +- A name too long even at the 6pt floor is shortened with an ellipsis rather than printing + through the divider — at 6pt "Eldritch Knight" still measures 43pt against a level box's + 35. + +- Each class's spellcasting ability, save DC and attack bonus print above its column's bar, + bold and near-black, so they read at a glance mid-turn. A sheet section carries one such + triple, so on a packed page holding several classes it is left empty, and filled only + where a page holds a single class. +- Packing runs on all four styles. The printed level numeral is covered with a + white rectangle cut to that style's measured digit box (`pdf/numeral-boxes`, + from `dev/scan_numerals.clj`) rather than a hexagon traced off style 1, which + works everywhere because the paper around every numeral is white (`a8752173`). + +**Added (packing, builder half)** + +- `pdf_spec/packing-classes` regroups `spells-known` — which is keyed by LEVEL — + into the per-class lists the packer takes. That regrouping is the point: the + shipped layout groups by `:ability`, which is why a Warlock and a Sorcerer share + one CHA section. +- **Pact Magic is separated in the character model.** A Warlock/Sorcerer's pact slots were + added to the shared ones and printed as a single inflated number — on the normal sheet, + not only when packed. Shared and pact slots are kept apart now, with their sum still + available to everything that reads it. +- A pact caster's whole list is reported at its highest pact slot level, because + that is how a Warlock casts — which is what lets it hold one box however high it + climbs. +- `:spell-layout` picks `:packed` or `:per-class`; the default is computed from + the build — packed only when there is more than one casting class AND the style's + numerals can be relabelled. A single caster already reads down its own page. +- The export accepts `:spell-headings` alongside `:spell-relabels`, bounds-checked + the same way. + +**Added (guards)** + +- A full character is written to every style and the values `write-fields!` could + not place must match `pdf/unsupported-fields` exactly. That report had always + been returned and never checked, which is how the losses above shipped. Exact + equality, so a stale declaration fails too once its template gains the field. +- Every indexed field family must run 1..n with no gap, and no two fields in a + master may share a name. +- `pdf/unsupported-fields` records what a style genuinely cannot print. Style 4 + is the Cthulhu Mythos sheet and carries "Conditions and Insanities" where the + others carry inspiration, so inspiration is all that is left in it. + +- Style 4 has no allies or backstory box, so both values are written into its general Notes + box under headings rather than dropped. That box is smaller than the two it replaces, so a + long backstory shrinks to fit and a very long one clips. An empty section prints no + heading. + +**Added (cards)** + +- Spell and magic item card backs carry `dungeonmastersvault.com`, centred at the foot of + every card. Backs were chosen over fronts because a blank back has room where a front is + filled to the edge by spell text. Text carried over from a front is laid out clear of the + stamp. + +- Character sheets carry the same line along the foot of every page, at a + position measured per style off RENDERED pages (`dev/scan_site_line.clj`) and + held by a test. A page that prints its own line is skipped rather than stamped + over, and the skip is per PAGE, not per style: style 4 prints the line on its + spell pages only, and its other pages are stamped like any other. +- Each stamped page gets its own appended content stream. Cloned spell pages + share the master's stream, so writing into it would have printed the line once + per clone on every one of them. + +- An export's two images are fetched concurrently, before either is drawn. Each + allows 10s to connect, 10s on the socket and a 20s transfer deadline, and the + fetch happens holding an export slot — so drawn one after the other, two slow + images occupied a slot for up to 80s. Started together they cost one image's + worst case rather than two. +- The route no longer calls `safe-image-url?` before fetching. `safe-image-bytes` + validates on its own, and ITS resolved addresses are the ones the connection is + pinned to, so the earlier call only resolved the host a second time. The cheap + scheme regex stays: it refuses `file://` and `ftp://` with no lookup at all. + +**Added (capacity)** + +- `ORCPUB_HTTP_MAX_THREADS`, `ORCPUB_PDF_CONCURRENCY` and + `ORCPUB_PDF_QUEUE_TIMEOUT_MS` let the operator size the export stack for the + host. Sheet generation is bounded separately from the HTTP pool, so a rush of + exports no longer competes with logins and saves for the same workers, and an + export that cannot get a slot is answered 503 with a measured `Retry-After` + rather than held open until the browser gives up. +- `docs/PDF-EXPORT-CAPACITY.md` documents what an export costs, what the numbers + mean, and how to size the settings, with the measurements behind them. +- A turned-away export gets a busy page that retries itself, counting down a + measured interval with jitter and carrying the original request forward, then + hands over to a button after `ORCPUB_PDF_MAX_RETRIES` attempts. The export is a + form POST into a new tab, so this needed no change to the builder and none to + how a finished sheet arrives. The page carries the site header, logo and + stylesheets, as the privacy and terms pages do. +- `lein e2e-server-busy` runs the e2e server with an export queue small enough to + reach by hand, for seeing the busy page on a dev machine. + +**Added (magic item cards)** + +- Magic item cards, opt-in from the builder alongside spell cards. Each card + carries the item's name, kind and rarity, an attunement badge in the header and + the clause at the foot, a charge track when the description names a number of + charges, and rarity-graded cornerwork. Descriptions that overrun continue on the + back. `dev/measure_item_card.clj` prints the clear space between every pair of + stacked elements on a worst-case card, so the spacing is measured rather than + eyeballed. + +**Changed (cards)** + +- Card icons are drawn from SVG paths instead of 32px rasters, so a 600 DPI printer is no + longer asked for about 150 device pixels from a 32 pixel source. Each icon is embedded + once per document and referenced where drawn: +7% on card pages, byte-identical on a sheet + with no cards. The `-bw` duplicates are gone, since colour is applied at the draw site, + and `resources/public/image/ATTRIBUTION.md` credits the icon authors, which nothing did + before. +- Card fonts and the image embedder are built once per document by the export + handler rather than once inside each card function. Both are per-document, so a + sheet printing spell cards AND item cards carried two complete copies of + Vollkorn and two of the card-back mark — 35% of the file on card pages. + +**Fixed (hardening)** + +- The work one request can buy is bounded at the request itself. + `routes/bound-request` drops a `spellcasting-class-N` name past the ceiling and + truncates every collection before any part of the export sees the body, so a + feature added later that reads a list is bounded without being wired up. + Unclamped, `spellcasting-class-9999` ran 310 seconds from a few dozen bytes and + died out of memory. +- `safe-image-url?` refuses the private addresses `InetAddress` has no predicate + for: `fc00::/7` (what private IPv6 actually uses — `isSiteLocalAddress` knows + only the deprecated `fec0::/10`), the NAT64 and 6to4 wrappers that carry an IPv4 + address inside an IPv6 one, `100.64.0.0/10`, and `0.0.0.0/8` and `240.0.0.0/4`. + Tests pin both directions, including that public addresses inside those same + wrappers stay fetchable. +- An image transfer is bounded in TIME as well as bytes. `setReadTimeout` bounds + each read, so a server dribbling a byte before each timeout held a connection — + and an export slot — indefinitely: measured, 40 bytes over 12.0 seconds with no + timeout firing. + +**Added (tests)** + +- `svg_path_test` covers each path command and then parses every SVG in + `resources/` as a net. That caught the extractor matching only double-quoted + attributes when the whole `black/` set uses single quotes — 148 icons that would + have rendered blank. +- `card_export_test` counts what a saved document actually contains, stripping + PDFBox's per-subset prefix, so two copies of one face cannot pass as two fonts. +- `pdf_image_fetch_test` drives the fetch transport against a real HTTP server: + the byte cap against a body with no declared length, the refusal to follow a + redirect, the transfer deadline against a trickling server, and what the pixel + budget does with a page served as a 200. + +**Fixed (hardening, cont.)** + +- The image fetch resolves a host once and connects to that same answer, closing a window + where DNS an attacker controls could answer public for the safety check and private for + the connection. Hostname and certificate verification are unchanged, and the pin is + skipped behind an egress proxy, where the client connects to the proxy instead. + +**Added (packing, every style)** + +- Column headings survive the annotation columns: each spell row records its + pre-reservation right edge, which the bar of a style with no `slots-expended` + field reads instead of the narrowed row — an 83pt compartment had read 27pt and + printed "Warlock" as "Warl…" (`a8752173`). +- The CANTRIPS word printed into a box-0 bar is covered by a band measured per + style (`pdf/cantrips-word-patch`, `dev/scan_cantrips_word.clj`); one band + either left style 3's word showing or painted through style 4's rules + (`a8752173`). +- The per-class ability, DC and attack sit on a backing strip, so they read over + the scrollwork styles 3 and 4 print above the bar (`a8752173`). +- `dev/stress_packing.clj` runs seven caster shapes on every style and fails on + a spell that goes missing without being reported (`3b87a48a`, `50d122fc`). + +**Fixed (packing)** + +- A packing that could not hold a class dropped it in silence — a Wizard 20 + beside a Cleric and a Druid printed without the Cleric, 33 spells gone. + `spell-packing/unplaced` reports what a packing could not place, `pdf_spec` + falls back to a page per class when anything is, and the builder does not + offer a layout that cannot hold the character (`3b87a48a`). +- A no-cantrips class leading a free column started at box 0, which the export + redrew as a level box from style 1's measurements: on style 3 the numeral + missed the ring, and on every style the class name was clipped by the input + drawn over it. Box 0 holds cantrips only; the server refuses a label on it + (`50d122fc`). + +**Added (builder)** + +- A **Spell Sheet Layout** choice in the PDF options — Automatic, one column per + class, one page per class — shown to a multiclass caster on a style that can + be packed, with a line saying what the current setting prints. An untouched + control sends nil so the computed default stays live (`642796c7`). +- The PDF options are grouped — Character Sheet, Cards, Appearance — and every + option carries a `?` that opens a line saying what it does. Click rather than + hover, for phones (`9d3a22ef`). +- `test/browser/spell_layout_pdf_e2e.js` builds a Warlock 5 / Sorcerer 5 through + the real builder and exports every style under both layouts against the running + server (`642796c7`). + +**Fixed (builder)** + +- The Appearance group followed spell cards alone, so printing only magic item + cards lost black & white and the card-back logo, both of which apply to them + (`9d3a22ef`). + +**Changed (page shell)** + +- One sticky header instead of a fixed copy above an inline one. Every header + control existed twice in the DOM — twice in the tab order, and the whole PDF + options panel with it. `.app` clips overflow with `overflow-x: clip` now, which + trims the same overflow without becoming the scroll container that stopped the + header sticking (`6cd529ee`). +- `test/browser/sticky_header_e2e.js` drives the phone case as a real device + descriptor: the app picks its layout off the user agent, so a narrow desktop + viewport renders the desktop tree into a phone width (`4ffc02a7`). + +**Changed (tooling)** + +- `scripts/test/run-cljs-tests.js` runs the compiled ClojureScript test build in + headless Chromium; `lein fig:test` only compiles it. The packer and annotation + tests are in the ClojureScript runner, since both run in the browser. +- Lint is clean: 30 warnings to 0 (`28fd620d`). + + +### perf/homebrew-builder-loop + +**Fixed** + +- **The character builder no longer freezes when you switch between Race and Class with a + large homebrew library.** Three internal caches were keyed on the whole class list, so + every lookup rebuilt every class's 20 levels. A Class-tab switch went from 1125 ms to 100 + ms in development and 654 ms to 92 ms in production, and the page holds about 48 MB less + (`c90016ac`, `4b67b3f7`). +- **A character change now rebuilds the character once, not twice.** The builder's preview + pane subscribed with a stray argument, which created a second, independent debounced + builder over the same character; both ran on every edit (`7eb968db`, `dc667154`). +- **Spell details are built when you open a spell, not when a list of spells is drawn.** + Listing 41 spells built 41 full descriptions nobody had asked to read (`ebe708f9`, + `2747553e`). + +**Changed** + +- **Modifier ordering is linear rather than quadratic**, so character rebuilds get cheaper as + a character grows: 23.0 ms → 3.0 ms on the JVM and 25.2 ms → 4.9 ms in the browser, with + output order proven identical in both runtimes across 808 generated graphs (`8785b16a`). + +**Added** + +- **A browser probe suite for the builder's performance** — longest-task-per-interaction + under CPU throttling, class-body cost, builds-per-click, CPU profiling by inclusive time, + and the localStorage measurements. `test/browser/README.md` lists what each answers + (`d08a792b`, `30bb6355`, `0986cf44`). +- **A functional test for the class handlers** — set-class, set-class-level, add-class and + delete-class driven for real against app-db. Neither test suite clicks anything, so these + had no coverage (`0634c5ce`, `1bac07c6`). +- **`docs/kb`** — an indexed knowledge base: the freeze investigation and its root cause, a + scan of every `memoize` site with the risky ones traced, the localStorage measurements and + a parked chunked-storage plan, and the verification lessons this cost (`2bb966d8`, + `0634c5ce`). + + +### feature/browser-side-character-images + +**Highlights** + +A character's portrait now reaches the sheet from hosts that used to refuse it, including +Pinterest and D&D Beyond: the browser reads the picture, and where it is refused the server +fetches it instead, with nothing asked of the user. When an address cannot work, one line +under the field says why and offers at most one thing to do about it. A picture can also be +pasted or copied straight in. + +**Added** + +- `orcpub.image-capture` reads a character's picture in the browser: a + CORS-attributed `` drawn to a canvas, scaled to the size the sheet prints + and encoded until it fits the 128 KB ceiling. Only the canvas route exists — + the app's CSP is `connect-src 'self'`, so `fetch` to an image host is blocked + and attempting it would log a violation on every export, while `img-src` allows + `https:`. +- `pdf/decode-image-bytes` takes those bytes on the server. The same 128 KB and + 2000×2000 ceilings as `safe-image-bytes`, checked against the ENCODED length + first so an oversized image never becomes a byte array, and the format read from + the bytes rather than from the mime type the client claimed. +- The export spec carries `:image-data` and `:faction-image-data`; when they are + present `/character.pdf` does not fetch at all. +- `POST /image-probe`, which the builder asks as soon as a browser read fails: + can THIS server fetch THAT picture? The bytes are kept for ten minutes, so the + export that follows costs the host no second request, and a negative answer is + remembered too. It answers a boolean and never the picture — the endpoint needs + no login, and returning fetched bytes would make it a general-purpose proxy — + and every address rule that guards the export guards it. +- A "Use copied image" button beside the field, so the last resort is one click + rather than an instruction. It reads a picture the VIEWER has copied; it cannot + do the copying, because a page-initiated copy of a cross-origin image puts its + markup on the clipboard and not its pixels -- the same rule that taints the + canvas, and the reason extensions can do this and pages cannot. +- Paste, for a host that lets nobody read its pictures. The clipboard carries the + DECODED image -- the browser's own "Copy image" put it there -- so none of the + host's rules reach it. Two clicks, and no download-and-upload round trip. This + is the answer for Pinterest and anything else that refuses page and server + alike. +- An upload under the Image URL field for a host that allows no read. It runs the + same ceilings, and falls back to the image loader when `createImageBitmap` + refuses a file the loader renders — it is the stricter decoder of the two. +- `test/browser/character_image_capture_e2e.js` drives both routes through the + real app. The server refuses loopback addresses, so an image reaching a PDF from + the test origin can only have arrived as bytes the browser read. + +**Added** + +- `orcpub.image-url/advise` reads the address alone and catches most real mistakes before + any request is made: a page's address pasted instead of the picture's, a login wall, a + missing or non-web scheme, a stray space, and `http://`, which this page's CSP will not + display whatever the host does. Dropbox and Google Drive links are offered a correction to + take or leave. Advice, never enforcement. + +**Changed** + +- http is upgraded to https automatically, once the https address is known to load. The + check costs no request that was not about to be made anyway; the field changes only after + it succeeds and says why, and a host that serves no https is told instead, with its + address left exactly as typed. +- **One line under a field, never four blocks.** A single unreachable picture could raise a + scheme warning, a suggested correction, a fetch failure and a panel of controls at once. + Only the most actionable now shows, and the other ways in wait behind one disclosure. +- Controls live outside notices. A notice says what is wrong; a red panel holding + a button, a sentence and a file picker is a control surface wearing an error's + colours, and both halves get harder to read. A correction is now a question -- + "Did you mean https://...?" -- under its notice, and supplying a picture by + hand is its own labelled block. +- Field notices are a component rather than a red line. `.field-notice` carries a + severity accent, a panel ground and two parts with two jobs: the fault in the + severity colour, and the instruction -- the only part anyone acts on -- brighter + and heavier beneath it. Run together in one colour and one weight, the vaguer + half reads first and the useful half is skipped. +- **`.red` was unreadable on the app's own background.** `#9a031e` sits at about + 9:1 on white and about 2:1 on near-black, less than half the readable minimum, + and the app is dark by default. The dark theme now takes a lighter red and the + light theme keeps the deep one; the same colour cannot serve both. This reaches + every use of `.red`, not just these fields. +- Inline rather than a hover tooltip, deliberately: these notices carry buttons, + and `.tooltiptext` disappears when the pointer moves toward it, is fixed at + 130px, and is absent on mobile. + + +- The builder says nothing about pasting or uploading until both routes are known to be + shut. Most hosts that refuse the browser serve the server perfectly well, so speaking up + earlier asked people to supply a picture that was about to arrive: of sixteen common + portrait hosts, nine let the browser read, and most of the rest allow the server. +- Exporting is held while a picture is still being read, so the browser's bytes + win that race instead of falling through to the server. `capture` carries a + deadline, so a read always ends and the hold is bounded. +- An oversized picture gives up size before quality, down to what the sheet can actually + show — the portrait box is 945px on its long edge at 300dpi. A picture already smaller + than that is never scaled, only re-compressed. A 5.8 MB noise PNG leaves the browser at 92 + KB and full quality, where spending quality first produced 37 KB and a worse picture. + + +- Pictures are read when the thumbnail loads and when the export panel mounts, + never on the export click: the export is a synchronous form submit into a new + tab, and an await in between spends the user activation that keeps that tab from + being blocked. Bytes are held in app-db keyed by URL, outside the character + entity — that entity is what gets persisted, and localStorage has a ceiling. +- `docs/CHARACTER-IMAGE-FETCH.md` leads with the browser path; the server fetch is + documented as the fallback it now is. + +**Changed** + +- `/image-probe` answers a reason rather than a boolean, so the builder can say what is + worth fixing: the link, the picture, or simply waiting. Telling someone to copy a picture + when they have mistyped a link is not help. The server never sends a sentence, and every + address refusal collapses to one code, so the endpoint cannot be read as a map of what + this network can reach. + +**Fixed** + +- **A picture the host served happily was refused for weight.** One 128 KB limit capped + both what the server would download and what could go into the PDF, so a 393 KB + Pinterest portrait was dropped although nothing had blocked it. The two limits are now + separate — 2 MB down, 128 KB into the document — and a heavy picture is scaled and + re-encoded to fit. + + +- The builder flashed "Image failed to load" at pictures that were fine. + `image-error` dispatched when it was CALLED, at render time, rather than + returning a handler -- so every fresh URL was marked failed before the browser + had tried it, and only the load took the mark back. + +- A picture whose host allows no read stopped displaying in the builder: the optimistic + "failed" mark set when the thumbnail renders was never withdrawn, because the load handler + had captured the flag while it was still clear. The clear no longer reads the flag, so an + ordinary load does not count as an edit. + +**Removed** + +- `create-monsters-pdf`, which was private with zero callers, and the + `draw-text-from-top` helper, `HELVETICA_OBLIQUE` font and + `orcpub.dnd.e5.monsters` require that it was the only user of. + +### feat/whats-new-panel + +**Highlights** + +The site now says what changed. New release highlights open once per browser and +then stay one click away in the footer, so a release is something people notice +rather than something they'd have to go read the changelog to find. + +**Added** + +- **What's New panel** — the current release's highlights open on the first visit + after it ships, and the footer link and version line reopen them any time. + Closing it stamps the release, so it stays shut until the next one (`ee3e4d8b`). +- **`orcpub.whats-new`** — the release entries and the id that gates the panel, in + one cljc file the panel and the tests both read (`ee3e4d8b`). +- **Twelve Summer Patch highlights under three headings** — library, characters, + printing — covering the builder freeze, the spell rows that never printed, the + two styles that could not export a multiclass caster, and the packed multiclass + layout, alongside the homebrew and portrait work (`9e3017d3`). + +### feat/option-picker + +**Highlights** + +Equipment items are picked from a filtering dropdown instead of a 1037-option native select. +Type to narrow it, scroll the whole list, or walk it with the arrow keys. + +**Added** + +- Equipment inventory sections use a filtering dropdown: type to narrow, click or press Enter + to add (`95d38f67`). +- Arrow keys walk the list and scroll the highlight into view, so it can be used without the + mouse (`ba52a219`). +- The matched text is highlighted, which shows why a row matched when the match lands + mid-word — filtering by `+1` marks the suffix, not the name (`4082bc20`). +- The dropdown shows how many items it holds and what the keys do (`4082bc20`). +- `scripts/test/run-browser-probes.js` runs every asserting browser probe and fails the run + if any fails. Neither test suite invokes `test/browser/`, so nothing was checking them + (`89d49918`, `6e60959f`). + +**Fixed** + +- An item with no name no longer throws while filtering (`78deb2ad`). +- The dropdown lines up with its input and flips above it rather than running off the bottom + of the screen (`95d38f67`). +- Two browser probes had been left pointed at a control that no longer existed: one was + failing unnoticed, the other had quietly stopped taking screenshots (`d51aa979`). + +**Changed** + +- The Equipment dropdown no longer caps what it shows. The previous 12-row cap left 294 of + 306 magic weapons unreachable unless you already knew the name (`ba52a219`). +- The dropdown menu was flat; it now has depth, a themed scrollbar, a highlight bar and a + short open animation that respects reduced-motion (`78deb2ad`). +- Removed the growable option-menu namespaces lifted from `redesign/growable-option-menus`. + Nothing referenced them, and they carry theming, layout modes and page structure — a + site-wide redesign, not a picker (`233d032e`). +- A probe that stops asserting, or sits silent for 180s, now fails instead of passing + (`7369cc33`, `ee0b3781`). + +### fix/header-flyout-under-sticky-toolbar + +**Fixed** + +- **Header dropdowns are clickable again** — Characters, Spells, Monsters, Items, + Encounters and My Content all rendered under the button row, which swallowed + the click on most of their items (`9eb3ac81`). + +### fix/tall-flyout-and-held-release + +**Fixed** + +- **Tall header menus stay on screen** — an opening flyout is capped to the room + below it and scrolls, so every item in My Content is reachable on a short + window instead of running off the bottom (`2573f976`). +- **The What's New panel opens when the cookie notice is dismissed** — it no + longer waits for a reload the visitor never makes (`2573f976`). + +### fix/release-panel-hold-ceiling + +**Fixed** + +- **An ignored cookie notice no longer hides the release panel** — the hold has a + ten-second ceiling, after which the panel shows anyway with the notice left + where it was, behind it. + +### test/overlay-reachability + +**Added** + +- `test/browser/overlay_reachability_e2e.js` — hit-tests every control an overlay + shows (header chrome, Orcacle, the PDF options panel, the release panel) and + fails if one is covered or off the window inside a floating layer. `SELFTEST=1` + proves it can fail; a state that audits nothing fails as a stale selector, so it + cannot quietly stop asserting. Overlays needing a login, an import or saved + content are listed as skips rather than left unmentioned. + +### hotfix/my-content-dropdown-and-probe-cost + +**Fixed** + +- **The "+ add content…" dropdown in My Content is readable and matches the app** — + its option list follows the theme instead of opening as a white box with + invisible entries, and the control takes the same border and text as every other + dropdown. `color-scheme` is set on `select` for both themes, so any unclassed + dropdown is covered too (`61ddd603`). + +**Changed** + +- **The release-panel probe runs in 30 seconds instead of 124** — same 18 checks. + Cases that did not need their own app boot are folded together, and the two + independent stories (the ordinary one, and the cookie notice) run in parallel + contexts, so the wall clock is the longer lane rather than the sum (`61ddd603`). + +### hotfix/boot-config-report + +**Added** + +- The server says it started and what it is configured with, on every start. Every setting + the app reads is listed with its value and where that value came from — `SET` from the + environment or `DEFAULT` — grouped by runtime, database, security, email, content and + capacity, plus a count of the 21 `APP_*` branding variables (`a80b5aec`, `c2754d28`). +- A value that was supplied and rejected is called out by name beneath the table, since its + row shows the default that is running rather than the value you asked for (`e65dfda6`). +- A setting whose absence breaks the site says **why and how to fix it**, in the boot banner, + the startup warning and the `500` a caller receives — the same text in all three, so + nobody has to work the remedy out from the symptom (`2c425afa`). + +**Fixed** + +- The database password is no longer printed at startup. It was in four places, not one: the + startup line and three `ex-info` maps, which reach logs and error reporters just as + readily. Credentials are blanked in both shapes they arrive in — a `password=`-style query + parameter and `user:pass@host` userinfo — while the host, port, database and user still + show (`24aa9971`). +- `SIGNATURE` missing now fails clearly everywhere. `create-token`, `unsubscribe-token` and + `unsubscribe` read the environment straight into `jwt/sign`, failing deep inside the + library with nothing an operator could act on (`2c425afa`). +- `EMAIL_SERVER_URL` missing is reported as breaking, because it is: the verification email + is sent inside the signup transaction, so unconfigured mail does not skip an email — every + registration fails with "please try again", and password reset fails too (`e66eeef8`). +- `DATOMIC_URL` unset defaults to a local development transactor, and the failure read + "cannot connect to localhost:4334", naming the symptom rather than the cause. It now says + the development default is in use (`e66eeef8`). +- `ENVIRONMENT.md` was missing `ORCPUB_PDF_MAX_CASTER_SECTIONS`, `ORCPUB_PDF_MAX_CARDS`, + `EMAIL_SERVER_URL`, `EMAIL_SERVER_PORT` and `EMAIL_ACCESS_KEY`, despite being the variable + reference (`e66eeef8`, `c2754d28`). + +**Changed** + +- Secrets report presence only — `SIGNATURE`, `DATOMIC_PASSWORD`, `EMAIL_ACCESS_KEY` and + `EMAIL_SECRET_KEY` show `set` or `NOT SET`, never a value, dropped before printing rather + than at print time (`c2754d28`). +- `ORCPUB_PDF_MAX_CARDS` defaults to `198` rather than `200`. Nine cards to a sheet, so 200 + was 22 sheets plus two cards stranded on a twenty-third (`ac886c74`). +- `ORCPUB_HTTP_MAX_THREADS` reports the pool size read back off the running server when it is + unset, rather than a formula copied from Pedestal that would drift (`cf7d8cdd`). + +### test/overlay-probe-user-menu-and-modals + +**Added** + +- The overlay probe drives the **import conflict modal** — the fixture pack is + imported twice, the second time with its source renamed so all 180 keys collide, + which is the conflict a user hits when two packs overlap (`26f8c602`). +- A **signed-in lane** that logs in through the real form when `ORCPUB_TEST_USER` + and `ORCPUB_TEST_PASSWORD` are set, and audits the user menu (`26f8c602`). +- The overlay probe walks **My Content's delete-all guard** — the quiet `Delete…` + button, the `.mc-liftpop` it unfurls, and the `.mc-confirmbar` underneath. It + cancels at the last step and then asserts the stored library is the same size it + was, so a run can never be one stray click from wiping a library. None of the + three steps is a modal, so a probe that waits on a modal selector skips. + +**Fixed** + +- **The import helper targets the labels that exist**, most specific first, so a + rename fails loudly instead of quietly matching something else (`950ab526`). +- **The cookie-banner fallback is scoped to the banner**, so it can no longer + close the release panel and record that as consent (`950ab526`, `031daf73`). + +### fix/message-banner-spacing + +**Fixed** + +- **Message banners keep their line breaks**, so a multi-part message reads as + separate lines instead of sentences run together with no punctuation (`2e13d2b2`). +- **The banner has room to breathe** — padding above it as well as below, a gap + between the text and the close icon, and the icon aligned to the first line + rather than halfway down a three-line message (`2e13d2b2`). + +### consolidate/notifications-and-exports + +**Highlights** + +Homebrew can now be rescued from a broken app. The boot shell carries a download +control that the app removes once it has actually rendered, so a missing bundle, +a failed init or a component that throws all leave it standing — and the error +screen puts it back. It reads storage at the moment you click it, not at page +load, so work done during the session is in the file. + +**Added** + +- **Homebrew rescue in the boot shell** — downloads what is stored when the app + itself cannot. Present by default and removed on a clean render, so nothing has + to detect the failure (`46b6f807`, `7bd0d7bc`). +- **Developer mode** — a named switch in the footer reveals "Dump library" and + "Debug info", off by default and remembered per device. Replaces two unlabelled + icons that sat on every page (`c1dad978`). +- **Export coverage for the save banner's link** — a source holding one item from + before the session and one saved seconds ago must export with both, and the file + must re-import into a clean store (`c1dad978`, `cb40740e`). + +**Fixed** + +- **Banners render markup again** — every message built from hiccup had been + rendering as its own source text since `e13d2b2f`, which removed every control + inside one, including the save banner's export link (`68c9873e`). +- **Per-source export runs the same checks as Export All** — the same source no + longer produces two different files depending on which button you press + (`f7b259fe`). +- **The size a rescue reports** — anything under a kilobyte read as "0 KB", which + looks like there is nothing to save (`7ccc9d3a`). + +**Changed** + +- **The footer's raw library dump is named and explained** — it stays reachable in + production, because it is the way out when validation is what is broken + (`17658bb5`, `c1dad978`). + +### fix/item-builder-save-label + +**Fixed** + +- **The item builder's save button says "Save Item"** rather than claiming browser + storage it does not use. + +### refactor/banner-parts-and-design + +**Changed** + +- **Import messages are structured** — `format-import-result` returns + `{:title :details}`, one idea per line, and the banner renders each line itself. + No newline sentinels, no `white-space: pre-line`, no regex collapsing blank + lines. A plain string is still accepted and split at the edge, so the remaining + legacy callers keep working while they move over. +- **The banner has a hierarchy** — a severity icon carrying the colour, a headline, + supporting detail at a lighter weight, a tinted box instead of a filled slab, + and a dismiss with a real touch target. +- **A successful import reads as success** — green, not warning-orange. +- **The banner says what happened and stops there.** The advice line — "To be + safe, export all content now" — is gone rather than promoted to a button: on My + Content it would have sat directly above the page's own Export All. A message + pointing at a control already on screen is noise. + +- **The save-validation error leads with the problem.** It opened with a line + naming the builder you were already standing in ("Spell:"), then the problem, + then the escape hatch — three stacked blocks for one short message. The problem + is the headline now, with the rest beneath it. +- **Exporting from the storage warning no longer closes the card.** The banner + dismisses on any click that reaches it, and the export link did not stop its + own, so the surface vanished mid-action — which is how a working control comes + to feel broken. +- **The browser-storage warning leads with the point.** It was one + 200-character sentence opening with "IMPORTANT!:" and ending with the export + link — the part that matters, last. Now: "Spell saved — in this browser only", + and under it "Clearing browser data loses it. Export this source to keep a copy." +- **The banner is sized to what it says** rather than to one fixed width. A flat + cap fixed the five-word slab and then squeezed a two-sentence warning into a + narrow column; `fit-content` with a ceiling does both jobs. +- **A save confirmation says what it saved** — "Saved “Flame Tongue”" rather than + "Your item has been saved.", for characters too when they have a name. + +**Fixed** + +- **A message that is markup renders as markup.** The builders' "please fill in + X" carries a bolded field name and a clickable "Save anyway with placeholders", + and the new banner ran `str` over it, printing the hiccup at the reader. Vectors + now pass through untouched. + +- **Callout action buttons carry a React key again** — the key was attached to the + `let` form rather than the element it returns, so every callout with actions + logged a missing-key warning. + +### fix/my-content-source-toolbar + +**Changed** + +- **An expanded source's search sits inline with its buttons** — search, show + disabled, then Export and Delete on one row, with side padding so nothing is + flush against the panel. On a phone the search takes the row and the toggle and + buttons wrap beneath it. + +### fix/filtered-list-staleness + +**Fixed** + +- **A filtered list keeps up with its contents again.** Creating, editing or + deleting a custom item left the visible list unchanged while any filter was + active, because the list read a snapshot written at the last keystroke and + nothing ever cleared it. Filters now derive from the live data (`27a92457`). +- **A 401 on custom items leaves a breadcrumb.** The request still fails + quietly by design; it no longer fails invisibly (`b7bd29df`). + +**Changed** + +- **`get-auth-token` lives in `event_utils.cljc`** instead of being redefined + where it was needed (`dbb71dce`). +- **Five API-backed subscriptions share one `reg-api-sub`** rather than + repeating the same fetch-and-store shape (`f1952497`). +- **The orphaned `::mi5e/remote-item` chain is switched off** with `#_` rather + than left looking live. Nothing subscribed to it; the item page that ships + uses `::mi/custom-item` (`7af6253b`). + +**Tests** + +- Thirteen tests cover filtered-list reactivity, two of which fail against the + old shape, plus the share overlay and eight regression guards (`436284ad`). + +### port/save-gate + +**Highlights** + +Homebrew edits can no longer quietly lose data. Changes made in the app go through one check that +keeps links intact or says what a deliberate delete breaks, merges edits made in two tabs, and asks +before touching another pack, and characters keep their homebrew through renames on every page. + +**Added** + +- **One gate for the app's library writes** — saves, imports, moves, restores and key changes pass + one check. A save, move or key change that would strand a link is refused; a deliberate delete or + a re-import goes ahead and says which links it left pointing at nothing. The server's first-load + fetch writes only into an empty library (`31329bcd`, `5209eb97`, `4f7ebf3b`). +- **Two tabs no longer overwrite each other** — a write from a tab holding an old copy is merged onto + the newer library, and a real conflict is reported instead of lost (`bf1ca6a5`, `f3578592`, + `4f7ebf3b`). +- **Links follow a key change** — changing an item's key moves the links in its own pack and offers + to move the ones in other packs; those move only when the author clicks (`1042c746`, `b5f2a1b6`, + `7c199b20`). +- **A copy of the library before its one-time tidy**, with Restore on My Content (`c0e38458`). +- **Repairs for links a past rename left behind** — My Content marks items that link to nothing and + suggests the renamed target; nothing changes until the author picks (`8e76bbf3`, `c0e38458`). +- **A character is asked which item it meant** when an import renames an item it used, once, in a + banner that waits for an answer (`c50e6536`). +- **Asking before deleting what other items use**, and saying what a re-import updated (`12eb5a23`). +- **A check on comments and docstrings** — a test flags history, long docstrings and long comment + blocks; each hit is fixed or approved with a reason, and field notes are kept by hash + (`90334b7a`, `d34bdb2f`, `76395a16`, `48f945b3`, `332ae1dd`). + +**Fixed** + +- **Saves land on the item the builder opened** — a stale or ambiguous address is refused rather + than guessed, and an older untagged entry is replaced only with consent (`b5b36d29`, `b9a98f4f`, + `4968a3be`). +- **Characters keep homebrew renamed since they were saved** — picks heal on every page, not only in + the builder, and a key two kinds of content share (the Dragonborn "Blue" ancestry and a background + picked as `:blue`) no longer blocks the heal (`43f93777`, `38dbaa91`). +- **The missing-content warning** also reports missing spells and languages, and a background whose + key another type holds (`4e88b04c`, `43f93777`). +- **"Link to nothing" is judged by content type** — a built-in language `Orc` no longer answers a + race's link to `:orc` (`7c199b20`, `a376470b`). +- **A move takes an item's subclasses and subraces along** (`3fb06300`, `ab8b0393`). +- **Restore shows what is still set aside, and keeps what the author typed** when its write fails + (`796e15dd`, `6cfccce3`). +- **Every on/off switch on My Content works from the keyboard** (`6cfccce3`). +- **Share links bundle a picked key only as the content types its choice can hold** (`4f7ebf3b`, + `a90fe7c6`). +- **The server's first-load `.orcbrew` fetch** writes only into a library that is still empty + (`4f7ebf3b`). + +**Changed** + +- **Homebrew backgrounds are offered under their stored key**; characters that picked one under its + name's key heal to it (`5209eb97`, `43f93777`). +- **Notices and import modals share one design** — each decision in one framed callout, names + instead of keys, no decorative icons (`b1feb5b3`, `7ae78fc6`). + +### fix/comment-debt + +**Fixed** + +- **Party add/remove answer with the updated party** — both returned the party as it was before the change, because they pulled from the request's snapshot. The app ignored the body, so nothing visible broke; a new test covers both (`d09964ce`). +- **Stale comments and docstrings corrected against the code** — among them the CSP notes that described a Report-Only dev policy that no longer exists, and PDF, spell-packing and share-link docstrings that named the wrong return shape (`ac0dc950`). + +**Changed** + +- **Comments and docstrings trimmed to spec** across 47 source files, clearing the comment check's recorded debt; history and measurements moved to the knowledge base (`ac0dc950`). +- **The linter's config is tracked** — `.gitignore` now ignores the linter's cache, not `.clj-kondo/config.edn` (`ac0dc950`). ## [breaking/2026-stack-modernization] diff --git a/dev/compare_item_cards.clj b/dev/compare_item_cards.clj new file mode 100644 index 000000000..b3861a76e --- /dev/null +++ b/dev/compare_item_cards.clj @@ -0,0 +1,47 @@ +;; Renders every card frame family across the five rarities, then the name +;; treatments, so the alternatives can be looked at instead of argued about. +;; +;; lein run -m clojure.main dev/compare_item_cards.clj +;; OUT=/tmp/styles.pdf lein run -m clojure.main dev/compare_item_cards.clj +;; +;; PDFBox's renderer cannot rasterise the embedded faces -- open the result in a +;; browser, not with PDFRenderer, or every card looks like an encoding bug. + +(require '[orcpub.pdf :as pdf] '[orcpub.dnd.e5.magic-items :as mi]) +(import '[org.apache.pdfbox.pdmodel PDDocument PDPage PDPageContentStream] '[java.io File]) + +(def rarities [:common :uncommon :rare :very-rare :legendary]) + +(defn sample + "One short item per rarity, so the frames are compared and not the prose." + [r] + (let [base (->> (vals mi/magic-item-map) + (filter #(= r (:orcpub.dnd.e5.magic-items/rarity %))) + (filter :orcpub.dnd.e5.magic-items/description) + (sort-by #(count (:orcpub.dnd.e5.magic-items/description %))) + first)] + (assoc base :orcpub.dnd.e5.magic-items/description + (str "A " (name r) " item, shown to compare the frame it is given. " + "The words are the same on every card so the decoration is what differs.")))) + +(defn page! [doc fonts img items opts] + (let [page (PDPage.)] + (.addPage doc page) + (with-open [cs (PDPageContentStream. doc page)] + (pdf/print-items cs doc fonts img 2.5 3.5 items 0 false false opts)))) + +(let [doc (PDDocument.) + fonts (pdf/load-fonts doc) + img (pdf/make-image-loader doc) + ladder (mapv sample rarities)] + ;; One page per family: the five ranks side by side. + (doseq [family pdf/card-flourishes] + (page! doc fonts img ladder {:flourish family})) + ;; Name treatments, all on the winning-ish frame. + (doseq [[face size track] [[:bold 12 0.15] [:bold 13.5 0.3] + [:bold-italic 12.5 0.2] [:plain 13 0.55]]] + (page! doc fonts img (mapv sample [:rare :legendary :uncommon]) + {:flourish :diamonds :name-face face :name-size size :name-tracking track})) + (.save doc (File. (or (System/getenv "OUT") "target/item-card-styles.pdf"))) + (.close doc) + (println "families:" (pr-str pdf/card-flourishes) "pages:" (+ (count pdf/card-flourishes) 4))) diff --git a/dev/e2e_boot.clj b/dev/e2e_boot.clj new file mode 100644 index 000000000..f410e45ce --- /dev/null +++ b/dev/e2e_boot.clj @@ -0,0 +1,95 @@ +(ns e2e-boot + "Boots the real server against an in-memory Datomic db and seeds a verified + user, in ONE JVM -- a mem:// database only exists inside the process that + created it, so seeding from a separate `lein run` would talk to a different, + empty database. + + Seeds TWO verified users, each owning custom items, because isolation is not + observable with one: a query that returned everybody's items would look exactly + like a correct one. kaylee/serenity99 owns three, zoe/washburne7 owns one. + + Used by scripts/e2e/run.sh. Not part of the production uberjar." + (:require [com.stuartsierra.component :as component] + [datomic.api :as d] + [orcpub.system :as s] + [orcpub.routes :as routes] + [orcpub.entity.strict :as se] + [orcpub.dnd.e5.character :as char5e] + [user :as dev])) + +(defn- item + "A minimal custom magic item owned by `owner`. Enough attributes to survive + expand-magic-items and to show up in My Items; not a realistic item." + [owner nm] + {:orcpub.dnd.e5.magic-items/name nm + :orcpub.dnd.e5.magic-items/owner owner + :orcpub.dnd.e5.magic-items/type :wondrous-item + :orcpub.dnd.e5.magic-items/rarity :rare + :orcpub.dnd.e5.magic-items/description (str nm " — seeded by e2e-boot.")}) + +(defn- fighter-carrying + "A level 1 fighter with one of its owner's custom items equipped, so a browser check can open a + character whose sheet needs an item from another account." + [character-name item-key] + {::se/summary {::char5e/character-name character-name + ::char5e/classes [{::char5e/class-name "Fighter" ::char5e/level 1}]} + ::se/selections [{::se/key :ability-scores + ::se/option {::se/key :standard-scores + ::se/map-value {::char5e/str 15 ::char5e/dex 14 ::char5e/con 13 + ::char5e/int 12 ::char5e/wis 10 ::char5e/cha 8}}} + {::se/key :class + ::se/options [{::se/key :fighter + ::se/selections [{::se/key :levels + ::se/options [{::se/key :level-1}]}]}]} + {::se/key :other-magic-items + ::se/options [{::se/key item-key}]}]}) + +(defn- knowing-language + "The same fighter, also knowing a homebrew language. The browser suite puts that language in the + owner's local library, which is where homebrew lives; the server never has it." + [character-name item-key language-key] + (update (fighter-carrying character-name item-key) ::se/selections conj + {::se/key :languages ::se/options [{::se/key language-key}]})) + +(defn -main [& _] + ;; The :dev system pins port 8890; run.sh passes E2E_PORT through as PORT. + (let [port (some-> (System/getenv "PORT") Integer/parseInt) + sys (component/start (cond-> (s/system :dev) + port (assoc-in [:service-map :io.pedestal.http/port] port))) + conn (get-in sys [:conn :conn])] + ;; TWO accounts, each with its OWN items. One account cannot show isolation: + ;; a bug that returned every user's items would look identical to correct + ;; behaviour. The names are deliberately distinct per owner so a leak between + ;; them is visible in a list without cross-referencing ids. + (dev/create-user! conn {:username "kaylee" + :email "kaylee@example.com" + :password "serenity99" + :verify? true}) + (dev/create-user! conn {:username "zoe" + :email "zoe@example.com" + :password "washburne7" + :verify? true}) + @(d/transact conn [(item "kaylee" "Kaylee Seeded Alpha") + (item "kaylee" "Kaylee Seeded Beta") + (item "kaylee" "Kaylee Seeded Gamma") + (item "zoe" "Zoe Seeded Only")]) + ;; Printed so a check can open the character without logging in. + (let [{:keys [status body]} (routes/do-save-character (d/db conn) conn + (fighter-carrying "Bree Tinker" :kaylee-seeded-alpha) + {:user "kaylee"})] + (println "E2E-CHARACTER" status (:db/id body))) + (let [{:keys [status body]} (routes/do-save-character (d/db conn) conn + (knowing-language "Wren Holloway" :kaylee-seeded-beta :e2e-cant) + {:user "kaylee"})] + (println "E2E-HOMEBREW-CHARACTER" status (:db/id body))) + ;; A character whose last share link expired, as prune! leaves it: a note of the character and the date. + ;; Ten days ago, well inside the window, so the heartbeat's first prune keeps the note. + (let [{:keys [status body]} (routes/do-save-character (d/db conn) conn + (knowing-language "Ash Marlow" :kaylee-seeded-gamma :e2e-cant) + {:user "kaylee"})] + @(d/transact conn [{:orcpub.share-expiry/character (:db/id body) + :orcpub.share-expiry/on (java.util.Date. (- (System/currentTimeMillis) (* 10 24 60 60 1000)))}]) + (println "E2E-EXPIRED-CHARACTER" status (:db/id body))) + (println "E2E-READY") + (flush) + @(promise))) diff --git a/dev/fix_missing_char_fields.clj b/dev/fix_missing_char_fields.clj new file mode 100644 index 000000000..2d14f5cec --- /dev/null +++ b/dev/fix_missing_char_fields.clj @@ -0,0 +1,81 @@ +;; Gives style 3 the hit-dice field its sheet prints a box for, and style 4 the +;; name the app writes to. +;; +;; lein run -m clojure.main dev/fix_missing_char_fields.clj +;; +;; Rewrites the masters in resources/ in place. +;; +;; Style 3 prints a TOTAL / HIT DICE box and has no `hd` field, so a level 20 +;; character's "20d6" had nowhere to go and the box printed empty on every sheet. +;; The new field is placed against the printed TOTAL label, which PDFTextStripper +;; puts at x 238.2, y 454.7 from the foot, rather than against a guess: the box +;; runs from just under that label down to the banner, and the death saves beside +;; it start at x 346.5. +;; +;; Style 4 HAS the second-page name field and calls it character-name-p2. pdf_spec +;; emits character-name-2, which every other style uses, so the value was reported +;; unplaceable and the box printed empty. Renamed rather than aliased -- one name +;; for one thing, and the alias would have to be carried by every caller. + +(require '[orcpub.pdf :as pdf] + '[clojure.java.io :as io]) +(import '[org.apache.pdfbox Loader] + '[org.apache.pdfbox.cos COSName] + '[org.apache.pdfbox.pdmodel.common PDRectangle] + '[org.apache.pdfbox.pdmodel.interactive.annotation PDAnnotationWidget] + '[org.apache.pdfbox.pdmodel.interactive.form PDTextField] + '[java.io File FileOutputStream]) + +(declare add-field!*) + +(defn- field-named [doc nm] + (.getField (.getAcroForm (.getDocumentCatalog doc)) nm)) + +(defn- add-field! + "A text field `nm` at `rect`, taking its styling from `model-name`'s widget. + + /AP is left off deliberately, as everywhere else that builds a field here: + write-fields! bakes an appearance from the value, and a shared /AP would make + this field render whatever the model last rendered." + [doc model-name nm ^PDRectangle rect] + (if (field-named doc nm) + (printf " %s already exists, not added again%n" nm) + (add-field!* doc model-name nm rect))) + +(defn- add-field!* [doc model-name nm ^PDRectangle rect] + (let [form (.getAcroForm (.getDocumentCatalog doc)) + model (field-named doc model-name) + mw (first (.getWidgets model)) + page (.getPage mw) + field (PDTextField. form) + widget (PDAnnotationWidget.)] + (.setPartialName field nm) + (doseq [k [COSName/DA COSName/MK COSName/F COSName/FT]] + (when-let [v (.getDictionaryObject (.getCOSObject mw) k)] + (.setItem (.getCOSObject widget) k v))) + (.setRectangle widget rect) + (.setPage widget page) + (.setWidgets field (java.util.ArrayList. [widget])) + (.setAnnotations page (java.util.ArrayList. + (conj (vec (.getAnnotations page)) widget))) + (.setFields form (java.util.ArrayList. (conj (vec (.getFields form)) field))) + (printf " added %s at x=%.1f y=%.1f w=%.1f h=%.1f%n" + nm (.getLowerLeftX rect) (.getLowerLeftY rect) + (.getWidth rect) (.getHeight rect)))) + +(defn- fix! [style f] + (let [{:keys [file]} (get pdf/sheet-masters style)] + (printf "style %d %s%n" style file) + (with-open [doc (Loader/loadPDF (.readAllBytes (.openStream (io/resource file))))] + (f doc) + (with-open [o (FileOutputStream. (File. (str "resources/" file)))] + (.save doc o))))) + +(fix! 3 (fn [doc] + (add-field! doc "hp-temp" "hd" (PDRectangle. 236.0 418.0 60.0 30.0)))) + +(fix! 4 (fn [doc] + (if-let [f (field-named doc "character-name-p2")] + (do (.setPartialName f "character-name-2") + (println " character-name-p2 -> character-name-2")) + (println " character-name-p2 ABSENT, nothing renamed")))) diff --git a/dev/fix_spell_row_fields.clj b/dev/fix_spell_row_fields.clj new file mode 100644 index 000000000..2b399b2f3 --- /dev/null +++ b/dev/fix_spell_row_fields.clj @@ -0,0 +1,122 @@ +;; Makes every printed spell row reachable by the field the app writes to. +;; +;; lein run -m clojure.main dev/fix_spell_row_fields.clj +;; +;; Rewrites the masters in resources/ in place. +;; +;; Two different faults, both of which silently lost a spell. pdf_spec emits +;; spells-LEVEL-ROW-1 counting from 1 with no gaps, so a template whose fields +;; skip a number drops the value at the gap and leaves its highest row blank. +;; +;; Styles 1 and 3, level 3: THIRTEEN fields for thirteen printed rows, numbered +;; 1-10, 12, 13, 14. Nothing is missing from the page -- the names are just +;; wrong, so spells-3-11 went nowhere and spells-3-14 was never written. Renamed +;; in ascending order, which keeps each target free as it is reached. +;; +;; Style 4, level 2: ELEVEN fields for thirteen printed rows, numbered 1-6 and +;; 9-13, with a 42.3pt hole between rows 6 and 9 -- three row-pitches where two +;; rows are drawn and have nothing on top of them. Two fields are added there. +;; +;; Style 1's PREPARED checkboxes carry the level 3 misnumbering too, in the +;; column beside those same rows -- the only style with the checkboxes at all. +;; Under print-prepared-spells? the tick for row 11 went nowhere and row 14's was +;; never set, so a prepared spell printed unticked. +;; +;; The clones made at export copy the master's fields, so fixing the master +;; fixes every generated page with it. + +(require '[orcpub.pdf :as pdf] + '[clojure.java.io :as io]) +(import '[org.apache.pdfbox Loader] + '[org.apache.pdfbox.cos COSName] + '[org.apache.pdfbox.pdmodel.common PDRectangle] + '[org.apache.pdfbox.pdmodel.interactive.annotation PDAnnotationWidget] + '[org.apache.pdfbox.pdmodel.interactive.form PDTextField] + '[java.io File FileOutputStream]) + +(declare add-row!*) + +(defn- field-named [doc nm] + (.getField (.getAcroForm (.getDocumentCatalog doc)) nm)) + +(defn- rename! + "Renames `from` to `to`, ascending so each target is free when reached. + + Refuses when `to` already exists. These renames SHIFT a run down by one, so a + second pass over an already-fixed file finds the next field sitting under the + old name and renames it too -- which produced two fields called spells-3-11-1 + and lost a row. Two fields of one name are one field with one value." + [doc from to] + (cond + (field-named doc to) + (do (printf " %s already exists, nothing renamed%n" to) false) + + (field-named doc from) + (do (.setPartialName (field-named doc from) to) + (printf " %s -> %s%n" from to) + true) + + :else + (do (printf " %s ABSENT, nothing renamed%n" from) false))) + +(defn- add-row! + "A text field named `nm` on `model`'s row pitch, `steps` rows below it. + + Copies the model widget's geometry and styling entries and moves the copy down + -- the same set add-spell-pages! copies for a cloned page, and for the same + reason: /AP is left off so write-fields! bakes an appearance from the value + rather than sharing the model's." + [doc model-name nm steps pitch] + (if (field-named doc nm) + (printf " %s already exists, not added again%n" nm) + (add-row!* doc model-name nm steps pitch))) + +(defn- add-row!* [doc model-name nm steps pitch] + (let [form (.getAcroForm (.getDocumentCatalog doc)) + model (field-named doc model-name) + mw (first (.getWidgets model)) + page (.getPage mw) + r (.getRectangle mw) + field (PDTextField. form) + widget (PDAnnotationWidget.)] + (.setPartialName field nm) + (doseq [k [COSName/DA COSName/MK COSName/F COSName/FT]] + (when-let [v (.getDictionaryObject (.getCOSObject mw) k)] + (.setItem (.getCOSObject widget) k v))) + (.setRectangle widget (PDRectangle. (.getLowerLeftX r) + (- (.getLowerLeftY r) (* steps pitch)) + (.getWidth r) + (.getHeight r))) + (.setPage widget page) + (.setWidgets field (java.util.ArrayList. [widget])) + (.setAnnotations page (java.util.ArrayList. + (conj (vec (.getAnnotations page)) widget))) + (.setFields form (java.util.ArrayList. (conj (vec (.getFields form)) field))) + (printf " added %s at y=%.1f%n" nm (.getLowerLeftY (.getRectangle widget))))) + +(defn- fix! [style f] + (let [{:keys [file]} (get pdf/sheet-masters style) + out (File. (str "resources/" file))] + (printf "style %d %s%n" style file) + (with-open [doc (Loader/loadPDF (.readAllBytes (.openStream (io/resource file))))] + (f doc) + (with-open [o (FileOutputStream. out)] (.save doc o))))) + +(doseq [style [1 3]] + (fix! style (fn [doc] + (doseq [[from to] [["spells-3-12-1" "spells-3-11-1"] + ["spells-3-13-1" "spells-3-12-1"] + ["spells-3-14-1" "spells-3-13-1"]]] + (rename! doc from to))))) + +;; 14.1pt: (145.7 - 103.4) / 3, the pitch across the hole rather than a guess. +(fix! 4 (fn [doc] + (add-row! doc "spells-2-6-1" "spells-2-7-1" 1 14.1) + (add-row! doc "spells-2-6-1" "spells-2-8-1" 2 14.1))) + +;; The prepared column beside style 1's level 3 rows, misnumbered the same way. +(fix! 1 (fn [doc] + (doseq [[from to] [["prepared-3-12-1" "prepared-3-11-1"] + ["prepared-3-13-1" "prepared-3-12-1"] + ["prepared-3-14-1" "prepared-3-13-1"]]] + (rename! doc from to)))) diff --git a/dev/inspect_export.clj b/dev/inspect_export.clj new file mode 100644 index 000000000..aaa16f69b --- /dev/null +++ b/dev/inspect_export.clj @@ -0,0 +1,123 @@ +(ns inspect-export + "Asserts the invariants of an exported character sheet. + + lein with-profile init-db run -m clojure.main dev/inspect_export.clj + + scripts/e2e/run.sh calls this on the PDF the browser produced. The browser + script can only check status, content type and size: PDF field names live in + compressed object streams, so they cannot be read without a parser, and node + has none here. This runs where PDFBox already is. + + Exits non-zero on any failure so the e2e run fails with it." + (:require [clojure.string :as str]) + (:import (org.apache.pdfbox Loader) + (java.io File))) + +(defn- report [ok? label detail] + (println (format " %s %s%s" (if ok? "ok " "FAIL") label + (if detail (str " - " detail) ""))) + ok?) + +(def ^:private still-unnamed + "Template names the preparation passes are meant to have replaced. The suffix + is what split-fields-across-pages! appends to a copy, and it has to be part of + the pattern or a copy that no pass renamed reads as named." + #"(?i)(check box|slotsremaining) \d+(-p\d+)?") + +(defn -main [& [path min-pages]] + (let [file (File. (or path "/tmp/e2e-pdf/character.pdf")) + wanted (when min-pages (Integer/parseInt min-pages))] + (when-not (.exists file) + (println " FAIL no exported PDF at" (.getPath file)) + (System/exit 1)) + (with-open [doc (Loader/loadPDF (.readAllBytes (java.io.FileInputStream. file)))] + (let [form (.getAcroForm (.getDocumentCatalog doc)) + ;; the field tree, not .getFields: that returns only root fields, so + ;; anything nested would go unchecked + fields (if form (iterator-seq (.iterator (.getFieldTree form))) []) + names (map #(.getFullyQualifiedName %) fields) + valued (into {} (for [f fields + :let [v (str (.getValueAsString f))] + :when (not (str/blank? v))] + [(.getFullyQualifiedName f) v])) + page-numbers (into {} (map-indexed (fn [i p] [p (inc i)]) (.getPages doc))) + pages-of (fn [f] (into #{} (keep #(page-numbers (.getPage %)) (.getWidgets f)))) + orphans (for [f fields w (.getWidgets f) :when (nil? (.getPage w))] w) + duplicates (->> names frequencies (filter #(> (val %) 1)) (map key)) + unnamed (filter #(re-matches still-unnamed %) names) + ;; a field with widgets on two pages is ONE value shown twice, so + ;; ticking it on one class's page ticks it on another's + spanning (filter #(> (count (pages-of %)) 1) fields) + + ;; spellcasting sections, in page order + sections (->> fields + (filter #(re-matches #"spellcasting-class-\d+" + (.getFullyQualifiedName %))) + (keep (fn [f] + (let [v (str (.getValueAsString f))] + (when-not (str/blank? v) + {:suffix (subs (.getFullyQualifiedName f) + (count "spellcasting-class-")) + :page (first (sort (pages-of f))) + :heading v + :class (str/replace v #"\s*\(continued\)$" "") + :continued? (str/ends-with? v "(continued)")})))) + (sort-by :page)) + slots-on (fn [suffix] + (for [level (range 1 10) + :let [f (.getField form (str "spell-slots-" level "-" suffix))] + :when (and f (not (str/blank? (str (.getValueAsString f)))))] + level)) + ;; a class's first page carries its slots; a later page of the same + ;; class must be marked and must not repeat them + mismarked (let [seen (volatile! #{})] + (doall + (for [s sections + :let [repeat? (contains? @seen (:class s))] + :when (do (vswap! seen conj (:class s)) + (not= repeat? (:continued? s)))] + (str "page " (:page s) " " (pr-str (:heading s)) + (if repeat? " is a repeat but unmarked" " is marked but is the first"))))) + repeated-slots (for [s sections + :when (and (:continued? s) (seq (slots-on (:suffix s))))] + (str "page " (:page s) " repeats levels " + (str/join "," (slots-on (:suffix s))))) + results + (concat + [(report (>= (.getNumberOfPages doc) (or wanted 1)) + (if wanted + (str "the sheet has at least " wanted " pages") + "the sheet has pages") + (str (.getNumberOfPages doc))) + (report (some? form) "the form survived" nil) + (report (> (count valued) 40) "fields carry the character's values" + (str (count valued) " of " (count fields))) + (report (not (str/blank? (get valued "class-level" ""))) "class and level are filled" + (get valued "class-level")) + (report (empty? orphans) "no widget belongs to no page" + (when (seq orphans) (str (count orphans) " orphaned"))) + (report (empty? duplicates) "no field name is shared" + (when (seq duplicates) (str/join ", " (take 4 duplicates)))) + (report (empty? unnamed) "every field carries a meaningful name" + (when (seq unnamed) (str (count unnamed) " left as in the template: " + (str/join ", " (take 3 unnamed))))) + (report (empty? spanning) "no field spans two pages" + (if (seq spanning) + (str (count spanning) " would mirror, e.g. " + (str/join ", " (take 3 (map #(.getFullyQualifiedName %) spanning)))) + "nothing mirrors between class pages"))] + (when (seq sections) + [(report (every? #(not (str/blank? (:class %))) sections) + "every spellcasting section names its class" + (str/join " / " (map :heading sections))) + (report (empty? mismarked) "a class's later pages are marked continued" + (if (seq mismarked) (str/join "; " mismarked) + (str (count (distinct (map :class sections))) " class(es) over " + (count sections) " page(s)"))) + (report (empty? repeated-slots) "slots appear only on a class's first page" + (when (seq repeated-slots) (str/join "; " repeated-slots)))]))] + (println (format "\n %d/%d checks passed on the exported PDF" + (count (filter true? results)) (count results))) + (System/exit (if (every? true? results) 0 1)))))) + +(apply -main *command-line-args*) diff --git a/dev/measure_item_card.clj b/dev/measure_item_card.clj new file mode 100644 index 000000000..26b0e2c48 --- /dev/null +++ b/dev/measure_item_card.clj @@ -0,0 +1,129 @@ +;; Builds the worst magic item card the layout can be asked for -- the longest +;; name, the longest attunement clause, a charge track, a legendary frame and a +;; description that runs onto the back -- renders it, and reports the gap between +;; every pair of stacked elements. +;; +;; lein run -m clojure.main dev/measure_item_card.clj +;; OUT=/tmp/torture.pdf lein run -m clojure.main dev/measure_item_card.clj +;; +;; Reading the gaps beats looking at the card: a sixteenth of an inch is invisible +;; on screen and obvious in the hand. Anything under MIN-GAP is reported as tight +;; so it can be fixed once rather than found later one card at a time. +;; +;; PDFBox's renderer cannot rasterise the embedded faces -- open the PDF in a +;; browser, or every card will look like a text encoding bug. + +(require '[orcpub.pdf :as pdf] '[clojure.string :as str]) +(import '[org.apache.pdfbox.pdmodel PDDocument PDPage PDPageContentStream] '[java.io File]) + +(def MIN-GAP + "Least clear space between two stacked elements before it reads as crowded." + 0.06) + +(def torture + "Everything at once, so nothing can hide behind an easy case." + {:name "Instrument of the Bards, Anstruth Harp of Deepest Sorrow" + :orcpub.dnd.e5.magic-items/name "Instrument of the Bards" + :orcpub.dnd.e5.magic-items/type :wondrous-item + :orcpub.dnd.e5.magic-items/subtype :instrument + :orcpub.dnd.e5.magic-items/rarity :legendary + ;; The longest attunement list in the data. + :orcpub.dnd.e5.magic-items/attunement [:bard :cleric :druid :sorcerer :warlock :wizard] + :orcpub.dnd.e5.magic-items/description + (str "This instrument has 12 charges and can be played while you are holding " + "it. It has a spell save DC of 18. You can use an action to play it and " + "expend 1 charge to cast one of the spells written on it. The instrument " + "regains 1d4 + 4 expended charges daily at dawn. " + (str/join " " (repeat 14 "The harp answers only to a hand that has known grief.")))}) + +(defn- text-span + "Where a run of text actually puts ink, given the y its box is anchored at. + + Lines are set a leading below the anchor and downward from there, so the + anchor itself is empty. Ink reaches about seven tenths of the size above a + baseline and two tenths below it. Comparing anchors instead of ink is how a + layout comes to look cramped while every number in it says it is fine." + [anchor size lines] + (let [leading (/ (* size 1.1) 72) + first-base (+ anchor leading) + last-base (+ anchor (* leading lines))] + [(- first-base (* 0.72 (/ size 72))) + (+ last-base (* 0.22 (/ size 72)))])) + +(defn- rule-span [at] [at at]) +(defn- mark-span [centre r] [(- centre r) (+ centre r)]) + +(defn elements + "The card's stack as [label ink-top ink-bottom], for the worst case." + [{:keys [down up]} h name-size name-lines body-lines] + [["frame (top)" (rule-span (:frame down))] + ["rarity rail" (mark-span (:rail down) 0.038)] + ["name" (text-span (condp = name-lines + 1 (:name-one-line down) + 2 (:name-two-line down) + (:name-three-line down)) + name-size name-lines)] + ;; The badge sits BESIDE the subtitle, right-aligned on the same row, so it + ;; is not part of the vertical stack. Its span is folded into the subtitle's. + ["subtitle + badge" (let [[t b] (text-span (:subtitle down) 7.5 1)] + [(min t (:badge down)) + (max b (+ (:badge down) 0.145))])] + ["header rule" (rule-span (:rule down))] + ["charge label" (text-span (- (:charge-label down) 0.084) 5.5 1)] + ["charge marks" (mark-span (:charge-marks down) 0.052)] + ["charge rule" (rule-span (:charge-under down))] + ["description" (text-span (:body-charged down) 8 body-lines)] + ["continued note" (text-span (- (- h (:continued up)) 0.095) 6.2 1)] + ["attunement clause" (text-span (- h (:clause up)) 6.8 2)] + ["foot ornament" (mark-span (- h (:ornament up)) 0.032)] + ["frame (bottom)" (rule-span (- h (:frame down)))]]) + +(defn report + "Prints the stack with the clear space between one element's ink and the next." + [rows] + (let [tight (atom 0)] + (printf "%-22s %7s %7s %8s%n" "element" "top" "bottom" "gap") + (doseq [[[l1 [t1 b1]] [_ [t2 _]]] (map vector rows (rest rows))] + (let [gap (- t2 b1)] + (when (< gap MIN-GAP) (swap! tight inc)) + (printf "%-22s %7.3f %7.3f %8.3f%s%n" l1 t1 b1 gap + (cond (neg? gap) " OVERLAP" + (< gap MIN-GAP) " tight" + :else "")))) + (let [[l [t b]] (last rows)] + (printf "%-22s %7.3f %7.3f%n" l t b)) + (printf "%ngaps under %.2fin: %d%n" MIN-GAP @tight))) + +(let [doc (PDDocument.) page (PDPage.)] + (.addPage doc page) + (let [fonts (pdf/load-fonts doc) img (pdf/make-image-loader doc)] + (with-open [cs (PDPageContentStream. doc page)] + (pdf/print-items cs doc fonts img 2.5 3.5 [torture torture torture] 0 false false nil))) + (.save doc (File. (or (System/getenv "OUT") "target/item-card-torture.pdf"))) + (.close doc)) + +(println "charges parsed:" (pdf/item-charges (:orcpub.dnd.e5.magic-items/description torture))) +(println "subtitle :" (pdf/magic-item-subtitle torture)) +(println "clause :" (pdf/attunement-phrase + (:orcpub.dnd.e5.magic-items/attunement torture) nil)) +(println) +;; The name shrinks to fit two lines, so ask the renderer what size it settled on +;; rather than assuming the nominal one. +(let [doc (PDDocument.) + fonts (pdf/load-fonts doc) + width (- 2.5 0.56) + lines-at (fn [pt] (count (#'pdf/split-lines (:name torture) (:bold fonts) pt width))) + floor (get-in pdf/card-layout [:down :name-floor]) + height (get-in pdf/card-layout [:down :name-height]) + fits? (fn [pt] (let [n (lines-at pt)] + (and (<= n 3) (<= (* n (/ (* pt 1.1) 72)) height)))) + size (or (first (filter fits? (take-while #(>= % floor) (iterate #(- % 0.5) 13)))) + floor) + body-lines (count (#'pdf/split-lines + (:orcpub.dnd.e5.magic-items/description torture) + (:plain fonts) 8 (- 2.5 0.4))) + capacity (int (dec (/ (* 72 (- 3.5 1.74 0.80)) (* 8 1.1))))] + (printf "name set at %.2fpt over %d lines; description %d lines, %d fit%n%n" + (double size) (lines-at size) body-lines capacity) + (report (elements pdf/card-layout 3.5 size (lines-at size) (min body-lines capacity))) + (.close doc)) diff --git a/dev/prepare_templates.clj b/dev/prepare_templates.clj new file mode 100644 index 000000000..81699d6da --- /dev/null +++ b/dev/prepare_templates.clj @@ -0,0 +1,101 @@ +(ns prepare-templates + "Bakes the static cleanups into the character sheet templates in resources/. + + lein with-profile init-db run -m clojure.main dev/prepare_templates.clj + + These transforms are pure functions of the template, so doing them per request + repeated identical work on every export. They are applied here once instead, + in this order -- the naming passes recognise a field by the name the template + gave it, and the split hands its copies a suffixed version of that name, so a + split that ran afterwards would leave its copies unnamed: + + pdf/prune-orphan-widgets! drop widgets belonging to no page + pdf/split-fields-across-pages! one field per page, so nothing mirrors + pdf/name-prepared-checkboxes! Check Box 25 -> prepared-1-1-1 + pdf/name-slots-expended! SlotsRemaining 19 -> slots-expended-1-1 + pdf/name-death-save-checkboxes! the six ticks on the character page + pdf/disambiguate-duplicate-fields! anything still sharing a name + pdf/share-checkbox-appearances! one appearance for every identical tick + pdf/share-duplicate-images! one copy of a page image used twice + pdf/add-image-predictors! re-encode Flate images with a predictor + + The per-character passes stay at runtime, since they depend on the character: + pdf/add-missing-spell-pages! and pdf/spill-overflow!. Nothing here runs per + request -- an export parses a baked template, writes values and serialises, + which measures 178 ms for a level 20 wizard with every spell row filled. + + Idempotent: a second run reports nothing to do. Rerun after replacing or + re-cutting any template, and commit the result." + (:require [orcpub.pdf :as pdf] + [clojure.java.io :as io] + [clojure.string :as str]) + (:import (org.apache.pdfbox Loader) + (java.io File FileOutputStream))) + +(defn- field-names [doc] + (if-let [form (.getAcroForm (.getDocumentCatalog doc))] + (map #(.getFullyQualifiedName %) (.getFields form)) + [])) + +(defn- stats [doc] + (let [names (field-names doc)] + {:fields (count names) + :duplicates (->> names frequencies (filter #(> (val %) 1)) count) + ;; the split suffixes its copies, so an unnamed one can read "Check Box 25-p2" + :anonymous (count (filter #(re-matches #"(?i)check box \d+(-p\d+)?" %) names))})) + +(defn- prepare! [^File file] + (let [before-bytes (.length file)] + (with-open [doc (Loader/loadPDF (.readAllBytes (io/input-stream file)))] + (let [before (stats doc) + removed (pdf/prune-orphan-widgets! doc) + ;; before naming: a field whose widgets sit on two pages would keep + ;; the first page's name and mirror onto the second + split (pdf/split-fields-across-pages! doc) + named (+ (pdf/name-prepared-checkboxes! doc) + (pdf/name-slots-expended! doc) + (pdf/name-death-save-checkboxes! doc)) + disambiguated (pdf/disambiguate-duplicate-fields! doc) + ;; last: the split creates checkboxes of its own, and they should + ;; share too + shared (pdf/share-checkbox-appearances! doc) + images (pdf/share-duplicate-images! doc) + predicted (pdf/add-image-predictors! doc) + after (stats doc) + ;; Compared rather than inferred from the return values: a field + ;; carrying no widgets at all is dropped without any widget being + ;; counted, so prune's tally can be 0 on a document it did change. + changed? (not= before after)] + (when changed? + (with-open [out (FileOutputStream. file)] (.save doc out))) + (println (format "%-46s %5d -> %-5d fields %4d KB -> %-5d KB %s" + (.getName file) + (:fields before) (:fields after) + (quot before-bytes 1024) (quot (.length file) 1024) + (if changed? + (format "pruned %d, split %d, named %d, renamed %d, ticks shared %d, images shared %d, images repacked %d" + removed split named disambiguated shared images predicted) + "already clean"))) + ;; Duplicates are a correctness problem -- same name, one shared value -- + ;; so they must all be gone. Anonymous names are only unhelpful, and the + ;; geometric pairing is tuned to the style 1 layout, so other styles keep + ;; some; they are reported rather than treated as a failure. + (assert (zero? (:duplicates after)) + (str (.getName file) " still has duplicate field names")) + {:changed? changed? :anonymous-left (:anonymous after)})))) + +(defn -main [& _] + (let [files (sort-by #(.getName %) + (filter #(re-find #"^fillable-char-sheetstyle-.*\.pdf$" (.getName %)) + (file-seq (io/file "resources")))) + before (reduce + (map #(.length %) files)) + results (doall (map prepare! files)) + anonymous (reduce + (map :anonymous-left results))] + (when (pos? anonymous) + (println (format "\n%d checkboxes keep their 'Check Box N' name: the row pairing is\nshaped for the style 1 layout." anonymous))) + (println (format "\n%d of %d templates changed; %.1f MB -> %.1f MB" + (count (filter :changed? results)) (count files) + (/ before 1048576.0) + (/ (reduce + (map #(.length %) files)) 1048576.0))))) + +(-main) diff --git a/dev/sample_character.clj b/dev/sample_character.clj new file mode 100644 index 000000000..32faa5889 --- /dev/null +++ b/dev/sample_character.clj @@ -0,0 +1,289 @@ +(ns sample-character + "Builds fully-populated characters and writes them through the real export + path, for eyeballing template changes against sheets that have data on them. + + An empty form hides most problems: missing fields, clipped values, fields + sitting over the wrong printed label. Both fixtures fill every spell row the + template offers so the spell pages get judged at their worst case. + + lein with-profile init-db run -m clojure.main dev/sample_character.clj + + Writes target/sample-wizard.pdf (level 20 single-class caster, 1 spell page) + and target/sample-multi.pdf (four 5-level casters, 4 spell pages) + + The second one exists because one spellcasting class only ever exercises the + \"-1\" field suffix. Four classes exercise -1 through -4 and the sheet4 + template, which nothing else here touches. + + Dev tooling; nothing in src depends on it." + (:require [orcpub.pdf :as pdf] + [orcpub.dnd.e5.spells :as spells] + [orcpub.dnd.e5.spell-lists :as sl] + [clojure.java.io :as io]) + (:import (org.apache.pdfbox Loader) + (java.io FileOutputStream))) + +;; A trap worth knowing before editing the field maps below. The template's +;; field NAMES do not describe what the app puts in them: +;; +;; features-and-traits <- the EQUIPPED item list (prints under EQUIPMENT) +;; treasure <- unequipped items and valuables (page 2) +;; features-and-traits-2 <- the actual features, actions and reactions, +;; on the continuation page (prints under +;; FEATURES & TRAITS) +;; equipment <- nothing. pdf_spec never emits this key; the box +;; it sits in carries the coin fields only. +;; +;; +;; Same trap in the ability boxes: `cha` holds the MODIFIER (the big box) and +;; `cha-mod` holds the SCORE (the small circle). Verified against a production +;; export, not inferred. +;; +;; See pdf_spec/equipment-fields and pdf_spec/traits-fields. Fill these the way +;; the app does, or the fixture tests a layout no real character produces. + +;; How many spell-name rows the style 1 spell page provides at each level. +;; Level 3 offers 13 boxes but has no spells-3-11 field; write-fields! reports it +;; as unplaceable on every run. +(def ^:private rows {0 8, 1 12, 2 13, 3 13, 4 13, 5 9, 6 9, 7 9, 8 7, 9 7}) + +(defn- spell-name [k] + (or (:name (get spells/spell-map k)) (name k))) + +(defn- spell-fields + "Fill the page's rows for one spellcasting class, at the levels it can cast. + `suffix` is the 1-based class index the template appends to every name." + [{:keys [list max-spell-level]} suffix] + (let [available (get sl/spell-lists list)] + (into {} + (for [level (range 0 (inc max-spell-level)) + :let [names (sort (map spell-name (get available level)))] + [idx nm] (map-indexed vector (take (get rows level) names))] + [(keyword (str "spells-" level "-" (inc idx) "-" suffix)) nm])))) + +(defn- caster-fields + "The header block and slot row for one spellcasting class." + [{:keys [name-str ability dc attack slots]} suffix] + (merge + {(keyword (str "spellcasting-class-" suffix)) name-str + (keyword (str "spellcasting-ability-" suffix)) ability + (keyword (str "spell-save-dc-" suffix)) (str dc) + (keyword (str "spell-attack-bonus-" suffix)) attack} + (into {} (for [[level n] slots] + [(keyword (str "spell-slots-" level "-" suffix)) (str n)])))) + +(defn- spellcasting-fields [casters] + (apply merge + (map-indexed (fn [i c] + (merge (caster-fields c (inc i)) + (spell-fields c (inc i)))) + casters))) + +;; ─── Fixture 1: level 20 single-class evoker ─────────────────────────────── + +(def ^:private wizard-20 + {:file "target/sample-wizard.pdf" + :template "fillable-char-sheetstyle-1-1-spells.pdf" + :casters [{:name-str "Wizard" :list :wizard :ability "Intelligence" + :dc 19 :attack "+11" :max-spell-level 9 + :slots {1 4, 2 3, 3 3, 4 3, 5 3, 6 2, 7 2, 8 1, 9 1}}] + :fields + {:character-name "Ysolde Vantreaux" :character-name-2 "Ysolde Vantreaux" + :class-level "Wizard 20 (School of Evocation)" :background "Sage" + :player-name "fixture" :race "High Elf" :alignment "Neutral Good" :xp "355,000" + :str "-1" :str-mod "8" :dex "+2" :dex-mod "14" :con "+3" :con-mod "16" + :int "+5" :int-mod "20" :wis "+1" :wis-mod "13" :cha "+0" :cha-mod "10" + :str-save "-1" :dex-save "+2" :con-save "+3" + :int-save "+11" :wis-save "+7" :cha-save "+0" + :int-save-check true :wis-save-check true + :ac "15" :initiative "+2" :speed "30 ft." :hp-max "122" :hp-current "122" + :hp-temp "0" :hd "20d6" :prof-bonus "+6" :passive "11" :inspiration "1" + :arcana "+11" :arcana-check true :history "+11" :history-check true + :investigation "+11" :investigation-check true :insight "+7" :insight-check true + :perception "+1" :medicine "+1" :nature "+5" :religion "+5" + :age "241" :height "5'11\"" :weight "134 lb." + :eyes "Pale grey" :skin "Fair" :hair "Silver, braided" + :weapon-name-1 "Quarterstaff" :weapon-attack-bonus-1 "+5" :weapon-damage-1 "1d6+3 bludgeoning" + :weapon-name-2 "Dagger" :weapon-attack-bonus-2 "+8" :weapon-damage-2 "1d4+2 piercing" + :weapon-name-3 "Fire Bolt (cantrip)" :weapon-attack-bonus-3 "+11" :weapon-damage-3 "4d10 fire" + :gp "1,240" :pp "60" :cp "0" :sp "0" :ep "0" + :personality-traits "Speaks to books as though they can hear her. They occasionally answer." + :ideals "Knowledge withheld is knowledge wasted." + :bonds "The Vantreaux archive burned. She is rewriting it from memory, one volume a year." + :flaws "Assumes she is the smartest person in the room, and is usually right." + :backstory (str "Apprenticed at eleven to a conjurer who did not survive his own summoning. " + "Finished her training alone, on stolen notes.") + :allies "The Candlewrights; Archivist Bell; a copper dragon who resents owing her a favour." + :other-profs "Common, Elvish, Draconic, Celestial, Deep Speech. Calligrapher's supplies." + :features-and-traits-2 (str "Arcane Recovery. Evocation Savant. Sculpt Spells. Potent Cantrip. " + "Empowered Evocation. Overchannel. Spell Mastery. Signature Spells.") + :features-and-traits "Quarterstaff, dagger, component pouch, scholar's pack, spellbook, bedroll, ink and pens." + :treasure "Ring of Spell Storing, Wand of the War Mage +2, Robe of the Archmagi, Ioun Stone (Mastery)" + :attacks-and-spellcasting "Evocation save DC 19. Overchannel 1/long rest."}}) + +;; ─── Fixture 2: warlock 5 / sorcerer 5 / wizard 5 / cleric 5 ─────────────── +;; +;; Caster level for slots is 15 (the three full casters; warlock's pact magic is +;; tracked separately), so slots run to 8th level. But no class here is above +;; level 5, so nothing above 3rd level can be prepared. High slots with empty +;; rows above 3rd is CORRECT for this build, not a bug -- it is exactly the +;; multiclass case worth looking at on a printed sheet. + +(def ^:private multiclass-slots {1 4, 2 3, 3 3, 4 3, 5 2, 6 1, 7 1, 8 1}) +(def ^:private pact-slots {3 2}) + +(def ^:private multi-20 + {:file "target/sample-multi.pdf" + :template "fillable-char-sheetstyle-1-1-spells.pdf" + :grow-to 4 + :casters [{:name-str "Warlock 5 (Great Old One, Pact of the Tome)" :list :warlock + :ability "Charisma" :dc 16 :attack "+8" :max-spell-level 3 + :slots pact-slots} + {:name-str "Sorcerer 5 (Draconic Bloodline)" :list :sorcerer + :ability "Charisma" :dc 16 :attack "+8" :max-spell-level 3 + :slots multiclass-slots} + {:name-str "Wizard 5 (School of Divination)" :list :wizard + :ability "Intelligence" :dc 15 :attack "+7" :max-spell-level 3 + :slots multiclass-slots} + {:name-str "Cleric 5 (Knowledge Domain)" :list :cleric + :ability "Wisdom" :dc 14 :attack "+6" :max-spell-level 3 + :slots multiclass-slots}] + :fields + {:character-name "Corvin Ashgrave" :character-name-2 "Corvin Ashgrave" + :class-level "Warlock 5 / Sorcerer 5 / Wizard 5 / Cleric 5" :background "Charlatan" + :player-name "fixture" :race "Half-Elf" :alignment "Chaotic Neutral" :xp "355,000" + :str "+0" :str-mod "10" :dex "+2" :dex-mod "14" :con "+2" :con-mod "14" + :int "+3" :int-mod "16" :wis "+2" :wis-mod "14" :cha "+4" :cha-mod "18" + :str-save "+0" :dex-save "+2" :con-save "+2" + :int-save "+7" :wis-save "+6" :cha-save "+10" + :int-save-check true :wis-save-check true :cha-save-check true + :ac "13" :initiative "+2" :speed "30 ft." :hp-max "104" :hp-current "104" + :hp-temp "12" :hd "5d8 / 5d6 / 5d6 / 5d8" :prof-bonus "+6" :passive "12" :inspiration "0" + :arcana "+9" :arcana-check true :religion "+9" :religion-check true + :deception "+10" :deception-check true :persuasion "+10" :persuasion-check true + :history "+9" :insight "+8" :investigation "+3" :perception "+2" + :age "34" :height "5'8\"" :weight "150 lb." + :eyes "One black, one pale" :skin "Ashen" :hair "Black, close-cropped" + :weapon-name-1 "Eldritch Blast (2 beams)" :weapon-attack-bonus-1 "+8" :weapon-damage-1 "1d10+4 force each" + :weapon-name-2 "Quarterstaff" :weapon-attack-bonus-2 "+6" :weapon-damage-2 "1d6 bludgeoning" + :weapon-name-3 "Sacred Flame (DC 14)" :weapon-attack-bonus-3 "--" :weapon-damage-3 "2d8 radiant" + :gp "310" :sp "45" :cp "18" :ep "0" :pp "6" + :personality-traits "Collects other people's holy symbols. Will not say why." + :ideals "Every power will answer to someone who asks in enough languages." + :bonds "Owes a patron he has never seen and cannot stop hearing." + :flaws "Cannot leave an offered bargain unexamined, however plainly it is a trap." + :backstory (str "Took orders, broke them, took a pact, broke that too, and studied enough " + "to know he has not gotten away with either. Four powers have a claim on " + "him and none of them have compared notes yet.") + :allies "The Ashgrave estate (in probate); a cleric of Oghma who wants his notes burned." + :other-profs "Common, Elvish, Infernal, Deep Speech, Celestial. Forgery kit, playing cards." + :features-and-traits-2 (str "Pact Magic (2 slots, 3rd level, short rest). Awakened Mind. " + "Entropic Ward. Book of Ancient Secrets. Font of Magic (5 points). " + "Twinned and Quickened Spell. Arcane Recovery. Portent (2 dice). " + "Channel Divinity 1/rest. Knowledge of the Ages. Fey Ancestry.") + :features-and-traits "Component pouch, Book of Shadows, spellbook, holy symbol, forgery kit, three unmatched rings." + :treasure "Rod of the Pact Keeper +1, Pearl of Power, Cloak of Protection" + :attacks-and-spellcasting + (str "Four spell lists, three save DCs. Pact slots (2 x 3rd) recharge on a short rest and are " + "tracked apart from the shared slots. Caster level 15 for slots; no class above 5th, so " + "nothing above 3rd level can be prepared.")}}) + + +;; ─── Fixture 3: eight spellcasting classes, twenty levels ────────────────── +;; +;; bard 2 / cleric 2 / druid 2 / paladin 4 / ranger 4 / sorcerer 2 / warlock 2 / +;; wizard 2. The template provides SIX spellcasting sections (-1 through -6), so +;; two of these eight have nowhere to print. write-fields! skips names it cannot +;; find without saying anything, so they vanish silently -- which is the point of +;; this fixture. build! reports what it dropped. +;; +;; Caster level for shared slots is 14: five full casters at 2 (=10) plus two +;; half casters at 4 (=2 each). Warlock pact magic is separate. But no class is +;; above 4th level, so NOTHING above 1st level can be prepared, while slots run +;; to 7th. An extreme version of the multiclass shape in fixture 2. + +(def ^:private wide-slots {1 4, 2 3, 3 3, 4 3, 5 2, 6 1, 7 1}) + +(defn- caster [name-str list ability dc attack slots] + {:name-str name-str :list list :ability ability :dc dc :attack attack + :max-spell-level 1 :slots slots}) + +(def ^:private multi-8 + {:file "target/sample-eight.pdf" + :template "fillable-char-sheetstyle-1-1-spells.pdf" + :grow-to 6 + :casters [(caster "Bard 2 (College of Lore)" :bard "Charisma" 14 "+6" wide-slots) + (caster "Cleric 2 (Life Domain)" :cleric "Wisdom" 13 "+5" wide-slots) + (caster "Druid 2 (Circle of the Land)" :druid "Wisdom" 13 "+5" wide-slots) + (caster "Paladin 4 (Oath of Devotion)" :paladin "Charisma" 14 "+6" wide-slots) + (caster "Ranger 4 (Hunter)" :ranger "Wisdom" 13 "+5" wide-slots) + (caster "Sorcerer 2 (Wild Magic)" :sorcerer "Charisma" 14 "+6" wide-slots) + (caster "Warlock 2 (Fiend, Pact of Blade)" :warlock "Charisma" 14 "+6" {1 2}) + (caster "Wizard 2 (School of Abjuration)" :wizard "Intelligence" 13 "+5" wide-slots)] + :fields + {:character-name "Wren Sixpence" :character-name-2 "Wren Sixpence" + :class-level "Brd2/Clr2/Drd2/Pal4/Rgr4/Sor2/Wlk2/Wiz2" + :background "Guild Artisan" :player-name "fixture" :race "Human (Variant)" + :alignment "Chaotic Good" :xp "355,000" + :str "+1" :str-mod "12" :dex "+1" :dex-mod "13" :con "+2" :con-mod "14" + :int "+1" :int-mod "12" :wis "+2" :wis-mod "14" :cha "+3" :cha-mod "16" + :str-save "+1" :dex-save "+1" :con-save "+2" + :int-save "+1" :wis-save "+8" :cha-save "+9" + :wis-save-check true :cha-save-check true + :ac "16" :initiative "+1" :speed "30 ft." :hp-max "134" :hp-current "134" + :hp-temp "0" :hd "2d8/2d8/2d8/4d10/4d10/2d6/2d8/2d6" :prof-bonus "+6" + :passive "12" :inspiration "1" + :religion "+7" :religion-check true :nature "+8" :nature-check true + :persuasion "+9" :persuasion-check true :performance "+9" :performance-check true + :arcana "+7" :medicine "+8" :insight "+8" :survival "+8" :history "+1" + :age "29" :height "5'6\"" :weight "142 lb." + :eyes "Hazel" :skin "Weathered" :hair "Brown, in need of a cut" + :weapon-name-1 "Longsword" :weapon-attack-bonus-1 "+7" :weapon-damage-1 "1d8+1 slashing" + :weapon-name-2 "Shortbow" :weapon-attack-bonus-2 "+7" :weapon-damage-2 "1d6+1 piercing" + :weapon-name-3 "Eldritch Blast" :weapon-attack-bonus-3 "+9" :weapon-damage-3 "1d10 force" + :gp "88" :sp "120" :cp "340" :ep "0" :pp "0" + :personality-traits "Has never finished anything and is cheerful about it." + :ideals "There is no wrong door. There are only doors you have not tried." + :bonds "Eight oaths, orders and pacts, all technically still in force." + :flaws "Cannot pass a training hall without enrolling." + :backstory (str "Started as an apprentice bookbinder and kept meeting people who were " + "better at something. Signed with every one of them. Answers to eight " + "teachers, none of whom know about the other seven.") + :allies "Everyone, briefly. Eight organisations each believe Wren is theirs." + :other-profs "Common, Elvish, Dwarvish, Sylvan, Infernal, Druidic, Celestial. Bookbinder's tools." + :features-and-traits-2 (str "Bardic Inspiration d6. Cutting Words. Channel Divinity. Preserve Life. " + "Wild Shape. Natural Recovery. Divine Smite. Aura of Protection. " + "Sacred Weapon. Favored Enemy. Natural Explorer. Colossus Slayer. " + "Font of Magic. Wild Magic Surge. Pact of the Blade. Arcane Ward.") + :features-and-traits "Longsword, shortbow, shield, holy symbol, druidic focus, component pouch, spellbook, lute." + :treasure "Cloak of Protection, Pearl of Power, a drawer of unfinished apprentice tokens" + :attacks-and-spellcasting + (str "Eight spellcasting classes; the sheet has room for six. Caster level 14 for shared " + "slots, but no class above 4th, so nothing above 1st level can be prepared.")}}) + +(defn- build! [{:keys [file template grow-to casters fields]}] + (let [all (merge fields (spellcasting-fields casters)) + out (io/file file)] + (io/make-parents out) + (with-open [doc (Loader/loadPDF (.readAllBytes + (.openStream (io/resource template))))] + ;; Mirrors the export path in routes.clj: grow the master to the shape the + ;; character needs, pages for classes past that, then overflow, then write. + (when grow-to (pdf/grow-spell-sections! doc grow-to :all)) + (pdf/add-missing-spell-pages! doc all) + (let [all (pdf/spill-overflow! doc all) + dropped (pdf/write-fields! doc all false {})] + (with-open [o (FileOutputStream. out)] (.save doc o)) + (println (format "wrote %-28s %d fields, %d casters, %d KB%s" + (.getPath out) (count all) (count casters) + (quot (.length out) 1024) + (if (seq dropped) + (format " [%d value(s) had no field]" (count dropped)) + ""))))))) + +(defn -main [& _] + (build! wizard-20) + (build! multi-20) + (build! multi-8)) + +(-main) diff --git a/dev/scale_orcbrew_pack.clj b/dev/scale_orcbrew_pack.clj new file mode 100644 index 000000000..cb982368c --- /dev/null +++ b/dev/scale_orcbrew_pack.clj @@ -0,0 +1,60 @@ +;; Generate scaled .orcbrew packs for perf work, by cloning a real pack's sources. +;; +;; lein with-profile +test run -m clojure.main dev/scale_orcbrew_pack.clj +;; +;; Writes dev-scratch/paks/pak-cN.orcbrew for N in 1 2 4 8 16. Each clone gets a distinct +;; source name and distinct entry keys, so N clones are N times the content rather than N +;; overwrites of the same keys, and every pack is read back with clojure.edn and counted +;; before it is reported. +;; +;; TWO TRAPS, both hit for real: +;; *print-length* is 50 under this project's lein profiles. Writing EDN with pr-str +;; without binding it to nil emits "..." and produces a silently truncated, unreadable +;; file — which then hangs a browser import for the full timeout. Hence the binding below. +;; And a source map can hold non-content keys (:disabled?) alongside the +;; :orcpub.dnd.e5/* content maps, so never assume every value is a map to count. + +(require '[clojure.edn :as edn]) + +(def orig (edn/read-string (slurp "test/fixtures/test-pak.orcbrew"))) + +(defn content-key? [k] (and (keyword? k) (= "orcpub.dnd.e5" (namespace k)))) +(defn content-source? [[_ v]] (and (map? v) (some content-key? (keys v)))) +(defn sfx [i k] (if (and (keyword? k) (nil? (namespace k))) (keyword (str (name k) "-c" i)) k)) + +(defn clone [[src-name content] i] + (let [new-name (str src-name " C" i)] + [new-name + (reduce-kv + (fn [m ctype entries] + (assoc m ctype + (if-not (and (content-key? ctype) (map? entries)) + entries + (reduce-kv (fn [e k v] + (assoc e (sfx i k) + (cond-> v + (:key v) (assoc :key (sfx i (:key v))) + (:option-pack v) (assoc :option-pack new-name) + (:source v) (assoc :source new-name)))) + {} entries)))) + {} content)])) + +(defn totals [pack] + (reduce (fn [acc [_ content]] + (if-not (map? content) acc + (merge-with + acc (reduce-kv (fn [m t e] + (if (and (content-key? t) (map? e)) + (assoc m (keyword (name t)) (count e)) m)) + {} content)))) + {} pack)) + +(doseq [mult [1 2 4 8 16]] + (binding [*print-length* nil *print-level* nil *print-namespace-maps* false] + (let [pack (into (into {} (remove content-source? orig)) + (for [i (range mult) s (filter content-source? orig)] (clone s i))) + p (str "dev-scratch/paks/pak-c" mult ".orcbrew") + s (pr-str pack)] + (spit p s) + (let [back (edn/read-string (slurp p))] + (println (format "%-32s %6.0f KB sources=%2d %s" + p (/ (count s) 1024.0) (count back) (pr-str (into (sorted-map) (totals back))))))))) diff --git a/dev/scan_cantrips_word.clj b/dev/scan_cantrips_word.clj new file mode 100644 index 000000000..93010a6c7 --- /dev/null +++ b/dev/scan_cantrips_word.clj @@ -0,0 +1,55 @@ +(ns scan-cantrips-word + "Measures the band the word CANTRIPS is printed in, in the bar of each style's + cantrips box, as offsets from the bar's middle in points. + + draw-column-heading! covers that word before writing a class name over it, and + the styles do not agree on where it sits: on style 3 it rides about four points + higher than on style 4, so a single patch band either left style 3's word + showing or painted over style 4's rules. Run this after changing a template's + cantrips box and copy the numbers into pdf/cantrips-word-patch." + (:require [clojure.java.io :as io] + [orcpub.pdf :as pdf]) + (:import [org.apache.pdfbox Loader] + [org.apache.pdfbox.rendering PDFRenderer])) + +(def dpi 300.0) + +(defn- word-rows + "Rows of `img` inside the given PDF-point rect that hold lettering, as page y. + + A row is lettering when between 5% and 60% of its columns carry ink. The bar's + rules run the width of the strip and the open paper between them carries none, + so both fall outside that band and only the word is left." + [img page-height x0 x1 y0 y1] + (let [s (/ dpi 72.0) + px #(int (Math/round (* s (double %)))) + top (fn [y] (px (- page-height y))) + cols (vec (range (px x0) (px x1)))] + (for [row (range (top y1) (top y0)) + :let [n (count (filter (fn [col] + (let [rgb (.getRGB img col row)] + (< (bit-and (bit-shift-right rgb 8) 0xff) 170))) + cols)) + frac (/ (double n) (count cols))] + :when (< 0.05 frac 0.6)] + (- page-height (/ row s))))) + +(defn -main [& _] + (doseq [style [1 2 3 4]] + (let [file (:file (get pdf/sheet-masters style))] + (with-open [doc (Loader/loadPDF (.readAllBytes (.openStream (io/resource file))))] + (let [[hx hy hw hh] (@#'pdf/cantrips-hexagon-box doc 1) + {[_ _] :narrow [wx ww] :wide} (@#'pdf/bar-compartments doc 0 1) + middle (+ hy (/ hh 2.0)) + widget (-> (.getAcroForm (.getDocumentCatalog doc)) + (.getField "spells-0-1-1") .getWidgets first) + page (.getPage widget) + idx (.indexOf (vec (.getPages doc)) page) + img (.renderImageWithDPI (PDFRenderer. doc) idx dpi) + ph (.getHeight (.getMediaBox page)) + rows (word-rows img ph (+ wx (* 0.35 ww)) (+ wx (* 0.65 ww)) + (- middle 16.0) (+ middle 16.0))] + (printf "style %d middle %.1f bar %.1f..%.1f ink %.1f..%.1f -> dy %.1f h %.1f%n" + style middle (- middle 16.0) (+ middle 16.0) + (double (apply min rows)) (double (apply max rows)) + (- (apply min rows) middle) (- (apply max rows) (apply min rows)))))))) diff --git a/dev/scan_numerals.clj b/dev/scan_numerals.clj new file mode 100644 index 000000000..457b1c32a --- /dev/null +++ b/dev/scan_numerals.clj @@ -0,0 +1,87 @@ +;; Measures each style's printed level numeral: where it sits, how big it is, and +;; what colour surrounds it. +;; +;; lein run -m clojure.main dev/scan_numerals.clj +;; +;; relabel-numeral! covers that numeral before drawing a new one, with the boxes +;; measured here (pdf/numeral-boxes). The first relabeller cut a patch to a +;; hexagon traced off style 1, and the styles do not merely offset that shape -- +;; style 3 rings its numerals, style 4 uses a small hexagon -- so a packed page +;; on 2, 3 or 4 printed both numbers, the old beside the new. +;; +;; Rather than trace four shapes, this measures what actually has to be covered: +;; the digit's own box, and the flat colour immediately around it. A rectangle of +;; that colour over that box hides the digit whatever badge it sits in. + +(require '[orcpub.pdf :as pdf] + '[clojure.java.io :as io]) +(import '[org.apache.pdfbox Loader] + '[org.apache.pdfbox.rendering PDFRenderer] + '[org.apache.pdfbox.text PDFTextStripper] + '[java.awt.image BufferedImage]) + +(def dpi 300.0) + +(defn- glyphs + "Every glyph on `index`, with the box PDFBox reports for it." + [doc index] + (let [out (atom []) + stripper (proxy [PDFTextStripper] [] + (writeString [text positions] + (doseq [p positions] + (swap! out conj {:ch (.getUnicode p) + :x (.getXDirAdj p) + :y (.getYDirAdj p) + :w (.getWidthDirAdj p) + :h (.getHeightDir p)}))))] + (.setStartPage stripper (inc index)) + (.setEndPage stripper (inc index)) + (.getText stripper doc) + @out)) + +(defn- pixel + "The colour at page point (x, y-from-bottom) as [r g b], 0-255." + [^BufferedImage img page-height x y] + (let [px (int (* x (/ dpi 72.0))) + py (int (* (- page-height y) (/ dpi 72.0)))] + (when (and (< -1 px (.getWidth img)) (< -1 py (.getHeight img))) + (let [p (.getRGB img px py)] + [(bit-and (bit-shift-right p 16) 0xff) + (bit-and (bit-shift-right p 8) 0xff) + (bit-and p 0xff)])))) + +(defn- surround + "Colours sampled just outside the digit's box, where the badge shows through." + [img page-height {:keys [x y w h]} page-h] + (let [bottom (- page-h y)] + (remove nil? + (for [[dx dy] [[-3.0 2.0] [(+ w 3.0) 2.0] + [(/ w 2.0) (+ h 2.5)] [(/ w 2.0) -2.5]]] + (pixel img page-height (+ x dx) (+ bottom dy)))))) + +(doseq [style [1 2 3 4]] + (let [{:keys [file]} (get pdf/sheet-masters style)] + (with-open [doc (Loader/loadPDF (.readAllBytes (.openStream (io/resource file))))] + (let [form (.getAcroForm (.getDocumentCatalog doc)) + pages (vec (.getPages doc)) + spell-page (some-> (.getField form "spells-0-1-1") .getWidgets first .getPage) + index (.indexOf pages spell-page) + page-h (.getHeight (.getMediaBox spell-page)) + img (.renderImageWithDPI (PDFRenderer. doc) index dpi) + gs (glyphs doc index)] + (printf "%nSTYLE %d%n" style) + (doseq [level (range 1 10) + :let [r (some-> (.getField form (str "spell-slots-" level "-1")) + .getWidgets first .getRectangle)] + :when r] + (let [want (str level) + near (filter #(and (= want (:ch %)) + (< (Math/abs (- (- page-h (:y %)) (+ (.getLowerLeftY r) 8))) 22) + (< -40 (- (:x %) (.getLowerLeftX r)) 2)) + gs)] + (when-let [g (first near)] + (printf " L%d digit box %.1f x %.1f at dx=%.1f dy=%.1f around: %s%n" + level (:w g) (:h g) + (- (:x g) (.getLowerLeftX r)) + (- (- page-h (:y g)) (.getLowerLeftY r)) + (pr-str (surround img page-h g page-h)))))))))) diff --git a/dev/scan_site_line.clj b/dev/scan_site_line.clj new file mode 100644 index 000000000..c7dd45218 --- /dev/null +++ b/dev/scan_site_line.clj @@ -0,0 +1,76 @@ +;; Finds where the site line can sit on each sheet style without printing over +;; the artwork. +;; +;; lein run -m clojure.main dev/scan_site_line.clj +;; +;; Reports, per style and per page, the x range in which a stamp-sized box is +;; free of ink at each candidate height. The intersection down a style's pages is +;; that style's :site-line in pdf/sheet-masters. +;; +;; This exists because the positions cannot be read out of the page's text. +;; PDFTextStripper reports glyphs and nothing else, so a first pass that placed +;; every style at one spot -- chosen from where each page's lowest TEXT sat -- +;; put the line straight through the corner flourish on the last page of styles +;; 1 and 2 and through the frame on style 4. Ink is what matters, so the page is +;; rendered and the pixels are counted. +;; +;; Pages a style prints its own footer on are skipped, the same as at export. + +(require '[orcpub.pdf :as pdf] + '[clojure.java.io :as io]) +(import '[org.apache.pdfbox Loader] + '[org.apache.pdfbox.rendering PDFRenderer] + '[java.awt.image BufferedImage] + '[java.io ByteArrayOutputStream]) + +(def dpi 150.0) +(def stamp-width (pdf/string-width pdf/site-stamp pdf/HELVETICA 6)) +(def stamp-height + "6pt of cap height plus room for a descender, in inches." + (/ 8.0 72)) + +(defn clear? + "Whether every pixel is near-white in the stamp-sized box whose bottom-left is + `xin`,`yin` inches from the page's bottom-left." + [^BufferedImage img xin yin] + (let [h (.getHeight img) + x0 (int (* xin dpi)) + x1 (int (* (+ xin stamp-width) dpi)) + ;; Inches up from the page foot, into rows down from the image top. + y-top (int (- h (* (+ yin stamp-height) dpi))) + y-bottom (int (- h (* yin dpi)))] + (and (>= x0 0) (< x1 (.getWidth img)) (>= y-top 0) (<= y-bottom h) + (every? (fn [y] + (every? (fn [x] + (let [p (.getRGB img x y)] + (and (> (bit-and (bit-shift-right p 16) 0xff) 244) + (> (bit-and (bit-shift-right p 8) 0xff) 244) + (> (bit-and p 0xff) 244)))) + (range x0 x1))) + (range y-top y-bottom))))) + +(defn- grown + "A style's master grown to `casters` sections, reloaded from its own bytes." + [style casters] + (let [{:keys [file marks]} (get pdf/sheet-masters style) + out (ByteArrayOutputStream.)] + (with-open [doc (Loader/loadPDF (.readAllBytes (.openStream (io/resource file))))] + (pdf/grow-spell-sections! doc casters marks) + (.save doc out)) + (Loader/loadPDF (.toByteArray out)))) + +(doseq [style [1 2 3 4]] + (let [{:keys [site-line prints-site-line?]} (get pdf/sheet-masters style)] + (printf "%nSTYLE %d shipping :site-line %s%n" style (pr-str site-line)) + (with-open [doc (grown style 2)] + (doseq [i (range (.getNumberOfPages doc))] + (if (and prints-site-line? (#'pdf/page-prints-site-line? doc i)) + (printf " p%d prints its own footer, skipped at export%n" (inc i)) + (let [img (.renderImageWithDPI (PDFRenderer. doc) i dpi)] + (doseq [y [0.06 0.08 0.10 0.12]] + (let [xs (filter #(clear? img % y) + (range 0.15 (- 8.5 stamp-width 0.15) 0.05))] + (printf " p%d y=%.2f %s%n" (inc i) y + (if (seq xs) + (format "clear x %.2f .. %.2f" (apply min xs) (apply max xs)) + "no clear run")))))))))) diff --git a/dev/spellcaster_pack.clj b/dev/spellcaster_pack.clj new file mode 100644 index 000000000..3b2f32faa --- /dev/null +++ b/dev/spellcaster_pack.clj @@ -0,0 +1,70 @@ +;; Generate .orcbrew packs that isolate ONE dimension: how many homebrew SPELLCASTING +;; classes/subclasses (each with a full spell list) and how many CUSTOM SPELLS are loaded. +;; +;; lein with-profile +test run -m clojure.main dev/spellcaster_pack.clj +;; +;; Clones the real spellcasting class out of test/fixtures/test-pak.orcbrew — a full +;; :spellcasting map with :level-factor, :known-mode, :spells-known, :cantrips-known and a +;; 285-entry :spell-list across levels 0-9 — so the shape matches real homebrew rather than +;; something invented. Each clone's spell-list is rewritten to reference that pack's own +;; CUSTOM spell keys, which is what a real homebrew caster does. +;; +;; Packs are kept under ~2.5 MB on purpose: localStorage caps at ~5 MB per origin, and a +;; pack that fails to persist silently measures a builder with no homebrew in it at all. +(require '[clojure.edn :as edn]) + +(def orig (edn/read-string (slurp "test/fixtures/test-pak.orcbrew"))) +(defn- all-of [t] (mapcat (fn [[_ v]] (when (map? v) (vals (get v (keyword "orcpub.dnd.e5" t))))) orig)) + +(def caster (first (filter :spellcasting (all-of "classes")))) +(def sub-caster (first (filter :spellcasting (all-of "subclasses")))) +(def spells (vec (all-of "spells"))) + +(defn- sfx [i k] (keyword (str (name k) "-s" i))) + +(defn make-spells + "n custom spells, cloned from the fixture's real ones, spread across levels 0-9." + [n src] + (into {} (for [i (range n) + :let [base (nth spells (mod i (count spells))) + k (sfx i (or (:key base) :spell))]] + [k (assoc base :key k :name (str "Custom Spell " i) :option-pack src)]))) + +(defn make-caster + "One homebrew spellcasting class whose spell-list is the pack's own custom spells." + [i src spell-keys] + (let [k (keyword (str "hb-caster-" i)) + by-level (into {} (for [lvl (range 0 10)] + [lvl (set (take 40 (drop (* lvl 7) (cycle spell-keys))))]))] + (-> caster + (assoc :key k :name (str "HB Caster " i) :option-pack src) + (assoc-in [:spellcasting :spell-list] by-level)))) + +(defn make-sub-caster [i src] + (let [k (keyword (str "hb-sub-caster-" i))] + (assoc sub-caster :key k :name (str "HB Sub Caster " i) :option-pack src + :class (if (even? i) :rogue :fighter)))) + +(doseq [[n-casters n-spells] [[1 50] [8 200] [32 400] [64 400] [128 400]]] + (binding [*print-length* nil *print-level* nil *print-namespace-maps* false] + (let [src (str "Spellcaster Pack " n-casters) + sp (make-spells n-spells src) + ks (vec (keys sp)) + pack {src {:orcpub.dnd.e5/spells sp + :orcpub.dnd.e5/classes (into {} (for [i (range n-casters)] + [(keyword (str "hb-caster-" i)) + (make-caster i src ks)])) + :orcpub.dnd.e5/subclasses (into {} (for [i (range n-casters)] + [(keyword (str "hb-sub-caster-" i)) + (make-sub-caster i src)])) + :disabled? false}} + p (str "dev-scratch/paks/spell-" n-casters ".orcbrew") + s (pr-str pack)] + (spit p s) + (let [back (edn/read-string (slurp p)) + c (get back src)] + (println (format "%-34s %6.0f KB casters=%-4d subcasters=%-4d spells=%-4d" + p (/ (count s) 1024.0) + (count (:orcpub.dnd.e5/classes c)) + (count (:orcpub.dnd.e5/subclasses c)) + (count (:orcpub.dnd.e5/spells c)))))))) diff --git a/dev/stress_packing.clj b/dev/stress_packing.clj new file mode 100644 index 000000000..5804070a8 --- /dev/null +++ b/dev/stress_packing.clj @@ -0,0 +1,181 @@ +(ns stress-packing + "Runs a set of caster shapes through the packed spell layout on every sheet + style, and checks that nothing is lost on the way. + + The check that matters is the last one: every spell name handed to the packer + has to come back out of the finished document's spell row fields, as many + times as it went in. A packed page renumbers its level boxes and moves classes + between columns, and a spell that falls off the end of a box, or into a box + with fewer fields than printed rows, disappears without an error -- which is + how three templates were silently dropping spells before the row fields were + repaired. + + Read off the fields rather than out of the page text: the form is not + flattened, so a field's value lives in its widget's appearance stream and + PDFTextStripper does not see it at all. + + Shapes cover a caster that fits in one page, one that spills to a second, a + pact caster beside full casters, a class with no cantrips, a class with only + cantrips, and a name long enough to be shortened in its column heading. + + Run: lein run -m stress-packing" + (:require [clojure.java.io :as io] + [clojure.string :as s] + [orcpub.common :as common] + [orcpub.dnd.e5.spell-annotations :as ann] + [orcpub.dnd.e5.spell-packing :as packing] + [orcpub.dnd.e5.spells :as spells] + [orcpub.pdf :as pdf]) + (:import [java.io File FileOutputStream] + [javax.imageio ImageIO] + [org.apache.pdfbox Loader] + [org.apache.pdfbox.rendering PDFRenderer])) + +(def ^:private names + (vec (sort (map (comp :name val) spells/spell-map)))) + +(defn- pick + "n distinct spell names, walking the list from `seed` so two classes in one + shape do not get the same spells." + [n seed] + (mapv #(nth names (mod (+ seed (* 7 %)) (count names))) (range n))) + +(defn- caster [class levels slots & {:keys [pact? dc attack]}] + (cond-> {:class class :levels levels :slots slots + :ability "CHA" :dc (or dc 15) :attack (or attack "+7")} + pact? (assoc :pact? true))) + +(def shapes + "Name, and the classes to pack. Ordered from the ordinary to the punishing." + [["one full caster" + [(caster "Wizard" {0 (pick 5 3) 1 (pick 8 41) 2 (pick 7 91) 3 (pick 6 141)} + {1 4 2 3 3 3})]] + + ["pact caster beside a full caster" + [(caster "Warlock" {0 (pick 3 7) 5 (pick 10 61)} {5 2} :pact? true) + (caster "Sorcerer" {0 (pick 5 21) 1 (pick 6 31) 2 (pick 5 51) 3 (pick 4 71)} + {1 4 2 3 3 2})]] + + ["four classes, one with no cantrips" + [(caster "Warlock" {0 (pick 2 3) 5 (pick 12 11)} {5 2} :pact? true) + (caster "Sorcerer" {0 (pick 4 21) 1 (pick 3 31) 2 (pick 2 41)} {1 4 2 3}) + (caster "Paladin" {1 (pick 3 51) 2 (pick 2 61)} {1 4 2 3}) + (caster "Bard" {0 (pick 3 71) 1 (pick 4 81)} {1 4})]] + + ["a class with nothing but cantrips" + [(caster "Warlock" {0 (pick 4 5)} {} :pact? true) + (caster "Cleric" {0 (pick 4 25) 1 (pick 5 45)} {1 4})]] + + ["a name long enough to be shortened" + [(caster "Eldritch Knight" {0 (pick 3 9) 1 (pick 6 29)} {1 4}) + (caster "Arcane Trickster" {0 (pick 3 49) 1 (pick 5 69)} {1 4})]] + + ;; No pact caster, so column 0 is free, and the first class alphabetically has + ;; no cantrips: its level 1 lands in box 0, the cantrips box. + ["a class with no cantrips leading column 0" + [(caster "Paladin" {1 (pick 5 55) 2 (pick 3 65)} {1 4 2 2}) + (caster "Ranger" {1 (pick 4 75) 2 (pick 2 85)} {1 4 2 2})]] + + ["every level filled, spilling to a second page" + [(caster "Wizard" {0 (pick 6 2) 1 (pick 12 12) 2 (pick 13 32) 3 (pick 13 52) + 4 (pick 13 72) 5 (pick 9 92) 6 (pick 9 112) 7 (pick 9 132) + 8 (pick 7 152) 9 (pick 7 172)} + {1 4 2 3 3 3 4 3 5 3 6 2 7 2 8 1 9 1}) + (caster "Cleric" {0 (pick 5 202) 1 (pick 10 222) 2 (pick 10 242) 3 (pick 8 262)} + {1 4 2 3 3 3}) + (caster "Druid" {0 (pick 4 302) 1 (pick 8 322) 2 (pick 8 342)} {1 4 2 3})]]]) + +(defn- spell-row-values + "Every non-blank spells-LEVEL-ROW-SECTION value in the document." + [doc] + (when-let [form (.getAcroForm (.getDocumentCatalog doc))] + (into [] + (comp (map #(vector (str (.getFullyQualifiedName %)) (str (.getValueAsString %)))) + (filter (fn [[n v]] (and (re-matches #"spells-\d+-\d+-\d+" n) + (not (s/blank? v))))) + (map second)) + (iterator-seq (.iterator (.getFieldTree form)))))) + +(defn- render! + "Builds the packed PDF for one shape on one style. Returns what came out." + [style classes out] + (let [{:keys [file site-line prints-site-line?]} (get pdf/sheet-masters style) + {:keys [fields relabels headings pages unplaced]} (packing/packed-fields style classes)] + (with-open [doc (Loader/loadPDF (.readAllBytes (.openStream (io/resource file))))] + (pdf/grow-spell-sections! doc pages :all) + (let [[ok refused] (pdf/apply-relabel-instructions! doc relabels pages style)] + (pdf/reserve-annotation-columns! doc) + (let [dropped (pdf/write-fields! doc fields false {})] + (pdf/annotate-spell-rows! doc #(some-> (get spells/spell-map (common/name-to-kw %)) + ann/annotation)) + (doseq [{:keys [class box section] :as h} headings] + (pdf/draw-column-heading! doc style box (or section 1) class h)) + (pdf/stamp-site-line! doc site-line (boolean prints-site-line?)) + (when out + (with-open [o (FileOutputStream. (File. (str out ".pdf")))] (.save doc o)) + (let [idx (.indexOf (vec (.getPages doc)) + (some-> (.getAcroForm (.getDocumentCatalog doc)) + (.getField "spells-0-1-1") + .getWidgets first .getPage))] + (when (nat-int? idx) + (ImageIO/write (.renderImageWithDPI (PDFRenderer. doc) idx 110) "png" + (File. (str out ".png")))))) + {:pages pages + :relabels-applied ok + :relabels-refused refused + :headings (count headings) + :dropped dropped + :unplaced unplaced + :written (spell-row-values doc)}))))) + +(defn -main [& args] + (let [out-dir (or (first args) "target/stress")] + (.mkdirs (File. out-dir)) + (println (format "%-46s %-6s %-6s %-9s %-7s %s" + "shape / style" "pages" "heads" "relabels" "packs?" "lost without saying so")) + (let [failures + (doall + (for [[shape classes] shapes + style [1 2 3 4]] + (let [out (str out-dir "/" (s/replace shape #"[^a-z]+" "-") "-style-" style) + {:keys [pages relabels-applied relabels-refused headings dropped + unplaced written]} + (render! style classes out) + wanted (mapcat (comp #(mapcat val %) :levels) classes) + ;; By count, not by set: two classes can know the same spell, + ;; and both copies have to be on the sheet. + have (frequencies written) + missing (for [[nm n] (frequencies wanted) + :let [got (get have nm 0)] + :when (< got n)] + (str nm " (" got " of " n ")")) + ;; A packing that cannot hold the character is allowed to + ;; place less, as long as it SAYS so -- pdf_spec reads the + ;; same report and prints a page per class instead. What is + ;; never allowed is a spell going missing in silence. + reported? (seq unplaced) + fits? (packing/fits? style (packing/packing-shape classes)) + silent (when-not reported? (vec missing))] + (println (format "%-46s %-6d %-6d %d/%-7d %-7s %s" + (str shape " / " style) pages headings + relabels-applied relabels-refused + (if fits? "yes" "no") + (cond + (seq silent) (pr-str silent) + (seq missing) (str "- (" (count missing) + " unplaced, reported)") + :else "-"))) + (when (or (seq silent) (seq dropped) (pos? relabels-refused) + ;; fits? and the field count have to agree, or the + ;; fallback fires on the wrong characters. + (not= fits? (empty? missing))) + [shape style {:silent silent :dropped dropped + :refused relabels-refused + :fits? fits? :missing (count missing)}])))) + failures (remove nil? failures)] + (println) + (if (seq failures) + (do (println (count failures) "FAILURE(S):") + (doseq [f failures] (println " " (pr-str f))) + (System/exit 1)) + (println "Every shape either packed with nothing lost, or said what it could not hold."))))) diff --git a/dev/stress_spell_sheets.clj b/dev/stress_spell_sheets.clj new file mode 100644 index 000000000..eeb02b5fd --- /dev/null +++ b/dev/stress_spell_sheets.clj @@ -0,0 +1,112 @@ +;; Fills every spell row of every style to the maximum with real spell names and +;; renders the result, so the sheets can be looked at rather than reasoned about. +;; +;; lein with-profile init-db run -m clojure.main dev/stress_spell_sheets.clj +;; +;; Writes target/stress-style-N.pdf and target/stress-style-N.png. +;; +;; Worst case on purpose: every row of every box carries one of the longest real +;; spell names, so clipping and shrink-to-fit show up where a typical name would +;; hide them. Goes through write-fields! rather than setting values directly, +;; because shrink-single-line-to-fit! is part of what decides the answer. +;; +;; The report says, per style and per level box, the size the rows settled on and +;; how much width is left beside the longest name -- which is the room an +;; annotation column has to live in. + +(require '[orcpub.pdf :as pdf] + '[orcpub.dnd.e5.spells :as spells] + '[clojure.java.io :as io] + '[clojure.string :as st]) +(import '[org.apache.pdfbox Loader] + '[org.apache.pdfbox.rendering PDFRenderer] + '[javax.imageio ImageIO] + '[java.io FileOutputStream File]) + +(def longest-names + "Real spell names, longest first. Filling with these makes the worst case the + normal case for this run." + (vec (reverse (sort-by count (map :name (vals spells/spell-map)))))) + +(defn- row-fields + "Every spells-LEVEL-ROW-1 field in the master, with its rectangle." + [doc] + (let [form (.getAcroForm (.getDocumentCatalog doc))] + (for [fld (iterator-seq (.iterator (.getFieldTree form))) + :let [n (.getFullyQualifiedName fld)] + :when (and n (re-matches #"spells-(\d+)-(\d+)-1" n)) + :let [[_ lvl idx] (re-matches #"spells-(\d+)-(\d+)-1" n) + r (some-> (first (.getWidgets fld)) .getRectangle)] + :when r] + {:name n :level (Integer/parseInt lvl) :idx (Integer/parseInt idx) + :w (.getWidth r) :h (.getHeight r)}))) + +(defn- baked-size + "The size a filled row actually settled on, read back from its appearance." + [doc field-name] + (let [form (.getAcroForm (.getDocumentCatalog doc)) + fld (.getField form field-name) + ap (some-> (first (.getWidgets fld)) .getAppearance .getNormalAppearance + .getAppearanceStream)] + (when ap + (let [b (java.io.ByteArrayOutputStream.)] + (with-open [in (.createInputStream (.getCOSObject ap))] (.transferTo in b)) + (when-let [m (re-find #"/\S+\s+([0-9.]+)\s+Tf" (String. (.toByteArray b)))] + (Double/parseDouble (second m))))))) + +(defn stress [style] + (let [file (:file (get pdf/sheet-masters style)) + doc (Loader/loadPDF (.readAllBytes (.openStream (io/resource file)))) + rows (vec (row-fields doc)) + ;; One long name per row, cycling so no two adjacent rows are identical. + values (into {} (map-indexed + (fn [i {:keys [name]}] + [(keyword name) (nth longest-names (mod i (count longest-names)))]) + rows)) + fields (merge values + {:character-name "Stress Test" + :class-level "Wizard 20" + :spellcasting-class-1 "Wizard" + :spellcasting-ability-1 "Intelligence" + :spell-save-dc-1 "19" :spell-attack-bonus-1 "+11"} + (into {} (for [l (range 1 10)] + [(keyword (str "spell-slots-" l "-1")) "4"])))] + (pdf/write-fields! doc fields false {}) + (with-open [out (FileOutputStream. (File. (str "target/stress-style-" style ".pdf")))] + (.save doc out)) + ;; Render the page section 1's rows actually live on. Style 4's master has + ;; TWO spell pages, so the last page belongs to section 2 and is empty here. + ;; Rendering it looked exactly like a total failure to fill anything. + (let [helv pdf/HELVETICA + per-level (for [[lvl rs] (sort (group-by :level rows))] + (let [sizes (keep #(baked-size doc (:name %)) rs) + widest (apply max + (for [r rs + :let [sz (or (baked-size doc (:name r)) 0) + v (get values (keyword (:name r)))]] + (* 72 (pdf/string-width v helv sz))))] + {:level lvl :rows (count rs) + :w (:w (first rs)) + :size-min (when (seq sizes) (apply min sizes)) + :size-max (when (seq sizes) (apply max sizes)) + :widest widest + :left (- (:w (first rs)) widest)})) + form (.getAcroForm (.getDocumentCatalog doc)) + page (some-> (.getField form "spells-0-1-1") .getWidgets first .getPage) + idx (if page (.indexOf (.getPages doc) page) (dec (.getNumberOfPages doc))) + n (.getNumberOfPages doc)] + (ImageIO/write (.renderImageWithDPI (PDFRenderer. doc) idx 110) "png" + (File. (str "target/stress-style-" style ".png"))) + (printf "%n (section 1 rows are on page %d of %d)%n" (inc idx) n) + (printf "%nSTYLE %d %s %d rows over %d pages%n" style file (count rows) n) + (printf " lvl rows row-w size widest name left for a column%n") + (doseq [{:keys [level rows w size-min size-max widest left]} per-level] + (printf " %d %2d %6.2f %5.2f-%-5.2f %8.1f %7.1f %s%n" + level rows w (or size-min 0) (or size-max 0) widest left + (cond (< left 0) "*** OVERFLOWS ***" + (< left 20) "tight" + :else ""))) + (.close doc)))) + +(doseq [style [1 2 3 4]] (stress style)) +(println "\ntarget/stress-style-{1,2,3,4}.{pdf,png}") diff --git a/dev/style4_one_spell_page.clj b/dev/style4_one_spell_page.clj new file mode 100644 index 000000000..0b06f261b --- /dev/null +++ b/dev/style4_one_spell_page.clj @@ -0,0 +1,141 @@ +;; Rebuilds style 4's master with ONE spell page instead of two. +;; +;; lein run -m clojure.main dev/style4_one_spell_page.clj +;; +;; This has been run and its output committed, and its INPUT -- the retired +;; two-spell-page master -- no longer ships in resources/. Re-running it means +;; restoring that file from git history first. It is kept as the record of what +;; the master is and what removing the page took. +;; +;; Writes resources/fillable-char-sheetstyle-4-1-spells.pdf, and +;; target/style4-{before,after}.png for checking the page is unchanged. +;; +;; Style 4 shipped two spell pages because its licence footer was taken to be +;; baked into the artwork, and a baked footer can be spread by cloning but never +;; removed -- so a :last style needed a plain page to clone and a marked page to +;; finish with. +;; +;; That is not what the pages are. Both reference the SAME background XObject, and +;; the marked page is the plain page plus one appended BT/ET block. So the MARKED +;; page is kept and the plain one dropped: the surviving page carries the footer +;; already, which makes the file a normal one-spell-page master like every other +;; style, and the footer lands on every clone. +;; +;; Two things make the removal actually work: +;; +;; - The dropped page's FIELDS have to go too. COSWriter writes what is reachable +;; from the trailer, so a page removed from the page tree while its widgets are +;; still listed in the AcroForm stays reachable through /Fields -> widget -> /P +;; and is written out anyway. Removing the page alone left the object count +;; unchanged at 2622 and the file 18 bytes LARGER. +;; - The survivor is section 2 and has to become section 1, or every caller +;; looking for spells-L-R-1 finds nothing. +;; +;; The footer block is also trimmed from six operators to four. `0 i` sets +;; flatness tolerance, which applies to path curves and not to glyph fills, and +;; `/GS2 gs` is the page default -- ca/CA 1.0, Normal, no soft mask -- differing +;; from GS0 only in stroke adjustment, which applies to strokes. + +(require '[orcpub.pdf :as pdf] + '[clojure.java.io :as io] + '[clojure.string :as st]) +(import '[org.apache.pdfbox Loader] + '[org.apache.pdfbox.pdmodel.common PDStream] + '[org.apache.pdfbox.cos COSName COSArray COSDictionary COSObject] + '[org.apache.pdfbox.rendering PDFRenderer] + '[javax.imageio ImageIO] + '[java.io FileOutputStream File]) + +(def src "fillable-char-sheetstyle-4-2-spells.pdf") +(def out "resources/fillable-char-sheetstyle-4-1-spells.pdf") + +(def plain-index 2) +(def marked-index 3) + +(def fields-on-page #'pdf/fields-on-page) + +(defn- content-of [page] + (let [b (java.io.ByteArrayOutputStream.)] + (with-open [in (.getContents page)] (.transferTo in b)) + (String. (.toByteArray b) "ISO-8859-1"))) + +(defn- set-content! [doc page ^String s] + (let [stream (PDStream. doc)] + (with-open [os (.createOutputStream stream)] + (.write os (.getBytes s "ISO-8859-1"))) + (.setContents page stream))) + +(defn- objects [doc] (count (.getXrefTable (.getDocument doc)))) + +(defn- drop-struct-tree! + "Removes the document's structure tree. + + This file is TAGGED, and the tree reaches a page through each element's /Pg, + so removing the page from the page tree and its fields from the AcroForm is not + enough on its own: the writer keeps whatever is reachable and the dropped page + comes along. Measured, the tree holds 1321 of the file's 2622 objects. + + Pruning only the 242 elements that name the dropped page was tried and reached + 27 of them: /K is a dictionary, an array, an integer MCID or a reference by + turns, and a correct pruner has to handle ParentTree and ClassMap as well. + Removing the tree is one line and reliable. + + The cost is style 4's accessibility tagging, which is a real loss and is worth + knowing about: styles 1 and 2 keep theirs, and style 3 already ships without + any. Restoring it means writing the pruner properly, not undoing this." + [doc] + (let [cos (.getCOSObject (.getDocumentCatalog doc))] + (.removeItem cos COSName/STRUCT_TREE_ROOT) + (.removeItem cos (COSName/getPDFName "MarkInfo")))) + +(with-open [doc (Loader/loadPDF (.readAllBytes (.openStream (io/resource src))))] + (let [pages (vec (.getPages doc)) + plain (nth pages plain-index) + marked (nth pages marked-index) + form (.getAcroForm (.getDocumentCatalog doc))] + (printf "before: %d pages, %d objects%n" (.getNumberOfPages doc) (objects doc)) + (ImageIO/write (.renderImageWithDPI (PDFRenderer. doc) marked-index 150) "png" + (File. "target/style4-before.png")) + + ;; 1. The dropped page's fields, or the page stays reachable through /Fields. + (let [doomed (set (map #(System/identityHashCode (.getCOSObject %)) + (fields-on-page doc plain))) + keep (vec (remove #(contains? doomed (System/identityHashCode (.getCOSObject %))) + (.getFields form)))] + (printf " dropping %d field(s) that live on the plain page%n" + (- (count (.getFields form)) (count keep))) + (.setFields form keep)) + + ;; 2. The structure tree, which reaches the page through /Pg. + (drop-struct-tree! doc) + + ;; 3. The page itself. + (.removePage doc plain-index) + + ;; 4. The survivor is section 2; every caller looks for section 1. + (pdf/renumber-page-section! doc marked 2 1) + + ;; 5. Trim the footer to the operators that do something. + (let [before (content-of marked) + after (-> before + (st/replace "0 i \n" "") + (st/replace "/GS2 gs\n" ""))] + (printf " footer block: %d -> %d bytes%n" (count before) (count after)) + (set-content! doc marked after)) + + (with-open [os (FileOutputStream. (File. out))] + (.save doc os)) + (printf "after : %d pages, %d objects%n" (.getNumberOfPages doc) (objects doc)))) + +;; Reload from disk: the object count and the render are what actually matter. +(with-open [doc (Loader/loadPDF (File. out))] + (let [form (.getAcroForm (.getDocumentCatalog doc)) + spell-page (some-> (.getField form "spells-0-1-1") .getWidgets first .getPage) + idx (.indexOf (.getPages doc) spell-page)] + (printf "reload: %d pages, %d objects; spells-0-1-1 on page %d%n" + (.getNumberOfPages doc) (objects doc) (inc idx)) + (ImageIO/write (.renderImageWithDPI (PDFRenderer. doc) idx 150) "png" + (File. "target/style4-after.png")))) + +(printf "%n%,d -> %,d bytes%n" + (.length (File. (str "resources/" src))) (.length (File. out))) diff --git a/dev/style_gallery.clj b/dev/style_gallery.clj new file mode 100644 index 000000000..d1865c1be --- /dev/null +++ b/dev/style_gallery.clj @@ -0,0 +1,77 @@ +;; Renders one fully-populated character on every sheet style, every page. +;; +;; lein with-profile init-db run -m clojure.main dev/style_gallery.clj +;; +;; Writes target/gallery-style-N.pdf and target/gallery-style-N-pM.png. +;; +;; The character is dev/sample_character.clj's level 20 evoker, which fills every +;; spell row the page offers, so the spell page is seen at its worst case rather +;; than half empty. Row counts come from spell-packing/sheet-geometry so each +;; style is filled to ITS capacity -- style 4's cantrip box holds 7 where the +;; others hold 8, and filling all four from one number leaves a value with no +;; field on style 4 every run. +;; +;; Goes through the same calls as routes.clj in the same order, so what comes out +;; is what a download gives: grow, add pages, merge, spill, write, stamp. + +(load-file "dev/sample_character.clj") + +(require '[orcpub.pdf :as pdf] + '[orcpub.dnd.e5.spells :as spells] + '[orcpub.dnd.e5.spell-lists :as sl] + '[orcpub.dnd.e5.spell-packing :as pk] + '[orcpub.dnd.e5.spell-annotations :as spell-annotations] + '[orcpub.common :as common] + '[clojure.java.io :as io]) +(import '[org.apache.pdfbox Loader] + '[org.apache.pdfbox.rendering PDFRenderer] + '[javax.imageio ImageIO] + '[java.io File FileOutputStream]) + +(def fixture @#'sample-character/wizard-20) + +(defn- spell-rows + "The spell-name fields for one class, filled to `style`'s own row counts." + [{:keys [list max-spell-level]} suffix style] + (let [available (get sl/spell-lists list) + geometry (get pk/sheet-geometry style)] + (into {} + (for [level (range 0 (inc max-spell-level)) + :let [names (sort (map #(or (:name (get spells/spell-map %)) (name %)) + (get available level)))] + [idx nm] (map-indexed vector (take (nth geometry level) names))] + [(keyword (format "spells-%d-%d-%d" level (inc idx) suffix)) nm])))) + +(defn- fields-for [style] + (let [caster (first (:casters fixture))] + (merge (:fields fixture) + {:spellcasting-class-1 (:name-str caster) + :spellcasting-ability-1 (:ability caster) + :spell-save-dc-1 (str (:dc caster)) + :spell-attack-bonus-1 (:attack caster)} + (into {} (for [[lvl n] (:slots caster)] + [(keyword (str "spell-slots-" lvl "-1")) (str n)])) + (spell-rows caster 1 style)))) + +(doseq [style [1 2 3 4]] + (let [{:keys [file site-line prints-site-line?]} (get pdf/sheet-masters style) + out (File. (format "target/gallery-style-%d.pdf" style)) + all (fields-for style)] + (with-open [doc (Loader/loadPDF (.readAllBytes (.openStream (io/resource file))))] + (pdf/grow-spell-sections! doc 1 :all) + (pdf/add-missing-spell-pages! doc all) + (let [all (pdf/merge-style-fields style all) + _ (pdf/reserve-annotation-columns! doc) + dropped (pdf/write-fields! doc (pdf/spill-overflow! doc all) false {}) + _ (pdf/annotate-spell-rows! + doc #(some-> (get spells/spell-map (common/name-to-kw %)) + spell-annotations/annotation))] + (pdf/stamp-site-line! doc site-line (boolean prints-site-line?)) + (with-open [o (FileOutputStream. out)] (.save doc o)) + (printf "style %d: %d pages, %d fields, %d KB%s%n" + style (.getNumberOfPages doc) (count all) (quot (.length out) 1024) + (if (seq dropped) (format " [%d unplaceable]" (count dropped)) "")))) + (with-open [doc (Loader/loadPDF out)] + (doseq [i (range (.getNumberOfPages doc))] + (ImageIO/write (.renderImageWithDPI (PDFRenderer. doc) i 110) "png" + (File. (format "target/gallery-style-%d-p%d.png" style (inc i)))))))) diff --git a/dev/svg_icon_proof.clj b/dev/svg_icon_proof.clj new file mode 100644 index 000000000..2420a579e --- /dev/null +++ b/dev/svg_icon_proof.clj @@ -0,0 +1,58 @@ +;; Renders an icon as vector art from its SVG beside the 32px raster of the same +;; icon, at the sizes a card draws them, and rasterises the sheet for inspection. +;; +;; lein run -m clojure.main dev/svg_icon_proof.clj +;; ICONS=magic-swirl,crystal-ball lein run -m clojure.main dev/svg_icon_proof.clj +;; +;; This is how to check a path conversion: put the two side by side and look. A +;; quarter inch on a 600 DPI printer is about 150 device pixels, which a 32 pixel +;; source cannot fill and a path does not have to. +;; +;; The rasters are no longer what the cards draw. The comparison is kept because +;; it is the check to repeat whenever an icon or the parser changes. + +(require '[orcpub.pdf :as pdf] '[clojure.string :as str] '[clojure.java.io :as io]) +(import '[org.apache.pdfbox.pdmodel PDDocument PDPage PDPageContentStream] + '[org.apache.pdfbox.pdmodel.font PDType1Font Standard14Fonts$FontName] + '[org.apache.pdfbox.rendering PDFRenderer] + '[javax.imageio ImageIO] '[java.io File]) + +(def icons (str/split (or (System/getenv "ICONS") "magic-swirl") #",")) +(def sizes "Inches. 0.25 is what a spell card actually draws." [0.15 0.25 0.4 0.7 1.1]) + +(let [doc (PDDocument.) page (PDPage.) img (pdf/make-image-loader doc) + helv (PDType1Font. Standard14Fonts$FontName/HELVETICA) + label (fn [cs t x y] (pdf/draw-text cs t helv 7 x y))] + (.addPage doc page) + (with-open [cs (PDPageContentStream. doc page)] + (doseq [[row icon] (map-indexed vector icons)] + (let [ops (pdf/load-svg-icon icon) + png (when (io/resource (str "public/image/" icon ".png")) + (img (str "public/image/" icon ".png"))) + top (+ 0.9 (* row 2.6))] + (label cs (str icon " (" (if ops (str (count ops) " path ops") "NO SVG") ")") + 0.7 (- 11 (- top 0.25))) + (label cs "vector from SVG" 0.7 (- 11 (+ top 0.05))) + (label cs (if png "32px PNG" "(never rasterised)") 0.7 (- 11 (+ top 1.35))) + (loop [[s & more] sizes x 2.1] + (when s + ;; Both rows share a baseline so the eye compares like with like. + (when ops (pdf/draw-svg-path! cs ops x (+ top 0.15) s 512)) + ;; Only some of these were ever rasterised; the row is skipped rather + ;; than faked for the rest. + (when png (pdf/draw-imagex cs png x (+ top 1.45) s s)) + (label cs (format "%.2fin" (double s)) x (- 11 (+ top 2.55))) + (recur more (+ x s 0.28))))))) + (.save doc (File. "target/svg-icon-proof.pdf")) + (ImageIO/write (.renderImageWithDPI (PDFRenderer. doc) 0 200) "png" + (File. "target/svg-icon-proof.png")) + (.close doc)) + +(doseq [i icons] + (let [ops (pdf/load-svg-icon i)] + (printf "%-20s %s%n" i + (if ops + (format "%d ops %s" (count ops) + (str/join " " (sort (distinct (map (comp name first) ops))))) + "no SVG in resources")))) +(println "\ntarget/svg-icon-proof.png") diff --git a/docs/CHARACTER-IMAGE-FETCH.md b/docs/CHARACTER-IMAGE-FETCH.md new file mode 100644 index 000000000..6bf7b3a34 --- /dev/null +++ b/docs/CHARACTER-IMAGE-FETCH.md @@ -0,0 +1,354 @@ +# Character image fetch — how it works, and what to check when it breaks + +A character's portrait and faction image are stored as **URLs**. Three ways the +pixels reach the PDF, tried in that order: + +1. **The browser reads them and sends the bytes with the export.** The normal + path. Hosts that block hotlinking judge the Referer and the datacenter IP, + neither of which describes the visitor's own browser. +2. **The server fetches the URL itself.** For a picture the browser was refused. + The builder asks about it up front — `POST /image-probe`, see below — so it + knows the answer before anyone clicks Export. This is the only outbound request + OrcPub makes to an address a visitor chose, on an endpoint that needs no login, + so it is deliberately hemmed in. +3. **Paste, or a file.** Local to the machine, so no host has a say. This is the + way in for a host that refuses everyone. + +The user is told nothing until step 2 has answered. An offer at the moment the +browser gives up would ask for a paste of a picture the server was about to fetch. + +Read this if portraits stop appearing in exported PDFs, or if a security review +asks what `/character.pdf` can be made to talk to. + +- Code, browser side: `orcpub.image-capture` — `capture`, `capture-file`, + `normalize`, `read-drawn` +- Code, server side: `orcpub.pdf` — `decode-image-bytes` (path 1), + and `validated-addresses`, `private-address?`, `pinned-connection-manager`, + `proxied?`, `open-image-stream`, `read-bounded-bytes`, + `within-pixel-budget?`, `safe-image-bytes` (path 2) +- Tests: `test/clj/orcpub/pdf_supplied_image_test.clj` (bytes the browser sent), + `test/clj/orcpub/pdf_image_test.clj` (which URLs are allowed), + `test/clj/orcpub/pdf_image_fetch_test.clj` (what happens once one is), + `test/browser/character_image_capture_e2e.js` (both paths through the real app) + +## Measured against real hosts (2026-09-06) + +With a browser that can reach the real internet (see `test/browser/README.md` for +the TLS flag that takes), against **real** URLs rather than invented ones: + +| host | browser reads it | server fetches it | +|---|---|---| +| `i.imgur.com` | yes | yes | +| `cdn.discordapp.com` | yes | yes | +| `upload.wikimedia.org` | yes (42 KB after fitting) | yes | +| `i.pinimg.com` | no | **yes** (393 KB, fitted) | +| `www.dndbeyond.com` | no | **yes** (148 KB, fitted) | + +Pinterest serves this server a 200 and 393 KB of JPEG; a D&D Beyond avatar is a +200 and 148 KB. Neither ever blocked us. What +refused it was our own 128 KB ceiling, applied to the download rather than to the +document — now split in two, so a heavy picture is fitted instead of dropped. A +Pinterest portrait reaches the sheet with nothing asked of the user. + +## Which hosts allow the browser to read (measured 2026-09-05) + +The browser can only read a picture whose host sends `Access-Control-Allow-Origin`. +Sampled with a browser-shaped request; ACAO is set at the edge, so it holds for the +host rather than the object. + +| Allows the read | Does not | +|---|---| +| `i.imgur.com` `*` | `i.pinimg.com` | +| `cdn.discordapp.com` `*` | `www.dndbeyond.com` | +| `static.wikia.nocookie.net` (Fandom) `*` | `i.postimg.cc` | +| `upload.wikimedia.org` `*` | `i.ibb.co` | +| `cdna.artstation.com` `*` | `live.staticflickr.com` | +| `images-wixmp-…` (DeviantArt), echoes Origin | `www.dropbox.com` | +| `lh3.googleusercontent.com`, `64.media.tumblr.com`, `raw.githubusercontent.com` `*` | `i.redd.it` | + +The two lists are largely complementary rather than overlapping: most of the +right-hand column allows hotlinking, so the server fetches those perfectly well. + +Do NOT read the right-hand column as "these block us". It records only that the +host sends no `Access-Control-Allow-Origin`. An earlier version of this table +claimed Pinterest and D&D Beyond refused the server too, on the strength of a 403 +-- but those URLs were invented, and both hosts sit behind S3, which answers +`AccessDenied` for a key that does not exist. Whether a given host will serve THIS +server is a question about a real URL, and the answer belongs to the runtime, not +to a table. + +Re-run the probe politely -- a request or two per host, spaced -- rather than in a +loop; these are other people's servers. + +## What is known before anything is fetched + +`orcpub.image-url/advise` reads the address alone -- no request, no waiting -- and +is where most real mistakes are caught, because most of them are visible in the +string: + +- **A page's address instead of the picture's.** The commonest paste there is: + a Pinterest pin page, an Imgur gallery, a Reddit post, a Flickr or DeviantArt or + ArtStation page. A fetch can only report these as a puzzle; the string says + plainly what they are. +- **A login wall.** Instagram and Facebook links cannot work for anyone. +- **A malformed address.** No scheme, a scheme that is not the web, a space in the + middle from a half-copied link. +- **`http://`,** which the page's own CSP will not display whatever the host does. + This one is not offered but VERIFIED and applied: the browser loads the https + address itself -- a plain ``, no server, and the same request the thumbnail + was about to make -- and the field is changed only once that succeeds, with a + note saying so. A host that serves no https is told, and its address is left + exactly as typed. +- **Viewer links with a known direct form** -- Dropbox `?dl=0`, Google Drive + `/file/d//view` -- which are offered as a correction to take or leave. + +A correction is only ever offered where it is mechanical. Nothing guesses a +picture's address from a page's; it says what the page is and how to get the real +one. The advice is debounced, because the field commits on every keystroke and +advice that objects to `htt` on the way to `https://` teaches people to ignore it. + +Advice, not enforcement. The rules that must hold are enforced where they cannot +be argued with -- address validation on the server, CORS in the browser -- so this +is free to be occasionally wrong, and its weakest rule (an unknown host with no +file name) is only a note. + +## What the builder tells a person, and why + +`/image-probe` answers a REASON, not a boolean, and the builder turns it into +wording. The server never sends a sentence: keeping the words on the client means +nothing the server says can reach a person unedited. + +| reason | shown as | what it points at | +|---|---|---| +| `ok` | nothing | the server can fetch it | +| `blocked-address` | that address cannot be fetched, check the host name and that it is a public image | the link | +| `not-found` | the host says there is nothing there | the link | +| `redirect` | the link redirects rather than being the picture | the link | +| `not-an-image` | not a PNG or JPEG | the link | +| `unreachable` | that host could not be reached | the link | +| `refused` | the host refused to serve it to us | supply the picture | +| `too-large` | larger than 2 MB | supply the picture | +| `too-many-pixels` | larger than 2000x2000 | supply the picture | +| `timeout` | the host took too long | supply the picture | +| `host-error` | the host had an error of its own | supply the picture | +| `rate-limited` | the host is asking us to slow down | try again shortly | +| `unknown` | could not be fetched | supply the picture | + +Split by what is worth fixing. Telling someone to copy a picture when they have +mistyped a link is not help, and telling someone to check a link the host is +refusing outright sends them round in circles. + +Every address refusal collapses into `blocked-address` on purpose — a private +range, a reserved range, a name that does not resolve, a DNS answer that did not +survive the pin. Separating them would make an endpoint that needs no login into a +map of what this server's network can reach, one request per answer. The wording +covers both cases instead. + +## The pre-flight probe + +`POST /image-probe` with `{:url "..."}` answers `"true"` or `"false"`: whether +THIS server can fetch THAT picture. The builder asks as soon as a browser read +fails, not at export time — the export posts a form into a new tab and never sees +the response, and an await between the click and the submit would spend the user +activation that keeps the tab from being blocked. + +- The bytes are kept (`probed-images`, 10 minutes, 64 entries) so the export that + follows costs the host no second request. A negative answer is cached too. +- It answers a boolean and never the picture. Handing back fetched bytes would + make an endpoint that needs no login into a general-purpose proxy for anything + inside the size limits. +- Every address rule that guards the export guards this: private and reserved + ranges, scheme, redirects. Otherwise it would be a port scanner returning one + boolean per request. `test/clj/orcpub/image_probe_test.clj` holds that line. + +## Which path a picture took + +Both ceilings — 128 KB and 2000×2000 — apply to both paths. Bytes from the +browser arrive base64 in `:image-data` / `:faction-image-data` and are decoded by +`decode-image-bytes`, which applies those ceilings and reads the format from the +bytes rather than from the mime type the client claimed. When they are present the +server does not fetch at all, so none of the failures in the table below can occur. + +When the browser cannot read a picture the export goes out with the address and +the server tries instead. **Nothing is said to the user at that point** — an offer +there would ask for an upload of what the server was about to fetch. The upload is +offered only after an export has gone out with the picture unread, and worded +conditionally, because this page never sees the export's response: the form posts +into a new tab. + +Exporting is held while a read is in flight, so the browser's bytes win the race +rather than losing it to a click. A read always ends — `capture` carries its own +deadline — so the hold is bounded. + +For a host that lets nobody read its pictures -- refusing the page and the server +alike -- the way in is PASTE. The browser's own "Copy image" puts the decoded +picture on the clipboard, and a paste into the Image URL field is read locally, so +none of the host's rules apply. The offer names that first and the file picker +second. Nothing here can be done by opening the picture in another tab: that tab +is a different origin and the opener cannot read it, and a service worker fetching +it no-cors gets an opaque response whose bytes it cannot read either. + +A refused read logs a CORS error in the browser console. That is the browser +reporting the host's rule and cannot be suppressed by the page. + +A picture drawn off a canvas is re-encoded to JPEG at up to 1000px on the long +edge, which is past 300dpi for the 2.35 × 3.15 inch box it prints in. An uploaded +file already inside both ceilings is carried untouched. + +## Symptoms and what they mean + +Failures are logged by `orcpub.routes` as `pdf: failed adding …` or surface as +`Failed to load image from URL`. The `:error` key in the exception data is the +thing to grep for. + +| What you see | `:error` | Cause | +|---|---|---| +| Portrait missing, host is internal or a bare IP | `:image-url-not-permitted` | The address is private or reserved. Working as intended — see the ranges below. | +| Portrait missing, public host | `:image-load-failed` with a `:status` | The host answered non-2xx. A `302` means it tried to redirect; redirects are refused on purpose. | +| Portrait missing, large image | `:image-too-large` | Over the 2 MB **download** ceiling. The 128 KB ceiling is separate and applies to what goes INTO the PDF: a picture between the two is scaled and re-encoded to fit, not refused. | +| Portrait missing, big dimensions | `:image-too-large-dimensions` | Over 2000×2000, refused from the header before decoding. | +| Portrait missing, slow host | `:image-transfer-timeout` | The body took more than 20s. | +| Portrait missing, unreadable file | `:invalid-image-format` | Not an image, or a format ImageIO cannot read. | +| **Every** image fails, log shouts `EVERY character image will fail to load` | `:image-pin-mismatch` | Something is routing the connection elsewhere while the JVM reports no proxy, so the pin engaged where it should have stepped aside. **See "Egress proxies" below.** | + +That last row is the one to remember. It is not a per-image failure — it is every +image, everywhere, at once. It is printed **once** per process, in full, naming the +host it was routed to and pointing back here, so it cannot drown in its own repeats +on a busy server. + +## The line to look for at boot + +Every start prints which of the two egress paths is live, before Jetty comes up: + +``` +pdf/image-fetch: DNS pinning ACTIVE (no proxy configured for external https) +``` +``` +pdf/image-fetch: DNS pinning OFF -- HTTP @ proxy.internal:3128 will resolve and fetch; it is the egress control point +``` + +**If that says OFF and you configured no proxy, or ACTIVE on a host that has one, +stop there** — the mismatch is the cause of whatever comes next. The same thing is +readable as data: `(orcpub.pdf/image-egress-status)` returns +`{:pinning? bool :proxy str-or-nil}`. + +## The limits, in one place + +| Rule | Value | Where | +|---|---|---| +| Schemes | `http`, `https` only | `validated-addresses` | +| Addresses | private/reserved refused | `private-address?` | +| Redirects | not followed | `open-image-stream` | +| Size | 128 KB | `max-image-bytes` | +| Dimensions | 2000 × 2000 | `max-image-pixels` | +| Connect | 10s | `open-image-stream` | +| Transfer | 20s total | `image-transfer-deadline-ms` | + +Refused address ranges: loopback, `0.0.0.0/8`, `10/8`, `100.64/10` (carrier-grade +NAT), `169.254/16` (cloud metadata), `172.16/12`, `192.0.0/24`, `192.168/16`, +`198.18/15`, `240/4`, multicast; and for IPv6 `::1`, `fe80::/10`, `fec0::/10`, +**`fc00::/7`**, plus the IPv4 addresses carried inside `64:ff9b::/96` (NAT64) and +`2002::/16` (6to4). + +The last three exist because `InetAddress` has no predicate for them and each was +verified reachable before it was added. `fc00::/7` is the one to notice: +`isSiteLocalAddress` knows only the deprecated `fec0::/10`, so a guard built on it +alone lets every real private IPv6 address through. + +## Resolve once, connect to that answer + +The host is resolved **once**, by `validated-addresses`, and those exact addresses +are handed to the connection. + +This matters because the obvious implementation is wrong. If the guard resolves +the name to check it, and the connection then resolves the name again, a DNS +server the attacker controls can answer a public address the first time and +`127.0.0.1` the second. The address that was validated is not the address that was +talked to. That is DNS rebinding. + +`pinned-connection-manager` gives the HTTP client a `DnsResolver` that returns the +validated addresses for that host and throws for anything else. + +**Two constraints on the shape, both verified against the Apache javadoc:** + +- The resolver must go on the **connection manager**, not on `HttpClientBuilder`. + Its javadoc for `setDnsResolver` says: *"Please note this value can be + overridden by the `setConnectionManager(HttpClientConnectionManager)` method."* + clj-http always sets a connection manager, so pinning on the builder compiles, + runs, and silently does nothing. + ([javadoc](https://hc.apache.org/httpcomponents-client-4.5.x/current/httpclient/apidocs/org/apache/http/impl/client/HttpClientBuilder.html)) +- The hostname must stay in the URL. Addressing the resolved IP directly with a + `Host:` header is the tempting shortcut, and it breaks certificate validation; + repairing that means overriding hostname verification, which is a bigger hole + than rebinding. Pinning the resolver keeps the default socket factories doing + ordinary certificate and hostname checks. + +`BasicHttpClientConnectionManager` is used with the four-argument constructor +`(socketFactoryRegistry, connFactory, schemePortResolver, dnsResolver)`. It +"maintains only one active connection" and "ought to be used by one execution +thread only", which is exactly the shape here: one manager per fetch, closed in a +`finally`. +([javadoc](https://hc.apache.org/httpcomponents-client-4.5.x/current/httpclient/apidocs/org/apache/http/impl/conn/BasicHttpClientConnectionManager.html)) + +## Deployment + +| Setup | Effect | To configure | +|---|---|---| +| **nginx** (`deploy/nginx.conf.template`) | None. It is an inbound reverse proxy; this fetch is outbound and never passes through it. | Nothing | +| **Docker Compose** | Works. The one resolution uses the container's DNS, so Docker's embedded resolver serves it normally. | Nothing | +| **Docker Swarm** (`./run --swarm`, overlay networks) | Same as Compose. Overlay addressing is internal and irrelevant to an external image URL; task egress NATs out normally. Each replica fetches independently — nothing is shared. | Nothing | +| **Egress proxy** | The pin is **skipped**, deliberately. | Nothing, if the proxy is set via the JVM's proxy properties | + +### Why an egress proxy changes things + +Behind a proxy the client connects to the **proxy**, so the resolver is asked for +the *proxy's* hostname — which the pin refuses, failing every HTTPS fetch with +`not the pinned host`. It is also pointless there: the proxy does the name +resolution, and the proxy is the egress control point. + +`proxied?` reads the same `ProxySelector` that clj-http's `SystemDefaultRoutePlanner` +uses, so the two cannot disagree, and steps around the pin when one applies. + +**Note that this only detects proxies the JVM knows about** — `https.proxyHost`, +`http.proxyHost` and friends. A transparent proxy or an egress gateway enforced at +the network layer is invisible to the JVM: the fetch still works, but the guarantee +is then your network's rather than this code's. + +Neither Compose nor Swarm sets a proxy today, so **the pinned path is the default +in production.** + +### Consequence worth knowing + +Because private ranges are refused, a character image URL **cannot** point at +another container on your Compose or Swarm network. That is deliberate — it is the +same rule that stops it reaching the transactor or a cloud metadata endpoint — but +it does mean there is no way to serve portraits from an internal host today. There +is no allowlist or exception mechanism. + +## Diagnosing a specific URL + +``` +lein repl +(require '[orcpub.pdf :as pdf]) +(pdf/image-egress-status) ;; which path is live? +(pdf/safe-image-url? "https://example.com/portrait.png") ;; may we fetch it? +(count (pdf/safe-image-bytes "https://example.com/portrait.png")) ;; try it +``` + +`safe-image-url?` returns false rather than throwing, so a rejected URL is +indistinguishable from one that failed to load — by design, since the sheet +already handles a missing image. `safe-image-bytes` throws with the `:error` key +from the table above, which is the faster way to find out *why*. + +## What has not been verified + +- **Not run inside a container.** The deployment rows above are read from + `docker-compose.yaml`, `docker/Dockerfile` and `deploy/nginx.conf.template`, + not from a container run. If images break after a deployment change, this is + the first assumption to re-test. +- **The pinned path has not been exercised against a real external host.** It is + covered against a local server through a resolved hostname, and a real HTTPS + fetch was verified end-to-end — but through the *proxied* path, because the + development sandbox requires a proxy. Production runs the pinned path. +- Whether a fetched image is re-encoded or downscaled to its drawn size. It is + capped at 128 KB, so the exposure is bounded either way. diff --git a/docs/CONTRIBUTING.md b/docs/CONTRIBUTING.md new file mode 100644 index 000000000..8c5549874 --- /dev/null +++ b/docs/CONTRIBUTING.md @@ -0,0 +1,154 @@ +# Contributing — branching, changelog & commit conventions + +The workflow for this fork. Read before committing or opening a PR. + +## Branch roles + +| Branch | Role | Commit here? | Promotes to | +|--------|------|--------------|-------------| +| `main` / `master` | Vestigial — tracks the abandoned upstream author. **Ignore.** | no | — | +| `develop` | **Production**: deployed, stable, the rollback baseline. Only *released* code. | no WIP | — (deploy target) | +| `integration` | Developer mainline. Features merge here to bake, then get promoted. | via merges/PRs | `develop`, per release | +| `refactor/content-extensibility` | The long-running content-extensibility initiative. | via sub-feature merges | `integration`, in slices | +| `feat/*`, `fix/*` | Small, focused changes. | yes | their epic or `integration` | + +`develop` is a *release* branch despite the name — do **not** pollute it with +unreleased code; it must stay something you can pull to recover the live site. + +## The flow + +- **Feature / fix:** cut `feat/` or `fix/` **off `integration`**, do the + work, PR it **back into `integration`**. +- **Release** (e.g. "Summer Patch", "Fall Update"): when `integration` is stable, + merge it to `develop`, tag, deploy. A release is a *promotion event*, not a branch. +- **Hotfix:** branch off `develop`, fix, merge to `develop` **and** back-merge to + `integration` (and the epic) so it isn't lost. +- **Big multi-part effort:** an epic branch (`refactor/content-extensibility`) that + receives small feature branches and graduates stable slices to `integration`. + +### Changes that must reach BOTH `integration` and the refactor line + +Cut them off the **common base** (`integration` / `develop`), not off the refactor +branch — so they merge cleanly into both. (This is why the content-library and +import fixes live on a branch based near `develop`.) + +## Releases are tags, not branches + +A named release ("Summer Patch", "Fall Feature Update") is **not a branch** — it's +the set of changes sitting on `integration` that haven't been promoted to `develop` +yet. Its identity is two things: + +- a **`[Name]` section in `CHANGELOG.md`** (accumulated as work lands), and +- a **git tag** cut when it deploys. + +To ship a release: merge `integration → develop`, `git tag ` (e.g. +`summer-patch`), deploy. The tag is the permanent, immutable marker; `integration` +then keeps collecting the next release. So "the Summer Patch" is simply **whatever is +on `integration` but not yet on `develop`** — no separate branch to manage. + +**Only** cut a `release/` branch when you need **two trains at once** — a +release frozen for final QA while new work continues — so the frozen one takes only +bug fixes on its own branch while `integration` keeps moving. Reach for it then, not +as a standing structure. + +## Changelogs — fold at merge-to-integration + +Every feature/fix branch keeps a `docs/branch-changelog.md` while in flight. Start it +by copying **[`docs/branch-changelog.template.md`](branch-changelog.template.md)** — +that file is the single source of truth for the house style (bullet rules + when a +`## Highlights` is earned); it travels with every branch so no one has to hunt for the +rules here. **When the branch merges into `integration`, fold its entries into the +current in-progress release section of root `CHANGELOG.md` (e.g. `[Summer Patch]`) and +delete the branch changelog — it's consumed.** That keeps root `CHANGELOG.md` a live +reflection of the pending-release diff on `integration`. Folding is a **deliberate +step**, not automatic — don't skip it. + +Two sections get special fold treatment: +- **`## Why this branch exists`** is reviewer context — **stripped at fold**, never + reaches `CHANGELOG.md`. +- **`## Highlights`** (≤3 sentences, only when the branch is an impactful new capability + or behavioral shift — not a bugfix bundle) is **kept**. Decide whether the branch + earns one *before* opening the PR / claiming it done; the PR checklist asks for it. + +Automation: +- **`scripts/fold-branch-changelog.sh ""`** does the mechanical fold — strips + the guidance + Why, keeps Highlights, demotes `## Section` → `**Section**`, inserts a + labeled block into `## []`, and `git rm`s the branch changelog. Review + + commit after; editorial curation stays yours. +- The **`changelog-guard` CI workflow** fails a push to `integration`/`develop` that + still contains `docs/branch-changelog.md` — the unskippable backstop if the fold gets + skipped. +- **`scripts/lint-changelog.py`** checks CHANGELOG.md against the mechanical half of the + house style: bullets under 80 words, one change per bullet, no jargon, and prose only + inside a Highlights block of at most three sentences. It runs in `changelog-guard` on + pull requests as well as pushes, so the feedback arrives while the branch can still be + edited. Run it yourself with `scripts/lint-changelog.py --stats`. +- **`scripts/setup-hooks.sh`** (run once per clone) enables a local **`pre-push`** guard + (`.githooks/pre-push`) that mirrors both CI checks — a fail-fast reminder before the push + leaves your machine. It only reminds; the fold stays a deliberate manual step. + +## Authorship — required, no exceptions + +Every commit must be authored **and** committed as `codeGlaze `. +The repo git config is set to this; be explicit anyway: + +``` +git -c user.name="codeGlaze" -c user.email="github@codeglaze.com" commit -m "..." +``` + +- **Never** author or commit under another identity, and never add a trailer crediting + one — no `Co-Authored-By:`, no session links, no tool banners, in commit messages, PR + titles or bodies, code comments, docstrings or filenames. +- If tooling appends a "generated by" line to a message or PR body, **strip it**. + +## Commit hygiene + +- Confirm the branch first: `git branch --show-current`. **Never push to a different + branch than assigned without explicit permission.** +- Tests green before committing (`lein fig:test && node scripts/test/run-cljs-tests.js` for CLJS; `lein test` for JVM). +- Descriptive messages — the diff is the record; say *why*, not just *what*. +- Compiled artifacts (`resources/public/css/compiled/styles.css` from `lein garden once`, + `resources/public/js/compiled/orcpub.js` from `lein fig:build`) are **gitignored**. + Recompile after a branch switch if you screenshot; never commit them. +- Branch names are descriptive (`feat/` / `fix/`) — no tool-generated + prefixes, no auto-generated gibberish. + +## Grepping Clojure + +`#_` discards the next form, so a grep hit inside one is indistinguishable from live code. +`classes.cljc` carries 160 discarded hits for `:name "` against 346 live ones — a third of +what grep shows in that file never compiles. Non-SRD subclasses are discarded that way, +one subclass per class kept. + +``` +scripts/clj-grep.py 'Beast Master' src/cljc/orcpub/dnd/e5/classes.cljc +``` + +Reports LIVE and DISCARDED separately and names the line each discard opens on. `--live` / +`--dead` narrow to one section as `path:line:text` for piping; `-i` for case-insensitive. + +## Push + +``` +for i in 1 2 3 4; do git push -u origin && break || sleep $((2**i)); done +``` + +Retry with backoff on network errors. Use `--force-with-lease` **only** for a +deliberate history rewrite you were explicitly asked to do (e.g. an author scrub). + +## Pull requests + +- Don't open one unless explicitly asked. +- Target **`integration`**, not `develop` (production stays release-only). +- Mirror the repo's PR template (`.github/PULL_REQUEST_TEMPLATE.md`); fill its sections + from the diff. Keep the body to what the diff does. + +## Where the docs live + +This guide, the changelog files, and the tooling that enforces them — `CHANGELOG.md`, +`branch-changelog.template.md`, `scripts/fold-branch-changelog.sh`, the `pre-push` hook, +and the `changelog-guard` workflow — live on the **main flow** (`integration` → `develop`). +They're what a contributor follows and what CI runs, so they stay here. + +Deeper reference knowledge — how a subsystem works, what a past investigation measured — +is kept off the main flow rather than landing in its history piecemeal. diff --git a/docs/DOCKER-SECURITY.md b/docs/DOCKER-SECURITY.md index 23a0a4e6b..268d89610 100644 --- a/docs/DOCKER-SECURITY.md +++ b/docs/DOCKER-SECURITY.md @@ -111,6 +111,44 @@ app-builder stage doesn't copy secrets into the Docker layer cache. The final `app` image only contains the jar, but intermediate builder layers are cached and can be inspected with `docker history` or extracted from shared CI daemons. +## Credentials in Logs + +A `datomic:sql` URI carries the database password in a query parameter: + +``` +datomic:sql://datomic?jdbc:postgresql://host:5432/datomic?user=datomic&password=hunter2 +``` + +The boot log used to print it verbatim, and it was also embedded in three `ex-info` maps, +which reach logs and error reporters just as readily. Anywhere boot output is kept had the +production password in it: terminal scrollback, container logs, a log aggregator, a +screenshot pasted into chat. + +`orcpub.config/redact-secrets` blanks credentials before anything logs them, in both shapes +they arrive in: + +| Shape | Becomes | +|---|---| +| `?password=` / `passwd` / `pwd` / `secret` / `token` / `api-key` | `password=****` | +| `scheme://user:pass@host` | `scheme://user:****@host` | + +Everything else is untouched, so host, port, database and user still show and the line stays +useful for diagnosing a connection. + +**Two rules when adding to the boot output:** + +1. Anything that might carry a credential goes through `redact-secrets` first. The startup + banner is the most likely place for the next leak, because it is where connection details + naturally get added. +2. Redacting is not a licence to log more. Redact what must be shown; do not print a secret + just because it will be masked. + +Pinned by `test/clj/orcpub/config_redaction_test.clj`, including the exact URI shape from the +production log, alternate parameter names, upper case, and a password containing `=` or `&`. + +**If a URI was ever logged unredacted, rotate the credential.** The fix stops new exposure; +it does nothing about logs already written. + ## Log Directory: `/log` Not `/logs` Datomic's stock config sample uses `log-dir=log` (no trailing s). Our template diff --git a/docs/ENVIRONMENT.md b/docs/ENVIRONMENT.md index 99ebff0eb..197116ab4 100644 --- a/docs/ENVIRONMENT.md +++ b/docs/ENVIRONMENT.md @@ -23,7 +23,7 @@ All configuration is managed via a `.env` file at the repository root. Copy `.en | Variable | Default | Description | |----------|---------|-------------| -| `DATOMIC_URL` | `datomic:dev://localhost:4334/orcpub` | Database connection URI | +| `DATOMIC_URL` | `datomic:dev://localhost:4334/orcpub` | Database connection URI. A `datomic:sql` URI carries the database password; it is redacted wherever it is logged (see [DOCKER-SECURITY.md](DOCKER-SECURITY.md#credentials-in-logs)). | | `DATOMIC_VERSION` | `1.0.7482` | Datomic Pro version for installer | | `DATOMIC_TYPE` | `pro` | Datomic distribution type | | `DATOMIC_PASSWORD` | — | Transactor password | @@ -36,6 +36,113 @@ All configuration is managed via a `.env` file at the repository root. Copy `.en | `SIGNATURE` | — | **Required.** JWT signing secret for authentication. All login and API calls fail without it. | | `ADMIN_PASSWORD` | — | Admin password | +### Capacity + +See [PDF-EXPORT-CAPACITY.md](PDF-EXPORT-CAPACITY.md) for what these cost and how to size them. + +| Variable | Default | Description | +|---|---|---| +| `ORCPUB_HTTP_MAX_THREADS` | Pedestal's own, which is 50 until roughly 16 cores | Jetty worker pool: how many requests of any kind are in flight. The rest queue in the accept backlog. | +| `ORCPUB_PDF_CONCURRENCY` | `max(8, 2 x cores)` | How many character sheets are generated at once. Bounded separately so a rush of exports cannot starve logins and saves. Each in flight holds roughly 11 MB of heap. | +| `ORCPUB_PDF_QUEUE_TIMEOUT_MS` | `30000` | How long an export waits for a slot before the server answers 503 with a `Retry-After` instead of holding the connection open. | +| `ORCPUB_PDF_MAX_RETRIES` | `3` | How many times the busy page retries itself before it stops and waits for the person to click. | +| `ORCPUB_PDF_MAX_CASTER_SECTIONS` | `13` | Most spellcasting sections one sheet may be grown to. Thirteen is every class in the game, which no character can exceed. | +| `ORCPUB_PDF_MAX_CARDS` | `198` | Most **cards** — not pages — of each kind a single export prints. Nine to a sheet, so 198 is exactly 22 sheets. Counted per kind, so spells, items and features are bounded separately. A level 20 wizard's spellbook is about 44. | + +A value that is present but not a positive integer is reported at boot and the +default is used, so a typo does not take the server down or silently mean zero. + +### Share links + +A character's owner can share it with a short link. The server keeps one copy of that character's share +data, checked when it is uploaded. + +| Variable | Default | Description | +|---|---|---| +| `ORCPUB_SHARE_MAX_UPLOAD_KB` | `64` | Largest compressed share data one character's link keeps. Over it, the owner's link carries the data itself instead. A packed level 20 character measured 12 to 25 KB. | +| `ORCPUB_SHARE_MAX_TEXT_KB` | `256` | Largest that share data may be once unpacked. This also bounds what a viewer's browser unpacks. | +| `ORCPUB_SHARE_MAX_ACCOUNT_KB` | `1024` | Share data one account keeps across all its characters. | +| `ORCPUB_SHARE_PRUNE_DAYS` | `180` | Days a share link may go unused, counting only time the server is running, before the link and its share data are deleted. The character is never deleted, and its owner's page says the link expired. `0` keeps links forever; `1` to `29` count as `30`, so a typo cannot delete links a day old. | + +The size limits reach the browser in response headers, so changing one needs no rebuild. Pruning runs +hourly, after the server's heartbeat, which records when the server was off so that time never counts. + +### What the server prints at boot + +Every start prints what it resolved, so you never have to guess whether a change was +picked up: + +``` +------------------------------------------------------------------------------------------------------- + orcpub started +------------------------------------------------------------------------------------------------------- + SETTING VALUE SOURCE + [RUNTIME] + PORT 8080 SET + DEV_MODE - DEFAULT dev-only behaviour and relaxed CORS + [DATABASE] + DATOMIC_URL ...?user=u&password=**** SET + DATOMIC_PASSWORD set + [SECURITY] + SIGNATURE set JWT signing key; every login and API call fails without it + CSP_POLICY - DEFAULT overrides the built-in policy + [EMAIL] + EMAIL_FROM_ADDRESS noreply@example.com SET + EMAIL_ERRORS_TO - DEFAULT + EMAIL_SECRET_KEY set + [CONTENT] + LOAD_HOMEBREW_URL - DEFAULT homebrew loaded at page load + [CAPACITY] + ORCPUB_HTTP_MAX_THREADS 50 DEFAULT worker pool: requests of any kind in flight + ORCPUB_PDF_CONCURRENCY 24 SET sheets generated at once + ORCPUB_PDF_QUEUE_TIMEOUT_MS 30000 DEFAULT how long an export waits for a slot + ORCPUB_PDF_MAX_RETRIES 3 DEFAULT busy-page retries before it waits for a click + ORCPUB_PDF_MAX_CASTER_SECTIONS 13 DEFAULT most spellcasting sections on one sheet + ORCPUB_PDF_MAX_CARDS 198 DEFAULT cards of each kind per export; 9 to a sheet, so 22 sheets + [BRANDING] + 3 of 21 set +------------------------------------------------------------------------------------------------------- + (!) ORCPUB_PDF_MAX_CARDS=oops was ignored: not a positive integer. The default above is in use. + (!!) SIGNATURE is not set, so every login, signup and authenticated API call fails. + It is the key JWT tokens are signed with; without it none can be issued or verified. + Fix: set a long random value and restart, either as a Docker secret at + /run/secrets/signature (preferred) or as the SIGNATURE environment variable. + e.g. openssl rand -base64 48 + Keep it stable: changing it signs out everyone and invalidates unsubscribe links. +------------------------------------------------------------------------------------------------------- +``` + +A setting that breaks the site says **why and how to fix it**, not just that it is missing — +and the same text appears wherever the failure surfaces: the boot banner, the startup +warning, and the `500` a caller receives. An operator reading any one of them is told the +same remedy rather than having to work it out from the symptom. + +| Column | Meaning | +|---|---| +| `SET` | the value came from the environment | +| `DEFAULT` | nothing usable was set; this is what is running anyway | +| `set` / `NOT SET` | a **secret**. Its value is never printed, only whether it is present — no `SOURCE`, because `DEFAULT` beside `NOT SET` would read as though there were a default password | +| `(!)` line | **a value was given and rejected.** Its row reads `DEFAULT` because the default is what is running; this line names what was thrown away | +| `(!!)` line | a setting whose absence breaks the site, called out rather than left to be spotted among thirty rows | + +Everything the app reads is reported, grouped: runtime, database, security, email, content +and capacity, plus a count of the 21 `APP_*` branding variables rather than 21 more rows. + +**These are the values actually in force, not a restatement of the documentation.** They are +read from the same functions the running code reads. `ORCPUB_HTTP_MAX_THREADS` is the +exception worth understanding: when it is unset we hand Pedestal nothing and Pedestal +chooses, so the number shown is read back off the **live Jetty thread pool** after it starts +— not a formula copied from Pedestal that would drift. If the pool cannot be read the column +shows `-` rather than a guess. + +The source column always describes where the **shown value** came from. A rejected setting +therefore reads `DEFAULT` — it was your value that was ignored, not the default — and the +`(!)` line under the table is what tells you to go fix it. + +The banner fires on Jetty's own started event, so `orcpub started` means listening, and it +prints as a single write — printed line by line, Jetty's logging interleaved with it and cut +the table in half. Plain ASCII, no colour: it is read in aggregators as often as terminals. + ### Plugins | Variable | Default | Description | @@ -48,6 +155,19 @@ All configuration is managed via a `.env` file at the repository root. Copy `.en |----------|---------|-------------| | `CSP_POLICY` | `strict` | Content Security Policy mode: `strict`, `permissive`, or `none` | | `DEV_MODE` | `"true"` (in :dev profile) | Enables dev-mode CSP (Report-Only instead of enforcing). Must be the string `"true"` (case-insensitive) -- any other value (including `"1"`, `"yes"`, or empty) is treated as false. | +| `ORCPUB_PWNED_CHECK` | `on` | Screen new passwords against the Have I Been Pwned corpus. Accepts `on`/`off`, `true`/`false`, `1`/`0`, `yes`/`no`. | + +**About the breach check.** Signup and password reset send the first five characters of +the password's SHA-1 to `api.pwnedpasswords.com` and score the reply locally, so the +password itself never leaves the server and the size of the reply is padded. A password +is refused only at **1000 or more** appearances: the corpus measures how common a +password is, not whether this person was breached, and refusing on a single appearance +turns that measure into a veto. Below the line the strength meter says so while somebody +is still typing. + +Set `ORCPUB_PWNED_CHECK=off` as the kill switch. The check already fails open on a +timeout, so this is for the case where the service is not down but slow enough to be felt +on every signup. Turning it off changes nothing else. CSP modes: - **strict** — nonce-based CSP with `strict-dynamic`. Dev mode uses `Report-Only` header (logs violations but doesn't block). Prod uses enforcing header. @@ -92,6 +212,15 @@ See `docker/transactor.properties.template` for the full transactor configuratio | `ORCPUB_ENV` | — | Set to `dev` to enable `add-test-user` in user.clj | | `FIGWHEEL_PORT` | `3449` | Figwheel WebSocket port for frontend hot-reload. Read by `scripts/common.sh`. | | `FIGWHEEL_CONNECT_URL` | *(auto-detected)* | Figwheel WebSocket URL override for remote environments (Gitpod, tunnels). Auto-detected for GitHub Codespaces. Example: `wss://my-remote-host:3449/figwheel-connect` | +| `E2E_MAIL_PORT` | `2525` | Port for the browser suites' throwaway SMTP server. `scripts/e2e/run.sh` starts it and points `EMAIL_SERVER_URL`/`EMAIL_SERVER_PORT` at it. | + +**Why the suites need a mail server.** The verification key and the password-reset key +exist only inside an email, and registration fails outright when no mail can be sent -- +the verification mail goes out inside the same try as the transaction. So signup, +verification and reset could not be driven end to end at all, and every suite worked +around it by starting from a user seeded straight into Datomic. `scripts/e2e/lib/mail-sink.js` +is a dependency-free SMTP server that accepts anything and hands the message back, which +is what lets `auth-flows.js` follow the link a real person would click. ### Branding, Social Links & Integrations diff --git a/docs/ORCBREW_FILE_VALIDATION.md b/docs/ORCBREW_FILE_VALIDATION.md index 75577eb02..6dcdd5515 100644 --- a/docs/ORCBREW_FILE_VALIDATION.md +++ b/docs/ORCBREW_FILE_VALIDATION.md @@ -127,8 +127,8 @@ The system automatically fixes these common corruption patterns: | `disabled? nil` | `disabled? false` | | `nil nil,` | (removed) | | `:field-name nil` | (removed) | -| `option-pack ""` | `option-pack "Default Option Source"` | -| Empty plugin name `""` | `"Default Option Source"` | +| No `:option-pack`, or `""` or `nil` | The name of the source the entry is filed under; in a single-source file, the one source the other entries name; otherwise `"Default Option Source"` | +| Empty plugin name `""` | `"Unnamed Content"` (numbered when there are several) | **This happens automatically** - you don't need to do anything! diff --git a/docs/PDF-EXPORT-CAPACITY.md b/docs/PDF-EXPORT-CAPACITY.md new file mode 100644 index 000000000..cba73b9f5 --- /dev/null +++ b/docs/PDF-EXPORT-CAPACITY.md @@ -0,0 +1,287 @@ +# PDF export capacity + +How much a character sheet costs to generate, what that means for a server, and +which settings to reach for. Written for whoever runs an instance and has to +decide how much machine to give it. + +Every number here was measured on the code as it ships, on a four-core host with +Datomic in memory. The last section says how to reproduce them. + +## The short version + +- A sheet costs about **11 MB of heap while it is being made**, and nothing once + it is sent. +- The server holds **35 MB** regardless of traffic, for the runtime and the + templates. +- **A thousand people exporting at once is a queue, not a memory problem.** It + drains in under a minute and needs well under a gigabyte. +- If you have cores and RAM to spare, raise `ORCPUB_PDF_CONCURRENCY`. That is + the setting that matters. + +## What one sheet costs + +Generating a sheet means opening a template, adding a page per spellcasting +class, writing the character's values into the form fields, and saving. Style 1, +per export: + +| character | heap while generating | time | file | +|---|---|---|---| +| no spellcasting | ~8 MB | 90 ms | 245 KB | +| one casting class *(most characters)* | ~11 MB | 218 ms | 285 KB | +| two casting classes | ~11 MB | 229 ms | 309 KB | +| six casting classes *(the worst the app allows)* | ~11 MB | 388 ms | 409 KB | + +The heap figure is flat because the cost is dominated by having a document open +at all, not by how many pages it has. + +## Why you may see a much larger number + +Profiling this code reports **allocation churn** — every byte the export ever +asked for — and for a six-class sheet that is 162 MB. It is a real number and it +is not memory consumption. Nearly all of it is dead the instant it is created, +and the JVM reclaims it without effort. + +The distinction is worth internalising because the two numbers differ by a factor +of fifteen: + +- **Churn** is throughput. It costs CPU, because the garbage collector has to + sweep it. It does not accumulate. +- **Footprint** is what is alive at once. It is what your heap has to hold, and + it is the small number. + +The proof is direct: **two hundred** six-class exports back to back complete in a +**48 MB** heap and fail at 40 MB. If churn were footprint, that could not happen. + +The cost of churn shows up as time, not capacity. The same two hundred exports: + +| heap | time per export | +|---|---| +| 64 MB | 362 ms | +| 48 MB | 543 ms | + +Give the collector room and the garbage is nearly free. Starve it and the export +slows by half. + +## What a thousand people at once means + +It does not mean a thousand exports at once. Two limits stand in the way, and +both are deliberate. + +Jetty's worker pool caps how many requests of any kind are in flight — Pedestal +sizes it at `(max 50 ...)`, so 50 until roughly sixteen cores. Inside that, +`ORCPUB_PDF_CONCURRENCY` caps how many of those may be generating a sheet, so a +rush of exports cannot take the workers that logins and saves need. + +Everyone else waits, which costs a socket and nothing else. Heap needed against +exports actually in flight: + +| concurrent exports | heap floor | per export | +|---|---|---| +| 8 | 96 MB | 7.6 MB | +| 32 | 384 MB | 10.9 MB | +| 50 | 448 MB | 8.3 MB | + +And the drain, measured by running exactly one thousand exports on four cores +with a 1 GB heap: + +| character | 1,000 sheets | rate | +|---|---|---| +| one casting class | 28.9 s | 34.6/sec | +| two | 46.7 s | 21.4/sec | +| six | 73.3 s | 13.6/sec | + +So in the realistic case the thousandth person waits under thirty seconds. Even +if every one of them were a six-class multiclass it is about seventy, and it +falls proportionally as you add cores. + +Budget for the request bodies as well: the handler caps a body at 2 MB, so 50 in +flight can hold 100 MB before any of it is parsed. **1 GB is comfortable for the +whole path.** + +## Settings + +All optional. A value that is present but not a positive integer is reported at +boot and the default is used, so a typo cannot take the server down or silently +mean zero. + +| Variable | Default | What it does | +|---|---|---| +| `ORCPUB_HTTP_MAX_THREADS` | Pedestal's, which is 50 until ~16 cores | Jetty's worker pool: requests of any kind in flight. | +| `ORCPUB_PDF_CONCURRENCY` | `max(8, 2 x cores)` | Sheets generated at once. | +| `ORCPUB_PDF_QUEUE_TIMEOUT_MS` | `30000` | How long an export waits for a slot before the server says it is busy. | +| `ORCPUB_PDF_MAX_RETRIES` | `3` | How many times the busy page retries itself before waiting for a click. | +> Every start prints these with their resolved values and where each came from — `set`, +> `default`, or `IGNORED` when a value was given but rejected. See +> [ENVIRONMENT.md](ENVIRONMENT.md#what-the-server-prints-at-boot). Read the boot log rather +> than inferring from behaviour whether a change took effect. + +| `ORCPUB_PDF_MAX_CASTER_SECTIONS` | `13` | Most spellcasting sections one sheet may be grown to. Thirteen is every class in the game. | +| `ORCPUB_PDF_MAX_CARDS` | `198` | Most cards of one kind a single export prints — nine to a sheet, so 22 whole sheets. A level 20 wizard's spellbook is about 44. | + +### Sizing them + +**`ORCPUB_PDF_CONCURRENCY` is the one to think about.** Two ceilings apply and +you want the lower: + +- **Heap.** An export in flight holds ~11 MB, so the limit is about + `(usable heap - 100 MB) / 11 MB`. A 4 GB heap allows roughly 350. +- **Cores.** Throughput is bounded by CPU. Raising the limit past what the cores + can chew through lengthens the queue without shortening anyone's wait — it + just means more people are half-served at once instead of quickly served in + turn. + +Two cores' worth of concurrency is a sound starting point, which is what the +default gives you. On a sixteen-core host with plenty of RAM, 32 to 64 is +reasonable; going to 300 because the heap allows it will make things worse, not +better. + +Raise `ORCPUB_HTTP_MAX_THREADS` alongside it if you set the export limit above +the HTTP pool, or exports will simply queue a level higher up. + +## What people see when it is saturated + +An export that cannot get a slot within `ORCPUB_PDF_QUEUE_TIMEOUT_MS` is answered +`503` with a small page saying the server is busy — and that page **retries itself**. + +The export is a plain form POST into a new tab, so the 503 response is the page +the person is already looking at. The retry lives there rather than in the app: +the page carries the original request body forward in a hidden field, counts down, +and resubmits. After `ORCPUB_PDF_MAX_RETRIES` it stops and waits to be clicked. +Nothing in the character builder changed, and how the finished PDF arrives is +untouched. + +The countdown is measured rather than guessed. The server keeps a weighted mean of +how long recent exports actually took, divides it into the queue ahead of the +caller, floors the result at one second and caps it at thirty — so it tracks your +host and the sheets people are really asking for. The page then applies ±25% +jitter, so a crowd turned away together does not come back in the same instant. + +The retry count is deliberately small. The queue drains in under thirty seconds in +the realistic case, so three attempts covers it; the point is to spare someone a +wait they would abandon anyway, not to retry forever. The counter is read back from +the page's own hidden field, and a hand-edited value counts as a first try rather +than buying extra attempts. + +The page looks like the rest of the site: the app's fixed gradient ground, its +`#1a1e28` panel, white type, the header bar with the logo, and `.form-button` for +the button. Its rules live in `orcpub.styles.core` with the rest of the +stylesheet, so `lein garden once` has to have run for it to look right. The +builder's markup and scripts are absent in this tab, so the page restates the +ground and panel rather than reusing app layout classes. + +### Seeing it on a dev machine + +The default limits are too generous to reach by hand, so there is a profile that +shrinks them — one sheet at a time, a quarter-second wait for a slot, two +self-retries: + +``` +lein e2e-server-busy +``` + +Then click Export in the builder twice in quick succession, or drive the whole +thing: + +``` +node test/browser/export_busy_retry_e2e.js +``` + +That test goes through the real UI — Export, pick a sheet style, Create PDF — +while holding every export slot, and checks that the new tab lands on the busy +page, carries the site header, retries itself unattended, stops at the limit, and +delivers the sheet once the rush passes. + +Measured with those limits: twenty simultaneous exports produced two sheets and +eighteen busy pages carrying `Retry-After` values of 1 to 5 seconds. At the +defaults, the same twenty all returned sheets. + +One thing to expect: a browser logs `Failed to load resource: 503` in its console +for each turned-away attempt. That is the browser reporting an HTTP status, not a +script error, and it is the correct status to send — a `200` would lie to caches, +crawlers and any non-browser client. + +## Reproducing the measurements + +Heap floors come from running a fixed number of exports under a constrained +`-Xmx` and bisecting until it fails: + +``` +java -cp "$(lein classpath)" -Xmx48m clojure.main your-export-loop.clj +``` + +Note that `lein classpath` includes `dev/`, whose `user.clj` loads Datomic at +REPL init and will not fit in a small heap. Drop that entry before running. + +Churn is measured with `ThreadMXBean.getThreadAllocatedBytes` around the call. +Do **not** use `totalMemory - freeMemory` for either question: it counts +allocation not yet collected and misses allocation already collected, so it +answers neither churn nor footprint. + +## What bounds the work one request can buy + +The queue limits how many exports run at once and how long a request waits for a +slot. Neither limits how long a slot is HELD, so the work inside one export has to +be bounded too. Two inputs decide it and both come from the caller: + +- **The spellcasting section count** is the largest N in any `spellcasting-class-N` + field NAME. Unbounded, a body of a few dozen bytes asked for thousands of cloned + pages at about 14 MB each: measured, `spellcasting-class-9999` ran for 310 + seconds and died with an out of memory error. Clamped by + `ORCPUB_PDF_MAX_CASTER_SECTIONS`, the same request answers 200 in 0.8 s. +- **The card count** is however many spells or items the caller lists, nine to a + page. A 2 MB body holds about 60,000 spell entries -- some 13,000 pages, a + quarter of an hour with a slot held throughout. Clamped by + `ORCPUB_PDF_MAX_CARDS`; a request for 4,000 now answers in 2.2 s and logs that + it printed the first 200. + +Note that the section count is derived in TWO places -- the handler and +`add-missing-spell-pages!` -- and clamping only the handler left the endpoint +exactly as vulnerable. That was caught by firing the request at a running server +rather than reasoning about the code. + +**So the ceiling is applied to the request, not to the generators.** +`routes/bound-request` runs once on the parsed body before any part of the export +sees it, and does two things: a `spellcasting-class-N` name past the section +ceiling is dropped outright, and every collection is truncated to the card +ceiling. Because the field never arrives, no downstream reader can derive a number +too large however many of them there are -- which is the failure the first fix +walked into. + +The point is that it holds for code nobody has written yet. A feature added later +that reads a list out of the request, or counts `spellcasting-class-N` names, is +bounded without being wired up. `export_capacity_test` proves this the only way +worth proving it: it disables every per-site clamp and checks the request is still +bounded, so if the boundary ever regresses the test fails rather than the +generators quietly covering for it. + +The per-site clamps stay as defence in depth. They are no longer what is doing +the work. + +## Character images and your deployment + +A character's portrait is a URL, and the **server** fetches it while building the +sheet. That is the only outbound request the app makes to an address a visitor +chose, so it is bounded on every axis: http and https only, private and reserved +addresses refused, redirects not followed, 128 KB, 2000x2000, and 20 seconds of +transfer. The host is resolved **once** and the connection is made to that answer, +so a name that resolves differently the second time cannot be used to slip past +the address check. + +Three deployment questions that follow from it: + +- **Behind nginx?** Yes, and nothing to configure. `deploy/nginx.conf.template` + is an inbound reverse proxy; the image fetch is outbound and never passes + through it. +- **In Docker?** The first resolution uses the container's own DNS, so Docker's + embedded resolver works normally. Note that `10/8`, `172.16/12` and `192.168/16` + are refused, so a character image URL cannot be pointed at another container on + the compose network. That is deliberate, and it is the same rule that stops it + reaching your database or the cloud metadata endpoint. +- **Behind an egress proxy?** Also nothing to configure. If the JVM is started + with `https.proxyHost` (or the usual proxy properties), the app detects it and + lets the proxy do the resolving, because it is then the egress control point. + The shipped compose files set no proxy, so the resolve-once path is what runs by + default. + +If you deliberately want the server to reach an internal image host, there is no +allowlist for it today — the refusal is by address range, with no exceptions. diff --git a/docs/README.md b/docs/README.md index 5cc437d52..dbbce5925 100644 --- a/docs/README.md +++ b/docs/README.md @@ -11,11 +11,16 @@ Guides for developers and power users working with OrcPub's homebrew content sys - [📋 Required Fields Guide](HOMEBREW_REQUIRED_FIELDS.md) - Required fields per content type **For Developers:** +- [🌿 Contributing & Workflow](CONTRIBUTING.md) - Branching, releases, changelog fold, commit & PR conventions - [ClojureScript Type Tolerance](clojurescript-type-tolerance.md) - What cljs silently tolerates vs. what crashes (and why); the string-op crash pattern behind render black screens - [🚨 Error Handling](ERROR_HANDLING.md) - Error handling utilities - [🗡️ Language Selection Fix](LANGUAGE_SELECTION_FIX.md) - Ranger favored enemy language corruption (#296) - [🐳 Docker User Management](docker-user-management.md) - Verified user setup for Docker deployments +**For Operators:** +- [🖼️ Character Image Fetch](CHARACTER-IMAGE-FETCH.md) - Why a portrait is missing from a PDF, the limits on the fetch, and how nginx/Compose/Swarm/egress proxies affect it +- [📄 PDF Export Capacity](PDF-EXPORT-CAPACITY.md) - What an export costs, and how to size the export stack + ## Key Design Decisions ### Why Progressive Import? @@ -56,7 +61,9 @@ Guides for developers and power users working with OrcPub's homebrew content sys **Gotcha:** Two different "key" concepts exist: language `:key` (data keyword like `:aquan`) and option `::entity/key` (derived from display name via `name-to-kw`). The fallback must produce names that round-trip correctly through `key-to-name` / `name-to-kw`. -> [LANGUAGE_SELECTION_FIX.md](LANGUAGE_SELECTION_FIX.md) +→ [LANGUAGE_SELECTION_FIX.md](LANGUAGE_SELECTION_FIX.md) + +### Why Centralized Error Handling? **Problem:** Inconsistent error handling across codebase. Some code logged, some didn't. User messages inconsistent. diff --git a/docs/TODO.md b/docs/TODO.md index b24f5f5b8..724287f9f 100644 --- a/docs/TODO.md +++ b/docs/TODO.md @@ -2,10 +2,13 @@ ## Datomic transactor crashes — investigate Postgres migration -**Status:** Open -**Severity:** Critical — transactor crashing 3–5× per day, 2–3 min downtime each -**Reported:** 2026-02-26 -**KB doc:** [docs/kb/datomic-crash-analysis.md](kb/datomic-crash-analysis.md) +**Status:** Unverified — no recurrence observed since the report; re-measure before +acting. The analysis below is from the crashing period and has not been retested +against the current stack (the peer is on Datomic Pro now, which the analysis +predates). +**Severity:** Critical *if it still happens* — at time of report, 3–5 crashes per +day at 2–3 min downtime each +**Reported:** 2026-02-26 ### Summary @@ -36,7 +39,6 @@ already done). What remains is the **storage backend migration**: - `docker/datomic/` — transactor container and config templates - `docs/migration/datomic-pro.md` — peer library already migrated to Pro -- `docs/kb/datomic-crash-analysis.md` — full root cause analysis with log evidence --- @@ -85,3 +87,240 @@ otherwise stubbornly persist. - `src/cljs/orcpub/dnd/e5/db.cljs` — `reg-local-store-cofx` (line ~252) - `src/cljs/orcpub/dnd/e5/events.cljs` — `set-combat-path-prop` nil guard - All `*->local-store` serializers use `(str data)` / `reader/read-string` + +--- + +## Content-library management — remaining work + +**Status:** Open +**Severity:** Low — enhancements; the shipped resolution already removes the data bug + +The My Content library, duplicate-key resolution, disabled-reason badges, the +mutual-exclusion UX, the opinionated (summary-first) import, the four-level +disable hierarchy, and move/copy content between sources are built and on +`feature/content-library-management` (see the KB doc for how they work). These are +the not-yet-built follow-ups, roughly in dependency order. + +### Example / demo content tier + +**Status:** Being built on `feature/demo-content-tier` — tracked there, not here. + +A read-only example/demo tier of content, with a per-account version marker and +copy-on-edit graduation (editing an example copies it into the user's own library +so upstream updates never clobber their edits). + +### Also parked (with reasons — do not lose) + +- **Account backup/restore** of libraries + prefs: blocked NOT by code (the + `share_url` codec already does gzip + fail-closed decode) but by **legal** + (hosting user-uploaded, often copyrighted content), **database/scale** (3–5 MB × + every user), and standing **admin resistance**. If ever entertained: server-side + at-rest/transport encryption, NOT end-to-end (lost key → lost backup); and + backup-restore (last-write-wins) *before* multi-device sync (sync needs conflict + resolution). +- **Compress localStorage plugins** with the existing gzip codec to fit more under + the browser ceiling — no cloud, no legal exposure. Caveats: makes stored + content opaque to inspection, and a hard cap is still needed (compression moves + the ceiling, doesn't remove it). The ceiling is now measured rather than assumed: + **5,177,344 characters** in Chromium, identical for ASCII and CJK fills, so it + counts UTF-16 units and compares directly against a library's character count + (`test/browser/localstorage_ceiling_e2e.js`). +- **Chunked per-source library storage** — *nice to have, not scheduled.* The + library is 13 sources in memory but ONE localStorage value (measured: 2,166,081 + chars for MegaPak), so every save rewrites the whole thing and every load parses + it back through a single blocking `read-string` (~750 ms). A full phased design + exists. + **Deliberately parked**: that ~750 ms is a one-time load cost, while the reported + problem is the click loop (race/class selection), where no storage read happens + at all. It also does not raise the ceiling. Revisit if cold load becomes the + complaint — or fold it into an IndexedDB move, which dissolves most of the plan's + complexity (natively per-key and async) and is the only real answer to capacity + (~916 MB of measured origin quota vs ~5.18 M chars for localStorage). +- **Native `` in the app (`test/browser/select_option_census_e2e.js`, +run against the WotC megapack) found only one picker outside Equipment that is +large: + +| Page | selects | total options | biggest | +| --- | --- | --- | --- | +| combat-tracker | 4 | 972 | **969** | +| monster-builder | 37 | 807 | 36 | +| class-builder | 8 | 67 | 22 | +| magic-item-builder | 18 | 44 | 9 | +| spell-builder | 2 | 18 | 10 | + +`monster-selector` (`src/cljs/orcpub/dnd/e5/views.cljs:7754`) renders all of +`::monsters/sorted-monsters` into one native `` is the right control +and should be left alone. + +The blocker is not size but semantics: `inventory-combobox` is an *add to a list* +control (it dispatches, clears the query and closes), while `monster-selector` picks +a *single current value* that must stay displayed. Generalising means parameterising +the selected-value display and the on-pick behaviour. diff --git a/docs/branch-changelog.md b/docs/branch-changelog.md new file mode 100644 index 000000000..7b32321a9 --- /dev/null +++ b/docs/branch-changelog.md @@ -0,0 +1,435 @@ +# Branch changelog — `integration-local` + +## Why this branch exists + +Hotfixes found while testing PR #33 before the roll-up to gitea. Several repairs +ran again on every load because nothing saved them, the save banner's export link +had regressed, damaged homebrew loaded without a word and failed later at export, +and a few notices and controls misreported or covered what they sat on. Each fix +comes with a check that fails without it. + +## Highlights + +Signing up, signing in and recovering an account have been rebuilt: passwords are judged +by length and shape rather than by character classes, screened against the breach corpus +and refused only when they are egregiously common, and a strength meter says so while you +are still typing. The nine account pages now share one shell, and a login no longer tells +a stranger whether a username exists. + +## Added + +- **New homebrew keys carry their source's tag** — two authors' "Stone Elf" become + `:stone-elf-trcs` and `:stone-elf-kbtx` instead of colliding, the name stays as typed, and a + source can set its own tag from My Content. The builder also gains a key row to change a key + on purpose (`ed4f5a4c`, merged in `e2457ff7`). The duplicate-save gating that came alongside it + was reverted (`d689a310`): combined with this, it let a save overwrite another library's item, + and it is being reworked on its own branch. +- **A check for homebrew no repair knows about** — a browser probe breaks one race on + conversion and one only when drawn, and checks the app, the notice, the set-aside, + Restore and import (`fa57ebbc`). + +- **A check that every fix is saved** — a browser probe applies each repair through + the app, reloads and exports, so a fix that only lives on screen fails a test + instead of reverting on refresh (`dd2ff2b7`). + +- **A character brings its custom items from the server** — whoever can open a character now sees the + custom items it has equipped, as they are now. Share links no longer carry items, so they are + shorter; homebrew still travels in the link, and older links still open (`7f375829`). + +- **Short share links** — a character's owner gets a link of about 85 characters that stays the + same and always shows the character's current homebrew, which the server keeps and checks; New link + revokes every earlier link, nothing is stored until the owner presses Share link, the limits are + config settings, and anyone else's link still carries the homebrew itself (`20179535`, `191b612e`, + `eab46b9f`, `e43b41ca`). + +- **A party keeps a shared character's homebrew** — a character added to a party from its share link + shows its homebrew on the party page, until its owner makes a new link (`c69dcf9d`). +- **Unused share links expire** — a share link nobody opens for 180 days of the server running is + deleted with its share data; the character stays, the owner can share again, and + `ORCPUB_SHARE_PRUNE_DAYS` sets the window, 30 days at least. A request without the link's token does + not count as use, and time the server was off does not count against a link (`59400455`, `110569dc`, + `3895027b`). +- **Stop sharing** — a character's owner can stop sharing it at any time: every link made before stops + showing its custom content, and nothing is stored again until Share link is pressed (`e3aa6d07`). +- **The owner is told when a share link expired** — the character page and list show the date beside + Share link on every visit, until the owner shares again or dismisses the note; the note keeps only the + character and the date (`e3aa6d07`). +- **The server records when it was running** — an hourly heartbeat kept in the database notes when the + server was off, and scheduled jobs run after it; share link pruning is the first (`e3aa6d07`). + +- **The password rules judge length and shape, not character classes** — a minimum of 12 + characters, with repeats, runs and too few distinct characters refused, and no rule about + symbols or capitals. NIST has told everyone to stop imposing composition rules, which push + people to Password1! and no further. The validator is shared between the browser and the + server so neither can disagree with the other (`43a56c60`, `ad5ebcb8`, `389b0360`). + +- **A strength meter that asks the validator instead of guessing beside it** — four rungs at + 12, 16, 22 and 28 characters, and it refuses to show a verdict the server would contradict. + The username and email go into the judgement, so a password made of them does not score + (`389b0360`). + +- **Breach screening against the Pwned Passwords corpus** — the first five characters of a + SHA-1 go to the service and the rest never leaves, so the password itself is not sent and + cannot be inferred from the size of the reply. A password is refused only at 1000 or more + appearances: the corpus measures commonness rather than danger to the person in front of + us, and refusing on a single appearance turned a measure into a veto. A service that cannot + answer is never a refusal (`b459e6b7`, `79e7a325`, `cef432af`). + +- **A notice when one account is signed into from several places** — with the time, the + browser and where from, held to one message per account per cooldown so a person with a + phone and a laptop is not mailed all day (`b2aefc9b`, `4f75360c`, `40d6ea23`). + +- **A browser suite for signing up, verifying, signing in and recovering an account** — + `scripts/e2e/auth-flows.js`, twenty-two checks. These flows had no coverage and no way to + get any: the verification key and the reset key exist only inside an email, registration + fails outright when no mail can be sent, and every other suite starts from a user seeded + straight into Datomic. `scripts/e2e/lib/mail-sink.js` is a dependency-free SMTP server + that hands the message back, and `run.sh` starts it and points the app at it, so a suite + follows the link a person would click. It pins the things nobody states: that a wrong + password and an unknown username answer identically, that an unknown address gets the + same answer and no mail, that five wrong guesses turn away the sixth wrong guess but the + owner still gets in, and that after a reset the new password logs in and the link cannot + be used twice (`e147938e`, `b30876ee`). + +## Fixed + +- **Signing out everywhere cannot be undone by the hourly refresh** — a refresh that read the + database just before a reset or sign-out-everywhere committed replaced the register with that + older read, and the withdrawn sessions worked again for up to an hour (`e4569552`). +- **A password is judged as it is saved** — signup and reset checked it as typed but stored it + trimmed, so `" pomegranate "` passed the 12-character minimum and was saved at 11 (`465666df`). +- **Moving a class or race keeps its subclasses and subraces** — a move into a source that already + had that key renamed the item but left its dependents on the old key, where they joined the + other source's item. They now follow it, in a bulk move too, and the old key is recorded + (`054e42c1`, `ec4a3c50`). +- **Blank icons show again** — the Edit button on character and item pages, New link and three + import-log lines used Font Awesome 4 names the app does not serve, so on a phone they were empty + buttons (`b50783d6`). +- **A mistyped `ORCPUB_SHARE_PRUNE_DAYS` is called out at boot** — it showed as set while the default + was in use (`04cb322b`). +- **Broken keys and card lists in homebrew are repaired** — an entry whose key was not + a keyword, or whose traits or options held something other than cards, crashed the + import or later the export; it now imports with what can be kept (`c0b757c7`). +- **Startup only reads homebrew** — the character options autosave needs are built on + the first save instead of on every page load, and a library that cannot be loaded is + set aside intact with a download, so the app still starts (`fa57ebbc`). +- **Homebrew that still breaks is set aside, not left to break the page** — an entry + that fails to build is set aside with a notice that names it and links to My Content, + and a page that fails on homebrew checks it and loads again on its own (`fa57ebbc`). + +- **Bad stored homebrew no longer stops the app from starting** — a race whose key + was text left every page on the loading spinner. A failed startup step no longer + stops the app mounting, and sorting keys no longer breaks on mixed types + (`e9548269`). + +- **A character is repaired on refresh too, and stays repaired** — the repair is + saved back to the builder's draft, so it happens once instead of on every load, + and its notice shows after the page navigates instead of being cleared by it + (`76dd9497`, `ae6be67b`). +- **Entries set aside on load leave the stored library** — they were set aside in + memory only and came back on the next refresh (`d3880ecb`). +- **The save banner's export link exports the source as it is now** — it exported + the copy it remembered from when the banner appeared, dropping later edits + (`44e97206`). +- **Damaged homebrew is repaired on import and on load** — a section stored as + text or as a list comes back with everything in it, and a file whose whole + content was stored as text imports instead of crashing. What nothing can read is + set aside with Export raw and Discard (`9ef8bfe5`, `a4c69f62`). +- **A partial import shows as a warning** — it arrived in the green success card. + Each line is now marked imported, repaired or skipped, and counts read "1 item" + (`a89b2102`). +- **A nameless item reads "Unnamed Spell" in the missing-fields dialog** — with its + key beneath, instead of a bare ":no-name" (`bd664a96`). +- **The import-log button no longer covers header menus** — it stays off the page + while the log is empty and draws under an open menu (`e30e0061`). +- **The save sparkle stays on the save button** — it drifted into the gap beside + it (`bcf713d6`). +- **A library stored as text loads again** — it loaded nothing (`0b5c05be`). +- **A stored value that isn't a library is handled once** — it was copied aside + again on every load (`0b5c05be`). +- **Save anyway in the selection builder keeps the selection** — a name like + "9 Lives" was saved under a key the next load set aside (`0b71b8b0`). +- **Renames chosen for existing items are kept when nothing incoming imports** — + they were dropped along with the incoming entries (`01217a11`). +- **An entry with no source takes the name of the source it sits in** — unless its + source field was there but blank, it was skipped on import or set aside on load. + Only an entry with no name to take goes to Default Option Source (`1f8fef2b`). +- **An import with an entry that isn't a map no longer crashes** — the entry is + skipped and listed in the import log (`09c22e11`). + +- **The email-support button on a character that will not load sends the report again** — clicking + it threw after the login change, so no report went out (`476e948c`). +- **A login the server has stopped accepting now logs you out** — the app kept showing you signed in + while every request failed, until you logged in again or reloaded (`6f1e6309`). +- **Share links no longer carry item ids or your username** — each custom item went into the link as + stored, with its database ids and its owner; links made before this drop both when opened + (`c34075f2`). +- **Only its owner can read a custom item by its id** — anyone with the id could fetch the item and its + owner's username; everyone else now gets "not found" (`2cae044b`). +- **A character page never shows an email address** — characters saved during nine days in May 2017 + named their owner by the email used to log in (`2cae044b`). +- **Logins, API calls and PDF downloads go to the server that served the page** — every + http://localhost page sent them to port 8890, which broke the Docker setup opened at + http://localhost and a server on any other local port (`d72bcc39`). + +- **Copy link shares the character it sits beside** — on the character page and in the character list + it carried the homebrew of whatever character was open in the builder (`a92b47b7`). +- **Pasted images and video are left out of shared homebrew** — a data: URI in any text field is + emptied before sharing and again when a link is opened (`ac2e45bf`). + +- **Only characters can be added to a party** — adding checked nothing, so any id could be added + (`c69dcf9d`). + +- **The mobile header no longer crowds itself** — on a phone the logo is capped, a + full-width child no longer measures wider than the bar it sits in, the import log + panel cannot exceed the screen, and a child that still outruns the bar is clipped + rather than scrolling the whole page sideways (`0a089349`). + +- **Login no longer says whether a username exists** — every failure answers the same way, and + the per-address throttle is consulted BEFORE the credentials rather than computed and thrown + away, so a spray is turned back without first being told whether it guessed a real account + (`5f4d34ba`, `70c3f49a`). + +- **Password reset no longer says whether an address is registered** — it answers 200 either + way, including once throttled, because an endpoint that changes its answer under a limit has + simply moved the oracle (`3570624e`). + +- **Reset keys are stored as a digest and expire** — the table holds a SHA-256 of the key + rather than the key itself, and it is good for two hours. An unknown key used to fall through + to signing a token for username nil (`cfd39eb9`). + +- **Registration is capped per host** — ten an hour, counted only once a form was otherwise + going to succeed, so a signup that was failing anyway is not held against the address. What + the limits turn away is counted and summarised hourly, so the numbers can be tuned against + something (`91acfcd1`, `cc5f8653`, `a744b6d4`). + +- **A margin utility no longer makes 194 places bold** — `.m-b-10` sat in a Garden selector + group that read as a descendant chain and was not one, so a margin class carried + `font-weight: bold` across the app. Measured rather than guessed: 194 uses in source, 11 + rendering on the logged-out pages, 7 computing a different weight, 5 actually looking + different (`042ee596`). + +- **The legal links printed twice on the register page** — the consent line links both + documents and the footer printed the same pair under it. The footer keeps the copyright there + and drops its copies (`ecb13193`). + +- **The gryphon panel tiled** — no `background-repeat` and no `background-size`, so once the + form column outgrew the image a second half-cropped gryphon drew below the first. It is a + Garden class now instead of an inline style map (`ecb13193`). + +- **The email help link was one word in the middle of a sentence** — "whitelist" as the whole + clickable target, which is a small target and names no destination when a screen reader reads + it alone. Both instances link a phrase now, and the login copy no longer reads "Didn't receive + validation the email?" or advise resetting a password to fix a missing validation email + (`ecb13193`). + +- **Sign out everywhere** — an account can shut every session from My Account, including + the one asking. It does NOT take the password: somebody who thinks their account is open + somewhere they cannot reach should be able to close it from the session they already + hold, and an attacker who has one gains nothing by signing themselves out. Stored as its + own attribute rather than reusing the reset timestamp, because signing out must not + require changing the password; the register keeps the later of the two, so neither can + reinstate what the other withdrew (`a1b7e3f9`). + +- **Resetting a password takes back the sessions that existed before it** — a JWT is + stateless and nothing could withdraw one, so every session from before a reset kept + working, including whoever's session prompted the reset. Tokens now carry when they were + minted, and one minted before an account's password moved is refused. This withdraws + tokens; it does not limit how many places somebody is signed in, and signing in again + works normally on as many devices as you like. The register is in memory rather than a + read on every authenticated request -- `check-auth` is otherwise pure signature + verification, and against storage-backed Datomic a per-request lookup on a non-indexed + attribute is not free. It is rebuilt from the database at boot, before the server takes + traffic, because a restart would otherwise reinstate the tokens it was holding, silently + (`f2c8d4e1`). + +- **A changed password now tells the account it happened** — nothing was sent. Six + outbound mails existed and none of them fired on a credential change, so a takeover was + silent until the owner tried to log in, and then all they learned was that they could + not. The notice says when and from what browser, and links the reset PAGE rather than + carrying a working credential (`8e3f5a21`). + +- **An email change tells the address that currently owns the account** — the verification + goes to the NEW address, which is the one place the owner cannot read if it was not + them, so the party losing the account was the only one never told (`8e3f5a21`). + +- **Moving an account to another address takes the password, not just a session** — it + asked for nothing but a session, so a borrowed laptop or a reset link followed and left + open was enough to walk off with an account. Checked before the address is even looked + at, so an unauthenticated caller cannot use it to learn which addresses are taken + (`8e3f5a21`). + +- **scripts/e2e/run.sh rebuilds a stale bundle instead of testing it** — the server is + `lein run` and compiles from source at every boot, while the bundle and stylesheet are + artifacts on disk that nothing rebuilt. Edit a `.cljs`, boot a genuinely fresh server, + and the page runs the OLD code while every signal says it is new; a suite then passes or + fails against code that is not the code you changed. It now names what is newer and + rebuilds the kind of bundle already there, with `E2E_SKIP_BUILD=1` to keep the artifacts + and say so loudly (`8e3f5a21`). + +- **Both places a password is made share the area, not just its parts** — password-pair + and password-meter were shared components; the COMPOSITION of them was not, so each + page wrote out the same pair, the same meter, the same mismatch derivation and the same + reveal handling for itself. That is how the reset form went a whole design pass with no + meter at all, and how it came to judge a password against NO username while the + register page judged it against one — so the page called a password fine and the server + then refused it for being the username. `password-fields` composes the area once, and + `registration/password-pair-faults` derives the faults once, for both. The reset page + reads the username from the session token the server set when it accepted the emailed + key: read, not trusted, since the server applies the same rule again from the token it + verifies (`f4a2d1c8`). + +- **What the reset page's "not your name" check does and does not know** — it judges + against the username only, read from the session token. The server judges against the + username and the email's local part; matching that on the page would mean putting the + address into a readable cookie, which discloses more than it protects, so the gap stays + and the server's refusal explains itself on submit. The chip fires only on the whole + username and never a prefix, so it confirms a guess rather than revealing one — and + reaching the page needs the emailed key, whose cookie already carries the username in + plain base64 (`jwt/sign` is JWS, signed and not encrypted) (`c3ddbb65`). + +- **All nine auth pages share a body, not just a shell** — `auth-page` abstracted the + card and the heading and took everything below the amber rule as an opaque blob, so + four pages hand-assembled the identical container skeleton, one of them nested it + wrongly and gave its own submit a double gutter, and the login page used a container of + its own that made its fields 350px where every other page's are the column's width. + That is why a gutter, a submit width, a line-height and a message slot each had to be + fixed once PER PAGE. `auth-form-page` takes a heading, a lede, fields and a tail; the + nine pages now come in two shapes and neither assembles its own containers + (`e8f6b2a1`). + +- **The pages that only announce an outcome get a layout of their own** — registration + complete, password changed, unsubscribed and check-your-email had none. The heading was + centred and everything under it was not, so the sentence sat against the card's left + edge with no gutter and the one link fell onto its own line beneath it. They now centre + in the space the card actually has, and the one thing to do looks like a button rather + than a link among nothing (`9657d6db`). + +- **The login page's other ways in are one group with one rhythm** — three separate + blocks, two of them holding a pair of `
` that pushed each answer three lines from + its question, and a third shaped differently again. A question and the thing that + answers it now share a line, the way the register page already asks its one, with the + help line set below a rule because it is not another way in. The LOGIN button is full + width and aligned to the fields, where at 174px and hard left it read as one option + among the links under it (`9657d6db`). + +- **The last three auth pages join the shared heading** — `verify-failed`, + `send-password-reset-page` and `password-reset-page` were still drawing their own + bold heading inside the old shell, with no amber rule and no form gutter, so three + of the nine pages shipped visibly unredesigned. They were the three somebody locked + out of their account actually sees. The two reset pages also both read as "reset + password"; the one that mails a link is "Reset your password" and the one that takes + the new password is "Choose a new password". + +- **Password reset told nobody why it would not go** — an ordinary password left SUBMIT + dimmed and the page silent. The rules gated the form while their reasons were suppressed: + `:messages password-messages` had been commented out since the dual-build era, which was + survivable at a minimum of eight and is not at twelve. The messages show, and the button + is never dimmed (`0380fc94`). + +- **The server's reason never left the server** — the reset endpoint answered + `{:status 400 :message "..."}`, but `:message` is not a Pedestal response key, so the + reply was a 400 with an empty body and the page showed a generic apology. Field-keyed + messages in `:body` now, the shape registration already used (`e147938e`). + +- **Two verdicts that disagreed** — a password the breach corpus refused was drawn as a red + field error directly above a meter reporting UNCOMMON, in green, about the same string. + The corpus verdict is the meter's: its own key, the fail colours, a Too common badge and + the reasoning on a line under the bar (`b30876ee`). + +- **The auth card's text was set solid** — the CSS reset sets `body{line-height:1}`, and a + unitless line-height is inherited as a NUMBER, so every element that did not set its own + rendered at its own font size and descenders ran into the next line. Eleven elements + measured under a 1.25 ratio, nine at exactly 1.00 (`b30876ee`). + +- **The gryphon panel tiled, and the legal links printed twice** — no `background-repeat` + and no `background-size`, so once the form column outgrew the image a second cropped + gryphon drew below it; and the consent line and the footer each linked the same two + documents (`ecb13193`). + +- **The email help link was one word mid-sentence** — "whitelist" as the whole clickable + target, in two places, one of which also read "Didn't receive validation the email?" and + advised resetting a password to fix a missing validation email (`ecb13193`). + +## Changed + +- **A character's sharing is one line under its title** — a status (Not shared, Shared, or Link expired + and the date) with its actions as text buttons: Share link until a link exists, then Copy link, New + link and Stop sharing. It replaces the share buttons in the page header, which split into extra rows + on a phone (`f1852203`, `5e417137`). +- **The character list row shares with one Copy link button** — it copies the link, or makes the share + and copies it when the character has none; the status, New link and Stop sharing stay on the + character page (`33628762`). +- **Share wording** — the startup log's share settings speak of share data, the Share link button no + longer describes server storage, and What's New describes short links, New link and Stop sharing + (`e3aa6d07`, `567f4bdb`). +- **A party forgets a character's link when its share ends** — New link, Stop sharing, expiry and + deleting the character delete the token the party saved; the character stays in the party + (`3cacef7f`). +- **The share settings are documented** in `docs/ENVIRONMENT.md` (`0940636e`). +- **Browser probes share one way to find Chromium** — they find Playwright's + current install without setting four variables by hand, and the runner reports + a missing browser once as a skip (`b48d312c`). +- **The e2e suites are named, and run.sh runs them on any machine** — the PDF check is now + export-character-pdf.js; run.sh lists the suites, honours E2E_PORT, finds Chromium the way the + probes do, and says when it turns CSP off for a development bundle; the PDF check hides the What's + New panel that had been taking its clicks (`eb8171d8`). +- **A browser check that a character shows its owner's items to others** — it opens a seeded character + logged out, as another account and as the owner, and checks the item reaches the sheet each time; + run.sh now waits for the test accounts to be seeded before a suite starts (`359095b8`). +- **Named Garden classes replace 28 inline style maps on the registration, login and + password pages** — 49 class definitions quarried off the stalled + `refactor/garden-inline-styles` branch without merging it, which is 648 commits behind + and conflicts in two files. Two of the classes it used are generated from value lists + rather than written out, so they emitted no CSS at all and are generated here. The + password meter keeps an inline width, because that width IS the measurement. Nothing + renders differently (`47bb73f4`, `2c9c553d`). +- **The nine auth pages share one heading** — `registration-page` was always the shared shell, + but six pages carried their own copy of the same orange drop-shadowed heading. The shell owns + it now, as ink with an amber rule, so it changes in one place (`3b1ac947`). + +- **The auth fields are notched outlines with a real label** — the label rides the input's + border rather than sitting in a placeholder that vanishes the moment somebody types, so the + field never stops saying what it is (`c9c6732b`). + +- **Errors read in GOV.UK order** — the label lifts out of the notch, the message sits under it + and the input follows, with a rail down the group, a 1px border and a soft tint instead of a + heavy outline plus a boxed message. A summary above the form counts fields rather than + messages, and each line focuses its field (`70b6a410`, `935c996f`). + +- **A password can be revealed, and the confirm box retires when it is** — two boxes while it + is masked, one while it is not, and `display: none` rather than dimmed so a submit cannot + fail pointing at a field nobody can see (`141894fd`). + +- **The email is confirmed, not the password, and a mistyped domain is offered a fix** — a + mistyped password is recoverable; a mistyped address makes a dead account holding the username + its owner wanted and mails a stranger on the way. Within two edits of a known domain, a + correction is offered under the field (`bd434342`). +- **The last three auth pages join the shared heading** — `verify-failed`, + `send-password-reset-page` and `password-reset-page` still drew their own bold heading + with no rule and no form gutter, so three of the nine shipped unredesigned, and they are + the three somebody locked out of their account sees. The two reset pages would both have + read "reset password"; the one that mails a link is "Reset your password" and the one + that takes the new password is "Choose a new password" (`d476dbd7`). + +- **Every password rule is visible as its own chip, and the meter says what to do next** + — five chips under the bar, one per rule, each showing whether this password satisfies + it before anything is pressed. The rules had only ever spoken through whichever one + happened to fail first, so nobody could see what was being asked of them until they had + broken it. Below the bar, a line that points forward at every rung except the top: a + password that is already good enough is the one moment somebody is looking at this field + and not being told off, and telling them to stop there is not what a ladder is for. Both + were designed with the meter and neither had been built. + +- **The reset form gets the meter and the reveal** — it refused by exactly the same rules + as registration while showing no meter, offering no reveal and demanding a confirmation + it never retired. The meter was a block inside `register-form` reading the registration + form directly; it is a component now, used by both (`0fbfab83`). + +- **A password is refused for the corpus only when it is egregiously common** — 1000 + appearances, not one. The count was already returned and both call sites flattened it to + a boolean. Reset also judges the password against the username, which registration did + and it did not (`cef432af`, `e147938e`). + +- **Five dead rules removed** — the four `password-strength-*` from the meter this replaced + and `success-header`, all with zero uses outside garden (`b30876ee`). diff --git a/docs/branch-changelog.template.md b/docs/branch-changelog.template.md new file mode 100644 index 000000000..12791b9ac --- /dev/null +++ b/docs/branch-changelog.template.md @@ -0,0 +1,43 @@ + + +# Branch changelog — `` + + + +## Why this branch exists + + + +## Highlights + + + +## Added + +## Fixed + +## Changed diff --git a/docs/issues/pact-magic-slots-overwrite.md b/docs/issues/pact-magic-slots-overwrite.md new file mode 100644 index 000000000..38e3184dd --- /dev/null +++ b/docs/issues/pact-magic-slots-overwrite.md @@ -0,0 +1,46 @@ +# Pact Magic slots overwrite the Spellcasting slots instead of joining them + +**Status (2026-09, `feature/one-template-per-style`):** the calculation is +fixed. `template_base.cljc` keeps `?shared-spell-slots` and `?pact-spell-slots` +apart and `?spell-slots` is their sum, so no slot is lost. The packed spell sheet +prints the two pools separately — a Warlock's column carries its own slot +counts. The web sheet and the page-per-class export still show one summed number +per level; showing two there is the display decision this note describes. + +`template_base.cljc` builds `?spell-slots` as + + (merge + (when ?pact-magic? (warlock-spell-slot-schedule (?class-level :warlock)))) + +`merge` replaces. Pact Magic and Spellcasting are separate pools in 5e -- PHB +p.164 says a character with both "can use the spell slots you gain from the Pact +Magic feature to cast spells you know or have prepared from classes with the +Spellcasting feature, and vice versa", which only means anything if both exist. +Replacing one with the other loses slots the character has. + +Measured, with `opt5e/total-slots` and `t-base/warlock-spell-slot-schedule`: + + Wizard 5 / Warlock 3 spellcasting {1 4, 2 3, 3 2} pact {2 2} + shown {1 4, 2 2, 3 2} level 2: 2 shown, 3 + 2 pact + + Wizard 9 / Warlock 5 spellcasting {1 4, 2 3, 3 3, 4 3, 5 1} pact {3 2} + shown {1 4, 2 3, 3 2, 4 3, 5 1} level 3: 2 shown, 3 + 2 pact + + Cleric 3 / Warlock 2 spellcasting {1 4, 2 2} pact {1 2} + shown {1 2, 2 2} level 1: 2 shown, 4 + 2 pact + +The last is the worst shape: a first-level slot count of 2 for a character with +six. It is worse the lower the pact level, because the Warlock's schedule holds +few slots at a low level while a full caster's table holds many. + +This is not a PDF problem. `?spell-slots` is a built-character property, so the +web sheet and the export both read the same wrong numbers. + +Not fixed here because the fix is a display decision rather than a calculation: +the two pools have to be shown as two things. The sheet has one SLOTS TOTAL box +per level and no second place to put a pact count, so this wants deciding +alongside the spell page layout work in `pdf-overflow-and-page-generation-plan.md`. + +What is NOT wrong: a Warlock has no `:level-factor`, so its levels correctly do +not feed `?total-spellcaster-levels`. The multiclass Spellcasting table itself is +right. diff --git a/docs/issues/pdf-export-size.md b/docs/issues/pdf-export-size.md new file mode 100644 index 000000000..64ef4a157 --- /dev/null +++ b/docs/issues/pdf-export-size.md @@ -0,0 +1,565 @@ +# PDF export size — measurements + +Investigated while hardening `/character.pdf`. No code changed; this records +what was measured and, importantly, one conclusion that was wrong before it was +corrected. + +## Where the bytes are, per style + +The four sheet styles are unrelated documents with unrelated size problems. + +| style | size (0-spells) | pages | images | what dominates | +|-------|-----------------|-------|--------|----------------| +| 1 (default) | 1,236 KB | 3 | none | 1,407 AcroForm fields | +| 2 | 241 KB | 3 | 17 small PNGs | already lean | +| 3 | 1,659 KB | 2 | 2 × 1275×1650 PNG | full-page raster | +| 4 | 2,610 KB | 2 | 2 × 2550×3300 JPG | full-page raster, 300 DPI | + +Styles 3 and 4 are scans: 2550×3300 is letter size at 300 DPI, with form fields +laid over the image. Their size is inherent to print quality and field pruning +does nothing for them. Reducing DPI is the only lever and it costs exactly the +thing users care about. + +Style 1 is pure vector with no images at all. Its weight is form fields. + +## The style-1 form is one form, reused + +Every one of the 28 templates carries the SAME 1,407 fields. The variants +differ only in how many spell pages are physically bound in: + + template pages fields anon-boxes orphaned widgets + 0-spells 3 1407 466 926 + 1-spells 4 1407 466 834 (-92) + ... ... + 6-spells 9 1407 466 374 (-92) + +Exactly 92 checkbox widgets attach per spell page present. The form has capacity +for roughly ten spell pages (926 / 92); the largest template uses six. So every +file carries fields for pages it does not contain — a non-caster sheet ships +with a complete spellcasting apparatus that has nowhere to render. + +## Pruning fields whose widgets have no page + +Rule: drop a field only when ALL of its widgets have no page reference. This is +structural, not name-based, so it cannot catch a functional field. + + style 1, 0-spells 1,236 KB -> 313 KB (74.7%) fields 1407 -> 221 + style 1, 1-spells 1,264 KB -> 511 KB (59.5%) + style 1, 3-spells 1,340 KB -> 791 KB (41.0%) + style 1, 6-spells 1,410 KB -> 1,472 KB (-4.4%) + ALL 28 TEMPLATES 59.4 MB -> 55.6 MB (6.4%) + +The headline number is 6.4%, not 75%. Styles 2-4 have no orphans at all and get +marginally LARGER from the re-save. The win is confined to style 1, and shrinks +as spell pages are added. + +It is still worth considering, because style 1 is the default and most +characters have zero or one spellcasting class — the two cases that save 75% and +60%. But "6.4% across the assets" is the honest figure for the work involved. + +## A wrong conclusion, and why the test missed it + +The first attempt pruned every field named "Check Box N" — 466 of them — and +measured 1,236 KB -> 440 KB with zero differing pixels across the rendered +pages. That was reported as safe. It was not. + +Six of those 466 are the DEATH SAVE circles on page 1. Pruning them leaves +circles that can never be filled. + +Two compounding errors hid it: + + * The comparison rendered an EMPTY form. An unticked checkbox draws nothing, + so it compared "absent" against "empty" and found them identical. Every + visible circle on the sheet is drawn in the page content stream; the widget + contributes only when ticked. + * It rendered pages 1-2 of a 3-page document. + +The correct test ticks every checkbox and renders every page. Under it, the +name-based prune differs by 299 pixels on page 1 (the death saves) and the +structural orphan-only prune differs by zero on all three pages. + +Any future work here must tick the boxes and render every page before claiming +a prune is safe. + + +## How the orphans got there, and the better fix + +Widget counts across the style-1 family, distinguishing "missing the /P +back-pointer" from "not in any page's annotation list at all": + + template widgets no /P on-a-page TRUE orphans + 0-spells 1931 1810 121 1810 + 1-spells 1931 1596 335 1596 + ... ... + 6-spells 1931 526 1405 526 + +The two columns are identical, so every widget missing /P really is absent from +every page. Exactly 214 widgets attach per spell page present -- the page's full +complement of spell name fields, prepared checkboxes and slot fields. + +That linear pattern is the signature of how the variants were made: one 9-page +master form, saved repeatedly with pages deleted. Deleting a page removes the +page and its annotations but leaves the fields in the AcroForm, orphaning 214 +widgets each time. Nothing is "riding on one page" -- they are on no page. + +pdf/fix-widget-page-refs! is a related but different workaround. It walks +pages -> annotations and sets /P on widgets that ARE on a page but lack the +back-pointer. It cannot see these, because they are not in any page's +annotation list. Anyone reading that function should not assume it covers this. + +### Prune at export, not in the assets + +The templates are static, but the exported PDF is built at runtime by +write-fields!. Dropping fields whose widgets are all page-less THERE gets every +user a smaller download with no asset migration, no risk to the source files, +and no 28-file regeneration to verify. It is a code change in one function +rather than an art pipeline. + +The asset-side fix -- rebuilding variants by page EXTRACTION rather than page +deletion -- is the real cure, but it is a much bigger job and the runtime prune +makes it optional rather than urgent. + +## Which styles are actually in colour + +Measured per-pixel chroma (max channel minus min) on a rendered page 1 of each: + + style coloured px max chroma + 1 0.00% 3 + 2 0.00% 3 + 3 8.10% 192 + 4 0.00% 4 + +Max chroma of 3-4 out of 255 is render noise. Styles 1, 2 and 4 are genuinely +black and white. Style 3 -- the official WotC sheet with the teal/gold ram crest +-- is the only one with real colour, and only ~8% of its pixels. + +Template sizes: + + style 1 1.21 MB + style 2 0.24 MB + style 3 1.62 MB + style 4 2.55 MB + +### Style 4 is the biggest lever + +Style 4 ("Cthulhu Mythos Sagas") is the largest file and is visually grayscale, +yet pages 1-2 are stored as DeviceRGB JPEG -- three channels carrying identical +values. DeviceGray is lossless by inspection here and should take roughly a +third off. No quality argument to have. + +### Style 3 is the one real colour sheet, stored in CMYK + +Grayscale would destroy style 3 specifically -- but ONLY style 3. An earlier +revision of this document said the same of style 4, which is wrong: style 4 has +no colour at all. + +It is stored as DeviceCMYK at 8 bits per component: four channels, for a +document that is viewed on screen and printed on home printers. Converting the +page-1 image to RGB, losslessly: + + original CMYK 8bpc (Flate) 684 KB + as RGB PNG 335 KB (51% smaller, lossless) + as RGB JPEG q=0.92 243 KB (64%) + as RGB JPEG q=0.85 199 KB (71%) + +RGB PNG halves it with no quality question at all -- it drops a channel the +display path never uses. The JPEG options trade quality for more and should not +be taken without someone looking at a printed sheet. + +Style 4 does not have the CMYK problem, but has the RGB-for-gray one above. +Images ARE shared across repeated pages in +both styles -- one object serves pages 3-8 -- so there is no duplication to fix. + +## Not investigated + + * Whether pruning breaks a user who duplicates a spell page by hand in + Acrobat. Orphaned widgets are not reachable and duplicating a page copies + its own annotations, so it should be fine — but that is reasoning, not a + test. + * Whether styles 3 and 4 could drop to 200 DPI without visible loss in print. + +## Related, already available + +`pdf/write-fields!` already takes a `flatten?` argument and `routes.clj` already +passes it through; nothing in the UI ever sets it. Flattening style 1 gives +1,236 KB -> 252 KB but produces a non-editable PDF, so it belongs as an option +rather than a default — people reasonably want to keep editing their sheet. + +## Correction: this is a style 1 problem, and style 1 is the only public style + +Two facts change the scope of everything above. + +**Only style 1 has orphans.** Checked all four families, fewest and most spell +pages: + + style 1, 0 spell pages 1931 widgets 1810 orphaned 94% + style 1, 6 spell pages 1931 widgets 526 orphaned 27% + style 2, 0 / 6 120 / 1404 0 0% + style 3, 0 / 6 112 / 844 0 0% + style 4, 0 / 6 129 / 807 0 0% + +Styles 2-4 scale their field count with page count, which is what a correctly +built variant looks like. Style 1 carries all 1931 widgets in every variant no +matter how many pages it has. + +**Only style 1 is reachable.** integrations/sheet-styles returns a single entry +("Original 5e Character sheet", value 1). Styles 2-4 are fork overrides gated by +user tier; the assets ship in the repo but the dropdown never offers them. + +So the fix is 7 files, not 28, and the runtime prune in write-fields! only has +to handle style 1 -- where it removes up to 94% of the widgets. The style 3 +CMYK->RGB saving noted above is real but unreachable in the open app, and the +style 4 RGB-for-gray one likewise. Leave both recorded, do neither first. + +Note also that routes.clj accepts valid-sheet-styles #{1 2 3 4} while the UI +offers only 1. A hand-crafted request can select a non-public style. These are +static assets in the repo, so this is not a disclosure issue, but the server +being deliberately broader than the client is worth knowing rather than +rediscovering. + +## Correction again: styles 2-4 are lean because they are INCOMPLETE + +The previous section called styles 2-4 "correctly built" because their field +count scales with page count. That reasoning was wrong, and backwards in the way +that matters. Comparing field NAMES rather than counts (ignoring the anonymous +"Check Box N" fields): + +- Style 4 has no `backstory` field and no `equipment` field. pdf_spec.cljc emits + both. write-fields! looks fields up by exact name and skips misses with a bare + `(when field ...)` -- no warning, no error. A user on style 4 fills in their + backstory and equipment, exports, and the PDF comes back without them, silently. +- Styles 2 and 3 lack `cantrips-6/7/8` and a set of `spells-N-M` names, so + high-level casters lose spells off the export the same way. + +These are absences, not renames. Style 4's extra field names are things like +Notes, other, Conditions and Insanities, Pronouns, Carry Capacity -- none of +which the code writes to. And a rename the code does not know about behaves +identically to a missing field, since the lookup is by exact name. + +So: style 1 is bloated but COMPLETE. Styles 2-4 are lean because they drop data. +Do not treat them as the model for rebuilding style 1 -- rebuilding style 1 by +page extraction to look like them would mean rebuilding it to lose data. + +This is a live bug for whoever can select those styles (fork/premium tiers). +Worth its own issue: either fill the missing fields into the templates, or have +write-fields! report names it could not place instead of skipping in silence. +The silent skip is what let this sit unnoticed. + +What survives from all of the above is one narrow change: prune widgets that +belong to no page, at export time, in write-fields!. That only ever removes +fields with nowhere to draw, so it cannot drop a value the way these templates +do. Everything else here is a note, not a plan. + +## Generating spell pages on demand (spiked, works) + +Rather than pruning ghost fields, remove the reason they exist. routes.clj picks +one of seven pre-baked variants per style (sheet0..sheet6) by counting +spellcasting classes. Instead: ship a base plus ONE spell-page master, clone the +master as many times as the character needs, and build that page's fields in +code with the names pdf_spec already emits. + +Spiked end to end against the real assets. What holds: + +- A page clones by making a new page dictionary that REFERENCES the source + /Contents, /Resources and /MediaBox. No pixel data copied: the master alone is + 421 KB, master plus six shared clones is 422 KB. +- Fields can be built per cloned page with arbitrary names, filled, and they + render. Verified by filling a level-1 spell slot on a CLONED page and + rendering it. About 12 KB of field structure per extra page. +- Clean masters can be cut from the existing assets using the orphan bug itself: + delete every page but the one you want, then prune the fields left behind. + Yields a 1-page 214-field spell master and a 2-page 118-field base, both with + zero orphans. + +Traps found, all of them load-bearing: + +- Splitter DROPS THE ACROFORM. An isolated page comes out with zero fields. Do + not use it to cut a page out. +- PDFMergerUtility duplicates the background image per merged copy (base plus 3 + spell pages ballooned to 1804 KB, larger than the original 9-page file) AND + renames colliding fields, turning spells-1-1 into spells-1-1-1. Every name + pdf_spec writes would silently stop matching. Do not use it either. +- Orphan pruning MUST work at the widget level, not the field level. Keeping a + field because any one of its widgets is live left 101 orphans, since some + fields own widgets across several pages. Filter each field's widget list and + drop fields left with none. + +Payoff: seven files per style become two, ghost fields stop existing rather than +being pruned after the fact, and the six-spellcasting-class ceiling goes away -- +extra pages cost about 12 KB each. + +Not yet done: wiring this into routes.clj in place of the sheet0..sheet6 cond, +and generating the per-page field names for pages beyond the sixth. + +## Test with a sheet that has data on it + +dev/sample_character.clj builds a level 20 evoker with every spell level filled +from the real wizard list and writes it through pdf/write-fields!: + + lein with-profile init-db run -m clojure.main dev/sample_character.clj + +Rendering an EMPTY form proves almost nothing -- it was rendering empty forms +that hid the death-save checkboxes earlier in this work. Filling one turned up +three bugs immediately, none of them visible on a blank page: + +1. **Page 1 fields sit one box off from the printed labels.** The tall right + column is the field `features-and-traits`, and the artwork beneath it reads + EQUIPMENT. The middle box holds `equipment`, printed as TREASURE. So a + character's features and traits print under a heading that says EQUIPMENT. + This is on style 1, the default and only public sheet. + +2. **`spells-3-11-1` does not exist in style 1.** The level 3 column offers 13 + rows but the template skips the 11th, leaving a dead row that renders as an + empty line with a bullet. Styles 2 and 4 have the field. A wizard with 13 + third-level spells silently loses one. + +3. **Modifier boxes are too small for two digits.** The widgets are 14.4 x 8.6 pt + and anything reaching "+10" clips -- "+11" renders as "+1" plus a fragment. + This is NOT limited to saves: the second fixture (a face-caster) clips + Deception, Persuasion AND the Charisma save on one sheet. Any modifier of +10 + or better is affected, which is routine at high level and unavoidable for a + level 20 caster in their own casting stat. + +Keep using this fixture for template work. Blank-form comparisons are how the +earlier mistakes in this document happened. + +### Second fixture: four spellcasting classes + +target/sample-multi.pdf is warlock 5 / sorcerer 5 / wizard 5 / cleric 5. One +spellcasting class only ever exercises the "-1" field suffix; this one exercises +-1 through -4 and the sheet4 template, which nothing else here touches. + +It reproduces all three bugs above independently, and widens the third from +saves to every modifier. + +What it confirms works: the four-class string fits CLASS & LEVEL by auto-sizing, +hit dice wrap to two lines, warlock pact slots stay separate from shared slots +(3rd level only, 1st and 2nd correctly blank), and long subclass names in the +spellcasting-class header auto-size rather than clip. + +One thing that LOOKS wrong on this sheet but is correct: slots at 4th through +8th level with no spells listed beneath them. Caster level is 15 for slot +purposes, but no single class is above 5th, so nothing above 3rd level can be +prepared. Do not "fix" that. + +### Third fixture: eight spellcasting classes (the real edge case) + +target/sample-eight.pdf is bard 2 / cleric 2 / druid 2 / paladin 4 / ranger 4 / +sorcerer 2 / warlock 2 / wizard 2 -- twenty levels across eight casting classes. + +**The sheet has room for six.** The template provides spellcasting sections -1 +through -6, and routes.clj tops out at sheet6. Classes seven and eight have +nowhere to print, and write-fields! drops names it cannot find in silence, so +they simply are not there. On this character that is 50 values gone: both +classes' names, abilities, save DCs, attack bonuses, every slot and every spell. +A player prints their character and two of their eight classes are missing, with +nothing on the page or in the logs to say why. + +This is bug 4 (the silent skip) and the six-class template ceiling landing +together, and it is the strongest argument for fixing the silent skip first. The +fixture now reports dropped names itself -- build! diffs the field map against +the template before writing -- which is exactly the check write-fields! should +be doing at runtime. + +Worth noting the same reporting immediately caught spells-3-11 on the other two +fixtures without anyone looking for it. + +Two more things this character surfaced: + +- **Page 9 of every six-caster export is blank.** The sheet6 template ends with + an empty FEATURES & TRAITS continuation page -- a label in the corner and + nothing else. It has no filled fields and ships in every multi-caster PDF. +- **Long class strings shrink rather than clip.** "Brd2/Clr2/Drd2/Pal4/Rgr4/ + Sor2/Wlk2/Wiz2" renders visibly smaller than neighbouring fields but intact. + That is the right failure mode, and worth preserving if the modifier boxes get + fixed to auto-size. + +No modifier on this character reaches +10, and nothing clips -- consistent with +the +10 threshold recorded above. + +## Retraction: two of the reported bugs were fixture errors + +Corrected after reading pdf_spec instead of trusting the template's field names. +The names do not describe their contents: + + features-and-traits <- the EQUIPPED item list (prints under EQUIPMENT) + treasure <- unequipped items, valuables (page 2) + features-and-traits-2 <- the real features, actions and reactions, on the + continuation page (prints under FEATURES & TRAITS) + equipment <- nothing. pdf_spec never emits this key; the box it + sits in carries the coin fields only. + +See pdf_spec/equipment-fields and pdf_spec/traits-fields. + +**RETRACTED -- "page 1 fields sit one box off from the printed labels."** They do +not. `features-and-traits` receives the equipment list and prints under +EQUIPMENT, which is correct. The fixture put features text into it and the +render looked mislabelled. The labels were right the whole time; the field NAME +is the misleading part. + +**RETRACTED -- "page 9 of every six-caster export is blank."** It is not. +features-and-traits-2 is emitted by pdf_spec (line 143), given a font size in +routes.clj (line 575), and present on a real page in all seven style 1 +templates. Page 9 was blank because the fixture never set that key. + +Both mistakes have the same cause: reading the field names and inferring +behaviour rather than reading the code that fills them. The fixtures now set +these keys the way the app does, and carry a comment naming the trap. + +Still standing, unaffected by this: spells-3-11 missing, the +10 modifier +clipping, the silent skip in write-fields!, and the six-class ceiling that drops +50 values on an eight-class character. + +One small live oddity, not a bug as such: pdf_spec emits no :equipment key, so +the page 1 text field of that name is never filled. Its box shows the coin +fields only. Probably deliberate; noted so the next person does not "fix" it. + +## Checked against a real export from the live site + +A production PDF (level 20 evocation wizard, 16 pages by PDFBox -- note the +upload banner claimed 27) settles several things measured only in fixtures here. + +### The ghost fields do ship, and they are most of the file + + 1931 widgets, 1596 orphaned -- an exact match for the 1-spells measurement + +And the size is NOT the artwork. Images are 329 KB of a 2679 KB file: one +800x1200 portrait at 275 KB plus 212 tiny 32x32 spell-card icons at 60 KB total. +The other 2.35 MB -- 88 percent of the download -- is the AcroForm. + +Pruning page-less widgets from this exact file: + + before 1407 fields 2679 KB + after 333 fields 1313 KB 51 percent smaller + +Verified safe three ways, not just by eye: zero differing pixels across all 16 +pages at 72 DPI; 248 fields carry a value before AND after with none lost and +none changed; and no surviving field has an off-page widget. The 668 dropped +names were all valueless. Pixel comparison alone would NOT have been enough -- +that is the check that missed the death-save checkboxes earlier in this +document -- so the value diff is the one that matters. + +This makes the runtime prune the single largest size lever, not a tidying job. + +### Confirmed: spells-3-11-1 really is absent in production + +Reading the live file's fields directly: spells-3-1-1 through spells-3-10-1 and +spells-3-12-1 onward exist; spells-3-11-1 returns no field. This character has +only five third-level spells so it does not bite her, but the gap is real in the +wild, not a quirk of the local assets. + +### Confirmed: the field-name mapping, and the ability boxes are inverted too + +features-and-traits on the live sheet contains "Backpack / Bag of Sand / Book / +Clothes, common / Elven Chain / Incense x5..." -- the equipment list, exactly as +the retraction above says. features-and-traits-2 holds the real traits. + +New from this file: the ability boxes are inverted from their names as well. +`cha` holds the MODIFIER (the large box) and `cha-mod` holds the SCORE (the small +circle). The fixtures had these backwards and are now corrected. + +### The hyphen encoding: neither a non-issue nor "every negative is a box" + +This section went wrong twice -- first claiming every negative renders as tofu, +then over-correcting to "my renderer's fault, nothing to fix". Both were wrong. +The calibrated version: + +The stored value is plain ASCII (U+002D U+0031). The baked appearance stream +encodes the hyphen as octal 255 (0xAD) where this branch's output encodes 0x2D. +WinAnsiEncoding lists "hyphen" at BOTH codes and PDFBox's name-to-code reverse +lookup returns 0xAD, so production is technically within spec. + +But 0xAD is the SOFT HYPHEN slot, and treating it as an invisible break +opportunity is reasonable behaviour. Two consumers demonstrably break on it: + +1. **PDFBox's renderer does not draw it.** Every screenshot in this document is + produced with that renderer, which is how the false alarm started -- but a + renderer declining to draw it is still a renderer declining to draw it. +2. **Flattened text extraction loses the sign.** Flatten the production PDF and + pull the text out and the stat block reads: + + +1 +2 +1 +7 +6 1 +8 +6 +3 1 +1 0 + + Eight soft hyphens where minus signs belong. Anyone copying a character into + another tool, or reading it with a screen reader, gets a number that has lost + its sign. Branch output: zero. + +A real PDF viewer DOES draw it (confirmed by screenshot), with slightly wider +spacing than a normal hyphen -- "- 1" against "+8". So the user-visible damage in +a normal viewing path is cosmetic at most, and the alarm was unjustified. + +Conclusion: 0x2D costs nothing, breaks neither consumer, and removes a class of +renderer disagreement. This branch already emits it; production does not. Worth +fixing there, framed as compatibility rather than as a visible defect. + +One thing checked and found NOT to be evidence either way: unflattened text +extraction. PDFTextStripper does not read widget appearance streams, so form +values do not appear in extracted text at all until the form is flattened. + +### The clipping bug, re-verified without a renderer + +Given the retraction above, the "+10 clips" claim was re-checked by measurement +rather than by looking at pixels. The appearance stream carries its own clip +rectangle (an "re W n" operator), so this is what EVERY viewer draws, not a +PDFBox artifact: + + clip rectangle width 12.4 pt + "+7" / "-1" at 8 pt Helvetica 9.1 pt fits + "+11" at 8 pt Helvetica 13.6 pt CLIPS + +So the bug is real, and the threshold is not "+10" as such -- it is any +THREE-CHARACTER modifier, which means +10 and above or -10 and below. Confirmed +on int-save, arcana, history and investigation in the fixture. + +Every field on the production sheet fits because none of Nedda's modifiers +exceeds +9. The bug needs a high-level character to show up, which is exactly +why the fixtures exist. + +(Her modifiers top out at +7, so the production file neither confirms nor +refutes it on its own; the measurement above is what settles it.) + +## Answered: yes, pages can be generated on demand with unique field names + +The scheme was first proven by a spike, since removed and superseded by +`pdf/add-spell-pages!` and `pdf/grow-spell-sections!`. It built +a 2-page base plus EIGHT spellcasting class pages from a single master page: + + 10 pages, 1830 fields, ZERO duplicate names, 757 KB + +against today's six-class template at 9 pages, 1407 fields, 1596 orphans and +about 1.2 MB. More classes, more pages, fewer bytes, no ghosts. Class 8 -- which +cannot exist on the current sheet at all -- renders correctly with its own +fields. + +Why unique naming is the crux, and not a detail: in PDF, fields sharing a fully +qualified name ARE THE SAME FIELD and share one value. Tick a checkbox on one +page and its same-named twin ticks on another. Any generated page therefore has +to mint its own names, which is what the rename step does and why the output has +no duplicates. + +That constraint also explains where the orphans came from. The template carries +hundreds of uniquely-numbered "Check Box NNNN" fields; someone needed +per-page-unique names for overflow pages, pre-made them, then produced the +variants by DELETING pages -- and deleting a page leaves its fields in the +AcroForm. The ghosts are the residue of solving exactly this problem by hand. + +The same mechanism covers every overflow case raised: + +- more spellcasting classes than the sheet has sections (proven above) +- more spell pages as a caster levels up and outgrows the rows +- features-and-traits-3, -4 ... when traits outgrow the single continuation page + +Remaining work before it could ship: decide where generation sits relative to +the sheet0..sheet6 selection in routes.clj, and teach pdf_spec to emit names +past the sixth class. + +## Correction: the 0xAD hyphen is an upstream PDFBox bug, since fixed + +An earlier note here guessed the cause was a last-write-wins name-to-code table +where 0xAD (173) overwrote 0x2D (45). Tested, and wrong: this PDFBox version maps +"hyphen" to code 45 only -- there is no 173 entry to win. + +The live file's 0xAD therefore comes from an OLDER PDFBox whose WinAnsiEncoding +still carried the PDF spec's duplicate hyphen at 0xAD. It is an upstream bug that +has since been fixed, so this branch gets 0x2D from the version bump rather than +from anything in our code. Do not credit the branch with fixing it. diff --git a/docs/issues/pdf-overflow-and-page-generation-plan.md b/docs/issues/pdf-overflow-and-page-generation-plan.md new file mode 100644 index 000000000..af1fa6e7e --- /dev/null +++ b/docs/issues/pdf-overflow-and-page-generation-plan.md @@ -0,0 +1,800 @@ +# Plan: generate pages on demand, and stop cropping text silently + +Findings and false starts are in `docs/issues/pdf-export-size.md`. This is the +build plan. + +## What is wrong today + +1. Seven pre-baked variants per style, chosen by counting spellcasting classes. + Each carries every field for six classes, so ~1600 of ~1900 widgets on a real + export point at no page. That is 88% of a 2.6 MB download. +2. Six spellcasting classes maximum. A seventh or eighth is dropped entirely — + 50 values on the eight-class fixture, with no error. +3. `write-fields!` skips names the template lacks, in silence. That is how 2 and + 4 went unnoticed for years, and how styles 2-4 shipped without `backstory`. +4. Text longer than its box is silently shrunk toward a 4pt floor and only then + cropped. A few sentences in "Bonds" already drops below 7pt. Nothing warns. +5. Any modifier of three characters (+10 or worse than -9) is cropped. + +## Order of work + +Each step is independently shippable and independently verifiable. + +### 1. `pdf/prune-orphan-widgets!` — biggest win, smallest change + +Drop widgets belonging to no page, then fields left with no widgets. Per-widget, +not per-field. Call from `write-fields!` before writing. + +Halves a real export (2679 KB to 1313 KB). Verify by value diff, not pixels: all +248 valued fields present and unchanged. Ships on its own with no other change. + +### 2. `write-fields!` reports names it could not place + +Return or log the field names with no home instead of skipping silently. +`dev/sample_character.clj` already does exactly this diff in six lines and it +immediately found `spells-3-11`. Without it, every later step fails quietly. + +Not a warning users see — a log line and a return value tests can assert on. + +### 3. `pdf/fit-text` — the SSOT for "does this fit" + + (fit-text s {:width w :height h :font-size fs}) => {:head "..." :tail "..." :lines n} + +Greedy word wrap at the widget's real width, count lines against height. Used by +steps 4 and 5 so overflow decisions are made in one place, and by tests to assert +capacity without rendering anything. + +Must measure the ON-PAGE widget. Every field has two; the first is the orphan. + +### 4. Overflow driven by a minimum font size + +The fields AUTO-SIZE (see the KB). They do not crop at a fixed word count; they +shrink to a 4pt floor and only then start losing text. So the cutoff is a +minimum legible size, not a character limit. + + (fit-text s widget {:min-font-size 7}) + => {:head "what fits at >=7pt" :tail "the rest" :font-size 7.0} + +Budgets at 7pt: ideals/bonds/flaws 25 words, personality-traits 44, +attacks-and-spellcasting 127, other-profs 147, equipment 447, backstory 987, +features-and-traits-2 3369. + +Small page 1 boxes cannot grow -- the artwork is fixed -- so their `tail` spills +to a continuation page under a heading naming the field, with a marker in the box +pointing at it. Large boxes spill to another continuation page of their own. + +Open question for review: what floor? 7pt is comfortable in print, 6pt is small +but readable and buys roughly 60% more before spilling. 7pt is the safer default +and the one assumed here. + +### 5. Continuation pages that grow + +`features-and-traits-2` holds ~3369 words at 7pt and a real level 20 wizard used +about a tenth of that, so this is not urgent — but it is the same mechanism as step 6 and costs +little once that exists. Generate `features-and-traits-3`, `-4` as `fit-text` +reports a remainder. + +### 6. Spell pages per class, unbounded + +Replace the `sheet0..sheet6` cond in `routes.clj` with: load a base, add one +spell page per spellcasting class, each with uniquely-named fields. + +Proven end to end by a spike, now removed and superseded by +`pdf/add-spell-pages!` — eight classes, zero duplicate +names, 757 KB, smaller than today's six-class file. Also removes the ceiling in +step 2 of the problem list and makes the pre-baked variants redundant. + +Requires `pdf_spec` to emit names past the sixth class; it stops there today +because nothing could receive them. + +### 7. Modifier clipping + +Separate from the rest. Either widen the widgets in the templates, or let those +fields auto-size like the class-level field already does. The auto-size path is +code-only and matches behaviour the sheet already has elsewhere. + +## What must not regress + +- No value may be dropped. Verify by diffing field values before and after, not + by comparing renders. +- No duplicate field names, ever. A duplicate silently ties two pages' values + together. Assert zero duplicates in a test. +- Existing exports must keep working: the same `pdf_spec` field names, same + fillable behaviour, same flatten option. + +## Testing + +- Unit: `fit-text` against known box sizes; prune preserves all valued fields; + generated pages produce no duplicate names. +- Fixtures: the three characters in `dev/sample_character.clj` (level 20 single + class, four classes, eight classes) each build with zero dropped names. +- A long-text fixture that overflows every small box, to prove spill works. +- Compare against the capacities in `docs/PDF-EXPORT-CAPACITY.md` so drift is caught. + +--- + +## Done, and what is left + +Items 1 through 7 shipped on `fix/pdf-endpoint-hardening`, plus two that were not +in the original plan: a field carrying widgets on two pages was found to mirror +one class's ticks and expended slots onto another's page, and a spell level box +can be relabelled so a spare one — the cantrips box included — can carry a +different level. + +### 8. Spell level packing — designed, not built + +The mechanism works: `reuse-cantrips-box!` and `relabel-spell-level!` will put +any level in any box, with the numeral, the slot labels and two slot inputs. +What decides *which* level goes in which box does not exist. `pdf_spec` still +emits one level per box, so a character with three level 1 spells and two level 2s +spends two pages on five spells. + +Packing raises questions about what a spell list on a printed sheet is for, and +they want answering before any of it is built: + +- **Which spells belong on the sheet at all?** Only the ones the player chose, or + also the ones granted for free — by class, subclass, background, race or + subrace? A Cleric already gets its whole domain list; a Sorcerer does not. +- **If granted spells are included, does the sheet say where each came from?** + Measured, and row width is not the obstacle it looks like. A spell row is + 157.95 x 9.94pt and the text auto-sizes to the HEIGHT, landing at 6.42pt, where + the longest name in the SRD -- Protection from Evil and Good -- uses 85.6pt, + just over half the row. Appending "(Life Domain)" reaches 125.9pt and still + fits: none of the 319 spell names overflow with a realistic source suffix. + Only something like "(Circle of the Land: Underdark)" does, at 175.5pt. + + So a parenthesised source is close to free, and the question is whether it + reads well rather than whether it fits. `target/mock-source.png` shows the + cases side by side at real size. +- **Can two classes share a page?** Two small lists — a Paladin's and a Ranger's + at low level, say — could sit in one sheet's boxes rather than taking one page + each. That needs a per-box class label rather than a per-page one, since the + page heading currently names a single class. + + The room for that label already exists: the band above every level bar is + empty except level 1's, which carries the printed SLOTS TOTAL / SLOTS EXPENDED + legend and still has clear space to its right. A class name set there in the + legend's own grey costs no rows and no width. Combined with + `relabel-spell-level!`, which renumbers a box to the level it actually holds, + a Cleric and a Paladin fit on one sheet with each box saying whose it is and + what level it is. Mocked up in `target/mock-shared.png`. +- **What does a box mean once packed?** Today a box is a level. If two levels + share a page, or two classes share a sheet, the numeral and the slot totals + have to say which class as well as which level, or a player reading the printed + sheet cannot tell whose slots those are. + +- **How much room is there to pack into?** A sheet has 100 spell rows, spread + 8 / 12 / 13 / 13 / 13 / 9 / 9 / 9 / 7 / 7 across levels 0 to 9. Rows, not + width, are the scarce resource, and they are unevenly distributed: a level 9 + box holds 7. A packer that moves a level into another box has to check the + destination is big enough, not just free. + +The last one is the constraint the others hang off: a box carries a level, and a +page carries a class. Packing across classes breaks the second, and that is a +larger change than packing levels within one class. + + +#### What each option is worth, measured + +The eight-class fixture in `dev/sample_character.clj` is the case that wastes the +most paper. Its eight classes hold spells at only 14 class-levels between them, +and a sheet has ten boxes: + + Bard 2 levels 0,1 Paladin 4 level 1 + Cleric 2 levels 0,1 Ranger 4 level 1 + Druid 2 levels 0,1 Sorcerer 2 levels 0,1 + Warlock 2 levels 0,1 Wizard 2 levels 0,1 + + today, a page per class 8 pages + a column per class, 3 columns a sheet 3 pages + a box per class-level, 10 a sheet 2 pages + packed by rows, 100 rows a sheet 2 pages (floor; ignores box fit) + +Correction to the third line: "a column per class" assumed one class per column, +which is not the constraint. What matters to a player is not hunting for their +Warlock list, and that is protected by never splitting a class across columns -- +a column can hold several classes as long as each is a contiguous run of boxes +within one. First-fit under that rule puts the eight-class fixture on **2 pages**: + + page 1 left Bard, Paladin page 2 left Sorcerer + middle Cleric, Ranger middle Warlock + right Druid right Wizard + +So the simple rule reaches the same 2 pages the general packer does, and gets +there without a search: walk the classes, put each in the first column with room. + +The columns are unequal, and both limits bind: + + left boxes 0,1,2 3 boxes, 33 rows + middle boxes 3,4,5 3 boxes, 35 rows + right boxes 6,7,8,9 4 boxes, 32 rows + +A class needing four spell levels only fits the right column. On page 1 above it +is rows that fill the columns, not boxes -- Bard 20 + Paladin 11 leaves 2 of 33. + +That fixture is the worst case for rows because it fills every row of every level +it touches; a real character picks far fewer, so real sheets pack tighter than +this. + +#### Three shapes, mocked up on the real sheet + +- **Classes down columns, never split** (`target/mock-columns.png`). Cleric, + Paladin and Ranger on one sheet, each column headed by its class in the empty + band above the bar, each box renumbered by `relabel-spell-level!` to the level + it actually holds. This is the chosen shape: a player finds a class by looking + down one column rather than flipping pages, which is the cost that actually + matters. A column takes more than one class when they fit. +- **Two classes in one box** (`target/mock-inbox.png`). A rule across the rows, + then the second class, its own save DC, and a box for slots left. Costs one row + of twelve. For the case where a class has one short list and a whole box is + more than it needs. +- **A source beside each granted spell** (`target/mock-source.png`). Free on + width, as measured above. + +#### What mocking them up turned up + +Two details that only appear once it is on the page: + +- The divider row is a heading, not a spell, so **its prepared checkbox has to + go**. Left in place it invites a tick against a class name. +- In a shared column, the boxes a class does not use **still carry their printed + numeral**. A Paladin column with an empty box numbered 4 reads as Paladin level + 4 spells. Unused boxes need blanking or relabelling, not just leaving. + + +#### Where a page should break, and two corrections + +**A section is a spellcasting ability, not a class.** `make-pages` in `pdf_spec` +groups by `:ability` and collects every class sharing it into `ability-classes`, +which is what the page heading lists. There are three casting abilities, so a +character has **at most three spell sections, ever**: + + INT wizard + WIS cleric, druid, ranger + CHA bard, paladin, sorcerer, warlock + +The earlier "eight classes, eight pages" figure was wrong. It came from +`dev/sample_character.clj`, which writes eight suffixes directly and bypasses +`pdf_spec`. Through the real pipeline those eight classes make three sections. +Three sections against three columns is not a coincidence worth designing around, +but it does mean the column work is bounded and small. + +**`pdf_spec/level-max-spells` is already the box row capacity** -- the same +`{0 8, 1 12, 2 13, ...}` measured off the artwork here. The two were derived +independently and agree. + +The current rule splits each level into chunks of its own box and takes the worst +level's chunk count as the page count. It breaks on **one crowded level**, so a +section with a fat level 1 and thin everything else spends whole pages carrying +level 1 leftovers while nine boxes sit empty: + + cleric 20, whole preparable list 105 spells now 2 packed 2 + druid 20, whole preparable list 106 spells now 2 packed 2 + wizard 20, whole list 204 spells now 3 packed 3 + paladin 20 / ranger 20 31 / 37 now 1 packed 1 + + wizard book, 27 at level 1, 4 elsewhere 59 spells now 3 packed 1 + bard, 20 at level 1, nothing over 3 35 spells now 2 packed 1 + the eight-class character, by ability 88 spells now 6 packed 3 + +Where a section's spells genuinely exceed a sheet, the current rule is already +optimal -- a Cleric 20's 105 spells need two pages however they are arranged. It +wastes only on the lopsided shapes, which is where the complaint comes from. + +**The rule to adopt: break when the section's rows exceed the sheet, not when one +level exceeds its box.** Fill boxes in level order; when a level outgrows its own +box, spill into the next free box, relabelled to that level; start a page only +when no box on the page has room. Single class, multiclass and granted spells all +fall out of that one rule, because they are all just rows in an ability's section. + +#### Granted spells are already modelled + +`mod5e/spells-known` takes `[level spell-key spellcasting-ability class & +[min-level qualifier class-key]]`, so a spell granted by a feat, background or +boon already carries: + +- an **ability**, which decides the section it lands in; +- a **class**, used as a source label -- a skill feat grants with `"Arcanist"` -- + which reaches `ability-classes` and so the page heading; +- a **qualifier**, `"at will"` or `"once per long rest"`. + +And `pdf_spec` already prints the qualifier beside the name as `(qualifier)`. +So the row-level marking asked about above partly exists: what a granted spell +does not show on its row is its **source**, which currently only reaches the +heading. Putting the source there too is a change to one `str` in `pdf_spec`, +and the measurement above says the row has the width for it. + + +#### What 5e actually keeps separate, and where the sheet does not + +The grouping should follow the rules rather than the layout, and the rules split +three ways (PHB p.164, Multiclassing): + +- **Spells known and prepared are per class.** "You determine what spells you know + and can prepare for each class individually, as if you were a single-classed + member of that class." A Cleric 5 / Druid 5 prepares WIS mod + 5 cleric spells + from the cleric list, and WIS mod + 5 druid spells from the druid list. Two + lists, two budgets. +- **Spell slots are one pool** shared across every Spellcasting class, from the + combined caster-level table. +- **Pact Magic is a separate pool**, usable on Spellcasting spells and vice versa. + +`pdf_spec` groups a section by `:ability`, which crosses the first of those. Both +a Cleric and a Druid cast off WIS, so they land in one section and their lists +interleave alphabetically. Verified by calling `make-pages` directly: + + section ability wis heading: Cleric, Druid + level 1 + Bless (Cleric, not printed) + Cure Wounds (Cleric, not printed) + Entangle (Druid, not printed) + Faerie Fire (Druid, not printed) + Goodberry (Druid, not printed) + Guiding Bolt (Cleric, not printed) + ... + +The row is built as `spell-name` plus `(qualifier)`; `:class` is carried on every +spell but never reaches the page. A player cannot tell which list a spell came +from, and it decides which preparation budget it counts against. + +Ability is the right grouping for the **header** -- save DC and attack bonus are +per ability, so one header serves both classes. It is the wrong grouping for the +**rows**. So the section can stay as it is, and the fix is at the box: a run of +boxes per class, labelled, which is the column shape already chosen above. That +lines up: three abilities, three columns, and each class's list kept whole. + +Slots are the character's, not the class's -- which contradicts the comment now +in `spell-page-fields` saying "slots belong to the class". They are repeated per +section today, harmless while every section shows the same combined pool, but the +reasoning should be corrected when this is next touched. + +Pact Magic is a real defect and is written up in +`docs/issues/pact-magic-slots-overwrite.md`: the pact schedule is merged over the +Spellcasting table, so a Cleric 3 / Warlock 2 shows 2 first-level slots where the +character has 4 plus 2 pact. + + +#### Flagging concentration, components and casting time on the row + +Everything needed is already on the spell, though not always as its own field: + + :duration "Concentration, up to 1 minute" -> concentration + :ritual true + :casting-time "1 bonus action" / "1 reaction, ..." / "1 action" / "10 minutes" + :components {:verbal true :somatic true :material true + :material-component "diamonds worth 300 gp, which the spell consumes"} + +Concentration has no field of its own; it is the start of `:duration`. A costly +material is a `\d+ gp` in the prose of `:material-component`. + +How much of the list each flag touches, of 319 spells: + + concentration 126 39% + longer casting time 59 18% + costly material (gp) 52 16% + ritual 29 9% + bonus action 14 4% + reaction 4 1% + +Cost, with the annotation appended as `C R BA V S M(300gp)`: **none of the 319 +rows overflow.** The widest is `Instant Summons R 1 minute V S M(1,000gp)` at +136.6pt of the 154 available; typical rows land between 44 and 67pt. Rendered at +print size in `target/mock-annotated.png`. + +Worth doing, but not all of it is worth the same. Ranked by what changes a +decision at the table: + +- **Concentration** is the one to have. It touches 39% of spells and is the only + flag that makes two spells mutually exclusive. +- **A costly material** stops the spell happening at all if it is not in the pack, + and the gp figure is the part worth printing, not the prose. +- **Bonus action and reaction** change what else can be done that turn. Only 5% + of spells between them, so they cost almost nothing to carry. +- **Plain V S M** with no cost is the least useful and the widest, being on + nearly everything. It is the part to make optional if any of it is. + +#### Inline flags do not scan, and the fix is alignment rather than spacing + +Appending the flags to the name -- `Bless C V S M` -- fits, but it does not +work. `C` is the same visual class as the letters beside it: single capital, same +weight, same spacing. Finding a letter among other letters is a serial search, so +the eye has to read every row. Shape, weight, size and position are found in +parallel; alphanumeric identity is not. Equal spacing makes it worse by grouping +`C` with the components rather than separating it from them. + +More whitespace does not fix a serial search. A fixed column does, by turning +"read each row" into one vertical sweep. + +The geometry allows it. At the 6.42pt the rows draw at, spell names run: + + min 8.6pt median 37.1pt 90th 56.0pt 99th 73.8pt max 85.6pt + +so the longest name ends at x 127.9 in a row spanning x 40.32 to 198.27. A +concentration column at x+98 and a components column at x+110 clear the longest +name by 10pt, and the widest component string -- `V S M 25,000gp` at 46.0pt -- +fits the remainder. + +Three variants rendered at print size in `target/mock-scan.png`: inline as +proposed, a bold `C` in a fixed column, and a filled square in a fixed column. +Both aligned versions are found in one sweep; the inline one is not. The bold `C` +is the one to take, because the square needs a legend and reads like another +checkbox next to the prepared column. + +Correcting what was written above: the annotation does NOT have to match the +name's size and colour. That is only true while it lives in the same text field. +Drawn separately -- or written into its own field -- it takes its own size, +weight and grey, which is what lets the components recede to 5.6pt grey while the +name stays 6.42pt black and the concentration mark stays 7pt bold. The mock draws +them, since they are computed values a player never edits. + +Drawing is also what keeps the file from growing back. Measured on the +six-caster template with all six spell pages filled and 594 rows annotated: + + names only, as today 1161.7 KB 1403 fields + annotations drawn 1168.3 KB 1403 fields +6.6 KB +0.6% + annotations as form fields 1550.8 KB 2591 fields +389.1 KB +33% + +11 bytes a row drawn against 671 as fields. This branch took a production export +from 2679 KB to 1313 KB by pruning orphaned widgets; adding a field per column +per row would give a third of that back. + + +### 10. Ship one template per style, not seven + +The 44.3 MB of templates in `resources/` is not 44 MB of artwork. Hashing every +image across all 28 files: + + 436 image objects, 74 distinct + 32.7 MB stored, 13.2 MB of unique pixels + + style 2 396 objects, 62 distinct 4.6x duplication + style 3 20 objects, 3 distinct 6.8x duplication + style 4 20 objects, 9 distinct 2.2x duplication + +The variants were cut from a master by deleting pages, so each of a style's seven +files carries its own copy of the base sheet's artwork. Style 3 ships three +distinct images twenty times. + +Shipping one master per style and generating the rest: + + style 1 2.7 MB -> 0.3 style 3 8.7 MB -> 1.0 + style 2 3.2 MB -> 0.4 style 4 29.7 MB -> 4.2 + + 44.3 MB -> 5.9 MB, 86% less + +The machinery exists and is in production: `add-missing-spell-pages!` already +clones a spell page for every class past the template's last, and cloning is +nearly free because the copy references the master's `/Contents`, `/Resources` +and `/MediaBox` rather than duplicating them -- measured at 421 KB for a master +and 422 KB for the master plus six clones. + +**The layouts were checked before proposing this**, since the four styles are +laid out differently and sharing a master across them would be wrong. Sharing is +per style, and within a style the seven variants are the same sheet: + + page 1 rendered and compared against the 6-spell variant, per style + + style 2 every variant pixel-identical + style 3 every variant pixel-identical + style 1 identical but for 392 px (1-spell) and 44 px (3-spell) + style 4 every variant differs, 0-spell by 45,907 px + +The style 4 and style 1 differences are encoding, not design. Rendering style 4's +0-, 1- and 6-spell page 1 side by side shows the same Cthulhu Mythos sheet in all +three; its backgrounds are DCTDecode, so each variant carries a separately +encoded JPEG of the same art and the differences are speckle along every edge. +Styles 2 and 3 come out identical because their images are lossless Flate, which +re-encodes deterministically. Style 1's 392 pixels sit entirely in one region +around the EQUIPMENT box and, cropped at 300 DPI, the two are indistinguishable +-- scattered along the outline and glyph edges, which is coordinate precision in +the content stream. + +**Page 1 is the wrong page to check this on**, and checking only it was the first +mistake. The variants differ by how many SPELL pages they carry, so the spell +pages are what a cloning scheme would copy. Comparing every spell page of the +6-spell variant against its first spell page: + + style 1 page 4 differs by 1396 px, the rest identical + style 4 page 8 differs by 628 px, the rest identical + +Style 1's is precision noise -- the two pages are the same layout down to the +numerals and the Wizards footer, with the differences scattered across the whole +page rather than gathered anywhere. + +Style 4's is not. Its last spell page carries a licence line the others do not: + + dungeonmastersvault.com by permission - Petersen Games LLC 2021 + +Where that line sits differs by style, and it is the real constraint on +generating pages: + + style 1, 2 on every page except the last + style 3 absent entirely + style 4 on the LAST page only -- page 2 of the 0-spell, page 8 of the 6-spell + +So for style 4 the structure is not "N identical spell pages". It is N-1 plain +spell pages plus one carrying an attribution. Cloning its first spell page to +make the rest would drop that line from the document altogether, and cloning the +1-spell master -- whose only spell page is also its last, and so carries the line +-- would stamp it on every page instead. Neither is acceptable for a licence +attribution. + +A master per style is still sound, but generation has to place exactly one +attribution, at the end, per that style's convention. + +The attribution does not have to be redrawn. The attributed page already exists +in the template, so a master keeps two spell pages -- a plain one to clone and +the real attributed one to finish with -- and the licence line stays the original +artwork rather than something reproduced. + +Proved on style 4, the hardest case. A four-page master (character, background, +plain spell page, attributed spell page), then four clones of the plain page +inserted before the attributed one: + + 8 pages, attribution on page 8 only, as the original + artwork identical on all 8 pages, 0 differing pixels at 100 DPI + +The clone copies `/Contents`, `/Resources`, `/MediaBox` and `/Rotate` and nothing +else, so a cloned page arrives without form widgets and renders 6836 px lighter +until they are added -- the prepared checkboxes are what those pixels are. +Creating the widgets is `add-spell-page!`'s existing job, not a new problem. + +**The saving is not in making the master smaller.** A four-page master is 4891.6 +KB against the eight-page template's 4958.6 -- only 67 KB less, because the pages +of one template already share their images, so dropping four spell pages drops no +image. The saving is shipping one file per style instead of seven: + + style 4 seven files 29.7 MB one master 4.9 MB + +Not done here, because it is a change to what ships and to how `routes.clj` +chooses a template, not a change inside one. The selection would go from picking +one of seven files by caster count to always loading the one-spell-page master +and cloning up, with a character that casts nothing needing its spell page +removed rather than added. That makes every multi-caster export depend on the +cloning path, which today only carries the cases past six classes. It wants its +own branch and its own before-and-after render comparison per style. + +Worth keeping in proportion: this is deployment size -- the container and the +uberjar. A player downloads one sheet, and those are 250 to 640 KB. + + +#### Which file to use as each style's master + +Not the narrowest file, but the smallest one containing every distinct *page kind* +that style needs. Style 4 marks only its last page, so its 1-spell file's single +spell page is the marked one and cloning it repeats the mark. Its 2-spell file +carries both kinds: + + style 4 master = the 2-spell file, 4454.7 KB + page 1 character page 2 background page 3 spell page 4 spell + attribution + + grown to six spell sections -> 8 pages, 4454.8 KB, attribution on page 8 only + the shipped 6-spell file is 4958.6 KB + +So the original convention comes out exactly right, 500 KB smaller than what +ships, with no attribution stamped or reproduced. Styles 1, 2 and 3 need only +their 1-spell file, since every spell page they have is alike. + +#### Attribution placement as a setting + +Worth having, and mostly achievable by choosing which page kind to clone where +rather than by drawing anything: + + :all clone the marked page every style with a footer + :last clone plain, finish marked needs a plain page to clone + :first marked first, then plain same + :every-other alternate the two kinds same + +The limit is that a footer baked into a page's content stream **cannot be +removed**, only added, so a placement other than `:all` needs a PLAIN spell page +to clone -- and as of the one-page-per-style masters, no style ships one. Style 4 +did, in the 2-spell file that was its master until the marked page was proved to +be the plain page plus an appended BT/ET block; that file is retired. + +So every style is `:all` today, and the other placements mean DRAWING the footer +rather than shipping a second page kind. For style 4 that is 4pt CartaMarinaBold +at x 24.9, y 12.1, an embedded subset already present in the document and +reusable rather than re-embedded -- which is cheaper than the 4.5 MB a second +master costs, and is the route to take if the setting is built. + +### 8b. Packing belongs in the browser (2026-09) + +Where the packing decision runs was left open above. It should run client-side, and +not only because it is cheaper. + +The server receives a flat map of field names to values. It does not know what a +spell level is, how many a character knows, or which class granted what -- packing +server-side means giving it a domain layer it does not have. The builder already +computes all of it. + +So the browser decides which level goes in which box, emits the field map to match, +and sends a small instruction list alongside it: + + {:relabel [{:section 1 :box 0 :label "3"} ...]} + +The server applies `relabel-spell-level!` and `reuse-cantrips-box!` per instruction +and needs nothing else. Those instructions are caller-supplied, so section, box and +label all need bounds-checking before use -- the same hole as the sheet style id, +which reached a resource path before it was validated. + +### 8c. Packing is a choice, not one layout (2026-09) + +Because packing runs in the builder it costs nothing to offer several ways to +organise a list, and the right one differs by character. The shapes worth having: + +- **One level per box.** What ships today. Predictable, and wasteful for a + character with three level 1 spells and two level 2s. +- **Packed.** Fill boxes in order regardless of level, relabelling as it goes. + Fewest pages. +- **One page per class.** Today's multiclass behaviour, and what a player wants + when two lists are both long. +- **Classes sharing a page.** Two short lists side by side, each labelled. The + case that started this. + +The default should follow the build rather than being asked for every time: + +- A prepared caster with a large list -- Wizard, Cleric, Druid -- wants prepared + spells, not the whole list. A known caster -- Sorcerer, Warlock, Bard -- has + nothing to hide, so everything. +- Domain, circle and oath spells are always prepared and do not count against the + limit. They belong on the sheet but want distinguishing from chosen spells. +- **Pact Magic must not merge with the others.** A Warlock's slots are a separate + pool at a separate level; packing its spells in with a Wizard's would put them + under the wrong slot count. This is the one hard constraint, not a preference. +- A single-class caster below about level 5 already fits one page, so packing + gains nothing and should not be the default there. A multiclass gains the most. + +So: a default computed from the build, and an override in the PDF options +alongside the existing print choices. + +### 8e. Pact Magic is what packing FIXES, not what blocks it (2026-09) + +An earlier reading of this said packing was blocked on Pact Magic. That was +wrong, and wrong in a way worth recording, because it nearly stopped the feature. + +The claim was that packing merges lists into fewer boxes and so would deepen the +pact problem. It rested on the slot total being per SECTION. It is not: each of +the nine level boxes carries its own `spell-slots-LEVEL-SUFFIX` field. Measured on +the masters -- style 1 has `spell-slots-1-1` through `spell-slots-9-1`, nine +fields, one a box, and style 4 the same. + +So what is actually true: + +- **Today** `make-page-map` groups by `:ability`, so a Warlock and a Sorcerer are + merged into one CHA section, and every box in it is written the character-wide + `(spell-slots level)`. The pact pool is invisible on the sheet already. +- **Packed by class**, a class holds its own contiguous run of boxes in one + column, and each of those boxes has its own slot field. The browser writes THAT + class's slot counts into them. A Warlock in its own column shows its own pact + slots at its own level. + +Packing by class is therefore the mechanism that separates the pools, and the +per-column class heading in 8c is what says which column belongs to whom. No +per-class pact flag is needed for it: the packer already keeps a class whole and +in one column, and a class is what the builder groups by before it ever reaches +an ability. + +The ability grouping is the thing to replace, not to preserve. + +#### A Warlock gets the first column, always (2026-09) + +A 5e Warlock casts every spell at its highest slot level. It therefore needs ONE +level box however high it climbs -- the numeral is relabelled as the character +levels rather than a new box being taken -- plus a cantrips box. + +So the first column is reserved for a pact caster: box 0 for cantrips, box 1 for +the level it casts at, and box 2 only because a level 20 Warlock knows 15 spells +against box 1's 12 rows. The rest of the sheet goes to casters whose slots come +from the shared table. + +This is simpler than fitting a Warlock like any other class AND it is what keeps +its slot pool separate: the boxes it holds carry their own `spell-slots` fields. +The spill carries an offset, so the continuation prints the rest of the list +rather than repeating it, and only the box a level starts in carries the total -- +a continuation is the same pool, and printing it twice reads as two sets of slots. + +#### The per-column class heading goes in the cantrips bar (2026-09) + +Resolved, and the earlier note that there was nowhere to put it was looking in +the wrong place. It scanned for a clear band ABOVE each box and found one above +two of the ten, which is true and beside the point. + +The bar of a CANTRIPS box is free. A cantrips box has no slots, so the +compartment a level bar gives to SLOTS TOTAL and SLOTS EXPENDED is dead space +there: on box 0 the bar reads CANTRIPS from x 112, leaving x 62 to 110 empty, and +on a level box reused for cantrips the two slot inputs are empty and meaningless. + +So the class name is drawn there, on the box its column starts with. Box 0 needs +a wider offset than the rest -- a cantrips bar puts its divider right after the +hexagon at x 51-59, where a level bar's sits at 93-102. + +A class with no cantrips at all -- a Paladin, a Ranger -- starts at a level box +whose slot inputs the player writes in, so it gets no heading and the section +header is what names it. + +#### The old note: the heading has nowhere to go (superseded) + +8c wants each column headed with the class it holds, so a reader can tell whose +list is whose. The artwork does not have room for it. + +Scanned at 200 dpi for an ink-free band 1.6in wide and 0.11in tall above each +box's numeral hexagon, on style 1 section 1: + + box 1 clear 0.32in above + box 6 clear 0.22in above + boxes 2, 3, 4, 5, 7, 8, 9 no clear band at any offset up to 0.4in + +So eight of the ten boxes have nothing above them. The wide band beside each +numeral is the SLOTS EXPENDED input, which the player writes in, so a heading +cannot go there either. + +What ships instead: the section header lists the classes in COLUMN ORDER, so +"Warlock, Sorcerer, Paladin, Bard" reads left to right against the columns. That +is weaker than a heading per column -- it does not survive two classes sharing a +column, which the packer allows -- and it is why packed layouts are opt-in rather +than the default. + +Resolving it properly means artwork: a band above each column, or a heading slot +in each box's bar. That is a change to the templates, not to the code. + +### 8f. The row counts had three copies (2026-09) + +`pdf_spec` carried its own `level-max-spells` -- style 1's numbers -- and split a +character's spells by them whatever style was being exported. A style 4 sheet was +handed 8 cantrips for a box with 7 fields and lost one to the unplaceable report, +and 12 first-level spells for a box that holds 13. It now reads +`spell-packing/sheet-geometry`, which a test ties to the templates themselves. + +### 8d. The browser can wrap text exactly, and will need to (2026-09) + +Packing has to answer how many rows a list takes and whether a description fits, +which is the same question the server answers when it wraps text. The browser can +answer it identically, and this was checked rather than assumed: + +- `string-width` is `getStringWidth / 1000 * size`, a plain sum of per-glyph + ADVANCE widths. No kerning, no ligatures, no shaping. +- `split-lines` is a greedy word wrap on whitespace. +- Every description, spell and item name in the data uses **84 distinct + characters**. A table of their advance widths is about 0.5 KB per face. +- Summing that table against PDFBox over 300 real descriptions: **0 mismatches**. + +So wrapping is arithmetic over a small table, not a font rendering problem, and +the browser reproduces the server's line breaks exactly rather than approximately. + +Do NOT build this for speed. Wrapping is 2.5% of an export -- 6.9ms of 275.7ms +for a sheet and eight item cards -- and shipping a width table to save that alone +is not worth the second implementation to keep in step. Build it as part of +packing, where the browser needs the answer for its own decisions and pre-wrapped +lines come along for nothing. + +If it is built, ship the table from the same TTFs the server loads, so the two +cannot drift; a hand-maintained copy of the numbers is how the line breaks would +quietly stop matching. + +### 9. Styles 2, 3 and 4 — a later branch + +**Landed on `feature/one-template-per-style` after this was written.** Packing +runs on all four styles: `pdf/relabel-numeral!` covers the printed numeral with +a rectangle cut to `pdf/numeral-boxes`, measured per style by +`dev/scan_numerals.clj`, and the column headings' CANTRIPS patch is measured per +style by `dev/scan_cantrips_word.clj`. Box 0 holds cantrips only, so the +style-1-only reuse path below (`reuse-cantrips-box!`, `printed-slot-labels`, +`cantrips-bar`) is no longer reached by the export. The rest of this section +is the state it described at the time. + +Worth separating, because half of this work did cover all four styles. Masters, +growing to a caster count, the no-caster variants and the attribution marks handle +styles 1 to 4 today -- `pdf/sheet-masters` names all four. + +What is style 1 only is the RELABELLING: `printed-slot-labels`, `cantrips-box-rise`, +`cantrips-bar` and `hexagon-offset` in `pdf.clj` are that sheet's artwork, single +values with no style dimension. The other three styles need their own numbers. + +Note also that nothing in production calls `relabel-spell-level!` or +`reuse-cantrips-box!` yet -- grep finds them only in tests. The mechanism works and +waits on item 8 to decide when to use it. + +The method is fixed: PDFTextStripper for positions and size, a high +DPI render for colour, printed landmarks for the offsets between blocks — so this +is measurement rather than design. Style 1 is the only public sheet, which is why +it went first. diff --git a/package.json b/package.json new file mode 100644 index 000000000..7f27995c4 --- /dev/null +++ b/package.json @@ -0,0 +1,8 @@ +{ + "name": "orcpub-browser-e2e", + "private": true, + "description": "Dev-only Node tooling for the browser e2e in test/browser/ (see its header). Not part of the Clojure/lein build.", + "devDependencies": { + "playwright": "^1.62.1" + } +} diff --git a/project.clj b/project.clj index 3679c2fa7..c183b7802 100644 --- a/project.clj +++ b/project.clj @@ -201,6 +201,15 @@ "externs" ["do" "clean" ["run" "-m" "externs"]] "rebuild-modules" ["run" "-m" "user" "--rebuild-modules"] + ;; Boot the FULL app (Pedestal + in-memory Datomic) on :8890 for browser e2e — + ;; no transactor, no external DB. Run `lein e2e-server`, then drive Playwright + ;; against http://localhost:8890 through the real UI. See test/browser/README.md. + "e2e-server" ["with-profile" "+e2e" "run"] + ;; Same server, but with the export queue small enough to hit on one + ;; machine: one sheet at a time, a quarter-second wait for a slot, two + ;; self-retries. Click Export twice quickly, or run + ;; test/browser/export_busy_retry_e2e.js, to see the busy page. + "e2e-server-busy" ["with-profile" "+e2e,+busy-export" "run"] ;; --fail-level error: exit 0 on warnings, exit 1 only on errors. ;; trampoline re-execs in a fresh JVM so System/exit propagates ;; (without it, lein suppresses the exit code). @@ -212,6 +221,15 @@ :profiles {;; Minimal profile to run dev/ command-line tools (e.g. report-preview) ;; with base runtime deps but WITHOUT the heavy cljs/devtools :dev-config. :tools {:source-paths ["dev"]} + ;; In-memory Datomic for local/e2e runs — the full stack boots with no + ;; transactor and no external DB. Used by the `e2e-server` alias; environ + ;; feeds :datomic-url to orcpub.config/get-datomic-uri. + :e2e {:env {:datomic-url "datomic:mem://orcpub"}} + ;; An export queue small enough to reach on one machine, for seeing + ;; the busy page. Stacks on :e2e -- see the e2e-server-busy alias. + :busy-export {:env {:orcpub-pdf-concurrency "1" + :orcpub-pdf-queue-timeout-ms "250" + :orcpub-pdf-max-retries "2"}} ;; Legacy cljsbuild build configs — retained for lein figwheel (legacy ;; dev server) and reference. lein-cljsbuild PLUGIN has been removed; ;; prod CLJS builds now use figwheel-main (prod.cljs.edn / fig:prod). @@ -245,7 +263,8 @@ [nrepl "1.3.0"] [cider/piggieback "0.5.3"] [day8.re-frame/re-frame-10x "1.11.0" :exclusions [zprint rewrite-clj]] - ] + ;; comment_discipline_test parses source with it + [rewrite-clj "0.5.2"]] :env {:dev-mode "true" :signature "dev-secret-do-not-use-in-production"} ;; need to add dev source path here to get user.clj loaded diff --git a/resources/fillable-char-sheetstyle-1-0-spells.pdf b/resources/fillable-char-sheetstyle-1-0-spells.pdf index b42e2e644..7c70a3440 100644 Binary files a/resources/fillable-char-sheetstyle-1-0-spells.pdf and b/resources/fillable-char-sheetstyle-1-0-spells.pdf differ diff --git a/resources/fillable-char-sheetstyle-1-1-spells.pdf b/resources/fillable-char-sheetstyle-1-1-spells.pdf index 70beb892f..e60acc54b 100644 Binary files a/resources/fillable-char-sheetstyle-1-1-spells.pdf and b/resources/fillable-char-sheetstyle-1-1-spells.pdf differ diff --git a/resources/fillable-char-sheetstyle-1-2-spells.pdf b/resources/fillable-char-sheetstyle-1-2-spells.pdf deleted file mode 100644 index 58992cdc4..000000000 Binary files a/resources/fillable-char-sheetstyle-1-2-spells.pdf and /dev/null differ diff --git a/resources/fillable-char-sheetstyle-1-3-spells.pdf b/resources/fillable-char-sheetstyle-1-3-spells.pdf deleted file mode 100644 index 203629d2b..000000000 Binary files a/resources/fillable-char-sheetstyle-1-3-spells.pdf and /dev/null differ diff --git a/resources/fillable-char-sheetstyle-1-4-spells.pdf b/resources/fillable-char-sheetstyle-1-4-spells.pdf deleted file mode 100644 index 1dfb3d7b8..000000000 Binary files a/resources/fillable-char-sheetstyle-1-4-spells.pdf and /dev/null differ diff --git a/resources/fillable-char-sheetstyle-1-5-spells.pdf b/resources/fillable-char-sheetstyle-1-5-spells.pdf deleted file mode 100644 index cb66229f8..000000000 Binary files a/resources/fillable-char-sheetstyle-1-5-spells.pdf and /dev/null differ diff --git a/resources/fillable-char-sheetstyle-1-6-spells.pdf b/resources/fillable-char-sheetstyle-1-6-spells.pdf deleted file mode 100644 index d2c99b86e..000000000 Binary files a/resources/fillable-char-sheetstyle-1-6-spells.pdf and /dev/null differ diff --git a/resources/fillable-char-sheetstyle-2-2-spells.pdf b/resources/fillable-char-sheetstyle-2-2-spells.pdf deleted file mode 100644 index 2e8e2e0d2..000000000 Binary files a/resources/fillable-char-sheetstyle-2-2-spells.pdf and /dev/null differ diff --git a/resources/fillable-char-sheetstyle-2-3-spells.pdf b/resources/fillable-char-sheetstyle-2-3-spells.pdf deleted file mode 100644 index b35fae1d7..000000000 Binary files a/resources/fillable-char-sheetstyle-2-3-spells.pdf and /dev/null differ diff --git a/resources/fillable-char-sheetstyle-2-4-spells.pdf b/resources/fillable-char-sheetstyle-2-4-spells.pdf deleted file mode 100644 index cb6ac4934..000000000 Binary files a/resources/fillable-char-sheetstyle-2-4-spells.pdf and /dev/null differ diff --git a/resources/fillable-char-sheetstyle-2-5-spells.pdf b/resources/fillable-char-sheetstyle-2-5-spells.pdf deleted file mode 100644 index 67a02c9db..000000000 Binary files a/resources/fillable-char-sheetstyle-2-5-spells.pdf and /dev/null differ diff --git a/resources/fillable-char-sheetstyle-2-6-spells.pdf b/resources/fillable-char-sheetstyle-2-6-spells.pdf deleted file mode 100644 index 22a3a8207..000000000 Binary files a/resources/fillable-char-sheetstyle-2-6-spells.pdf and /dev/null differ diff --git a/resources/fillable-char-sheetstyle-3-0-spells.pdf b/resources/fillable-char-sheetstyle-3-0-spells.pdf index 385f22767..743390b30 100644 Binary files a/resources/fillable-char-sheetstyle-3-0-spells.pdf and b/resources/fillable-char-sheetstyle-3-0-spells.pdf differ diff --git a/resources/fillable-char-sheetstyle-3-1-spells.pdf b/resources/fillable-char-sheetstyle-3-1-spells.pdf index a45146980..905dc7bac 100644 Binary files a/resources/fillable-char-sheetstyle-3-1-spells.pdf and b/resources/fillable-char-sheetstyle-3-1-spells.pdf differ diff --git a/resources/fillable-char-sheetstyle-3-2-spells.pdf b/resources/fillable-char-sheetstyle-3-2-spells.pdf deleted file mode 100644 index 01930b69c..000000000 Binary files a/resources/fillable-char-sheetstyle-3-2-spells.pdf and /dev/null differ diff --git a/resources/fillable-char-sheetstyle-3-3-spells.pdf b/resources/fillable-char-sheetstyle-3-3-spells.pdf deleted file mode 100644 index 1b5a20768..000000000 Binary files a/resources/fillable-char-sheetstyle-3-3-spells.pdf and /dev/null differ diff --git a/resources/fillable-char-sheetstyle-3-4-spells.pdf b/resources/fillable-char-sheetstyle-3-4-spells.pdf deleted file mode 100644 index 25d37ac38..000000000 Binary files a/resources/fillable-char-sheetstyle-3-4-spells.pdf and /dev/null differ diff --git a/resources/fillable-char-sheetstyle-3-5-spells.pdf b/resources/fillable-char-sheetstyle-3-5-spells.pdf deleted file mode 100644 index 2d2ff06b9..000000000 Binary files a/resources/fillable-char-sheetstyle-3-5-spells.pdf and /dev/null differ diff --git a/resources/fillable-char-sheetstyle-3-6-spells.pdf b/resources/fillable-char-sheetstyle-3-6-spells.pdf deleted file mode 100644 index 657eda0ba..000000000 Binary files a/resources/fillable-char-sheetstyle-3-6-spells.pdf and /dev/null differ diff --git a/resources/fillable-char-sheetstyle-4-0-spells.pdf b/resources/fillable-char-sheetstyle-4-0-spells.pdf index 14d4de431..1faade274 100644 Binary files a/resources/fillable-char-sheetstyle-4-0-spells.pdf and b/resources/fillable-char-sheetstyle-4-0-spells.pdf differ diff --git a/resources/fillable-char-sheetstyle-4-1-spells.pdf b/resources/fillable-char-sheetstyle-4-1-spells.pdf index 6d4d6266f..f4d9b5c4c 100644 Binary files a/resources/fillable-char-sheetstyle-4-1-spells.pdf and b/resources/fillable-char-sheetstyle-4-1-spells.pdf differ diff --git a/resources/fillable-char-sheetstyle-4-2-spells.pdf b/resources/fillable-char-sheetstyle-4-2-spells.pdf deleted file mode 100644 index 483a138cb..000000000 Binary files a/resources/fillable-char-sheetstyle-4-2-spells.pdf and /dev/null differ diff --git a/resources/fillable-char-sheetstyle-4-3-spells.pdf b/resources/fillable-char-sheetstyle-4-3-spells.pdf deleted file mode 100644 index 47d80a021..000000000 Binary files a/resources/fillable-char-sheetstyle-4-3-spells.pdf and /dev/null differ diff --git a/resources/fillable-char-sheetstyle-4-4-spells.pdf b/resources/fillable-char-sheetstyle-4-4-spells.pdf deleted file mode 100644 index 0a75558fd..000000000 Binary files a/resources/fillable-char-sheetstyle-4-4-spells.pdf and /dev/null differ diff --git a/resources/fillable-char-sheetstyle-4-5-spells.pdf b/resources/fillable-char-sheetstyle-4-5-spells.pdf deleted file mode 100644 index 0bb70eb12..000000000 Binary files a/resources/fillable-char-sheetstyle-4-5-spells.pdf and /dev/null differ diff --git a/resources/fillable-char-sheetstyle-4-6-spells.pdf b/resources/fillable-char-sheetstyle-4-6-spells.pdf deleted file mode 100644 index 773f78eb4..000000000 Binary files a/resources/fillable-char-sheetstyle-4-6-spells.pdf and /dev/null differ diff --git a/resources/public/image/ATTRIBUTION.md b/resources/public/image/ATTRIBUTION.md new file mode 100644 index 000000000..958e4f911 --- /dev/null +++ b/resources/public/image/ATTRIBUTION.md @@ -0,0 +1,33 @@ +# Icon attribution + +Most of the icons in this directory come from **[game-icons.net](https://game-icons.net)** +and are used under the **[Creative Commons Attribution 3.0 Unported](https://creativecommons.org/licenses/by/3.0/)** +licence, which requires that the authors be credited wherever the icons appear. + +The five vendored for the PDF spell cards: + +| Icon | Author | +|---|---| +| `arrow-dunk` | Lorc | +| `magic-swirl` | Lorc | +| `sands-of-time` | Lorc | +| `shiny-purse` | Lorc | +| `clockwise-rotation` | Delapouite | + +The remainder of the set predates this file and the per-icon authorship was not +recorded when it was added. The great majority of game-icons.net is the work of +Lorc, Delapouite and Skoll, all under the same licence. Anyone adding an icon +should record its author in the table above; the author is named on the icon's +page on the site. + +## Which files the PDF export reads + +The card icons are read as **`.svg`** and filled as vector paths, so they print at +the device's resolution rather than at the 32 pixels the old rasters carried. The +`.png` copies that remain are for the web UI. `orcpub.pdf/draw-svg-icon!` falls back +to a `.png` of the same name when no `.svg` is vendored, so either will work, but a +new icon should be added as SVG. + +Colour is applied at the draw site rather than baked into the file, which is why +there are no longer `-bw` duplicates of these five: one path fills red, solid black +or 40% black as the sheet style asks. diff --git a/resources/public/image/arrow-dunk-bw.png b/resources/public/image/arrow-dunk-bw.png deleted file mode 100644 index bad31cd92..000000000 Binary files a/resources/public/image/arrow-dunk-bw.png and /dev/null differ diff --git a/resources/public/image/arrow-dunk.svg b/resources/public/image/arrow-dunk.svg new file mode 100644 index 000000000..cc5f64881 --- /dev/null +++ b/resources/public/image/arrow-dunk.svg @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/resources/public/image/clockwise-rotation-bw.png b/resources/public/image/clockwise-rotation-bw.png deleted file mode 100644 index f47ec4684..000000000 Binary files a/resources/public/image/clockwise-rotation-bw.png and /dev/null differ diff --git a/resources/public/image/clockwise-rotation.svg b/resources/public/image/clockwise-rotation.svg new file mode 100644 index 000000000..100227be3 --- /dev/null +++ b/resources/public/image/clockwise-rotation.svg @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/resources/public/image/magic-swirl-bw.png b/resources/public/image/magic-swirl-bw.png deleted file mode 100644 index fb155f6b3..000000000 Binary files a/resources/public/image/magic-swirl-bw.png and /dev/null differ diff --git a/resources/public/image/sands-of-time-bw.png b/resources/public/image/sands-of-time-bw.png deleted file mode 100644 index 4c0858468..000000000 Binary files a/resources/public/image/sands-of-time-bw.png and /dev/null differ diff --git a/resources/public/image/sands-of-time.svg b/resources/public/image/sands-of-time.svg new file mode 100644 index 000000000..2007b70ad --- /dev/null +++ b/resources/public/image/sands-of-time.svg @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/resources/public/image/shiny-purse-bw.png b/resources/public/image/shiny-purse-bw.png deleted file mode 100644 index f4ddedddc..000000000 Binary files a/resources/public/image/shiny-purse-bw.png and /dev/null differ diff --git a/resources/public/image/shiny-purse.svg b/resources/public/image/shiny-purse.svg new file mode 100644 index 000000000..a041c633e --- /dev/null +++ b/resources/public/image/shiny-purse.svg @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/resources/public/js/cookies.js b/resources/public/js/cookies.js index 47902b11d..f237f49dc 100644 --- a/resources/public/js/cookies.js +++ b/resources/public/js/cookies.js @@ -73,7 +73,22 @@ function Pop() { "path=" + cookie.path; }; + // Opt-out for automated runs and local testing. The banner is position-fixed at the + // bottom of the page and overlays whatever is under it - notably the import + // conflict-resolution modal's buttons - so a UI test clicking there gets + // "subtree intercepts pointer events" and fails in a way that looks like an app bug. + // localStorage is the harness-friendly form (set it before load, no interaction); + // the query param is for driving a browser by hand. + let suppressed = function() { + try { + if (new URLSearchParams(window.location.search).has("no-cookie-banner")) { + localStorage.setItem("orcpub:no-cookie-banner", "1"); + } + return localStorage.getItem("orcpub:no-cookie-banner") === "1"; + } catch (e) { return false; } + }; this.init = function(param) { + if (suppressed()) return; if (checkCookie(cookie.name)) return; if (typeof param === "object") { if ("ButtonText" in param) content.btnText = param.ButtonText; diff --git a/run b/run index fe74f797c..79fbeb9a4 100755 --- a/run +++ b/run @@ -1118,7 +1118,6 @@ if [ "$UP_MODE" = "true" ]; then # docker compose config outputs Compose Specification format; # docker stack deploy expects legacy v3 schema. JSON + jq bridges the gap. - # See docs/kb/docker-swarm-compat.md for the full incompatibility list. docker compose config --format json | jq ' del(.name) | .services |= with_entries( diff --git a/scripts/clj-grep.py b/scripts/clj-grep.py new file mode 100755 index 000000000..eaeadfea5 --- /dev/null +++ b/scripts/clj-grep.py @@ -0,0 +1,259 @@ +#!/usr/bin/env python3 +"""Grep Clojure sources, separating live code from `#_` discarded forms. + +`#_` discards the NEXT form, however long it is. A plain grep hit inside one looks +exactly like live code, so `classes.cljc` reports a Beast Master and a Circle of the +Moon that the app has never compiled -- 27 subclasses are discarded that way, one per +class kept, the SRD 5.1 set. + +Default output reports both, in separate sections. It does NOT hide discards: knowing +the code exists but is switched off is usually the answer you wanted. + +Usage: + scripts/clj-grep.py PATTERN [PATH ...] both sections (default: src/ test/) + scripts/clj-grep.py --live PATTERN [...] live hits only, as path:line:text + scripts/clj-grep.py --dead PATTERN [...] discarded hits only, same shape + scripts/clj-grep.py -i PATTERN [...] case-insensitive + +Exits 0 when the selected section has hits, 1 when it does not (grep convention). +""" + +import os +import re +import sys + +CLJ_EXT = (".clj", ".cljs", ".cljc", ".edn") +OPEN = {"(": ")", "[": "]", "{": "}"} +CLOSE = set(")]}") +# Reader prefixes that bind to the form after them; a discard has to step over these +# to find the form it actually kills. +PREFIX = set("'`~@^#") + + +def discarded_ranges(src): + """Character spans killed by `#_`, as a list of (start, end) index pairs. + + One pass with a reader that knows strings, character literals and line comments, + because each of those can hold delimiters that would otherwise unbalance the scan. + """ + spans = [] + i, n = 0, len(src) + while i < n: + c = src[i] + if c == ";": + i = src.find("\n", i) + if i < 0: + break + elif c == '"': + i = skip_string(src, i) + elif c == "\\": + i = skip_char_literal(src, i) + elif c == "#" and i + 1 < n and src[i + 1] == '"': + i = skip_string(src, i + 1) + elif c == "#" and i + 1 < n and src[i + 1] == "_": + # `#_#_ a b` discards two forms, so count the run before consuming any. + j, pending = i, 0 + while src.startswith("#_", j): + pending += 1 + j += 2 + while j < n and src[j] in " \t\n\r,": + j += 1 + for _ in range(pending): + j = skip_ws_and_comments(src, j) + if j >= n: + break + end = form_end(src, j) + spans.append((j, end)) + j = end + i = j + else: + i += 1 + return spans + + +def skip_string(src, i): + """Index just past the string literal opening at `i`.""" + i += 1 + while i < len(src): + if src[i] == "\\": + i += 2 + continue + if src[i] == '"': + return i + 1 + i += 1 + return i + + +def skip_char_literal(src, i): + """Index just past a `\\x` / `\\newline` / `\\(` literal opening at `i`.""" + i += 1 + if i < len(src): + i += 1 # the character itself, even when it is a delimiter or a quote + while i < len(src) and (src[i].isalnum() or src[i] == "-"): + i += 1 # named literals: \newline, \space, + return i + + +def skip_ws_and_comments(src, i): + n = len(src) + while i < n: + if src[i] in " \t\n\r,": + i += 1 + elif src[i] == ";": + nl = src.find("\n", i) + i = n if nl < 0 else nl + 1 + else: + return i + return i + + +def form_end(src, i): + """Index just past the single form starting at `i`.""" + n = len(src) + # Step over reader prefixes (' ` ~ @ ^ # and #{ ), plus metadata maps, which bind + # to the form after them rather than being the form. + while i < n and src[i] in PREFIX: + if src[i] == "#" and i + 1 < n and src[i + 1] == '"': + return skip_string(src, i + 1) + if src[i] == "^": + i += 1 + i = form_end(src, skip_ws_and_comments(src, i)) + i = skip_ws_and_comments(src, i) + continue + i += 1 + if i < n and src[i] in OPEN: + break + if i >= n: + return n + c = src[i] + if c == '"': + return skip_string(src, i) + if c == "\\": + return skip_char_literal(src, i) + if c in OPEN: + depth, i = 0, i + while i < n: + ch = src[i] + if ch == ";": + nl = src.find("\n", i) + i = n if nl < 0 else nl + elif ch == '"': + i = skip_string(src, i) + continue + elif ch == "\\": + i = skip_char_literal(src, i) + continue + elif ch in OPEN: + depth += 1 + elif ch in CLOSE: + depth -= 1 + if depth == 0: + return i + 1 + i += 1 + return n + # An atom: runs to the next delimiter or whitespace. + while i < n and src[i] not in " \t\n\r,()[]{}\";": + i += 1 + return i + + +def line_starts(src): + starts, pos = [0], src.find("\n") + while pos >= 0: + starts.append(pos + 1) + pos = src.find("\n", pos + 1) + return starts + + +def scan(path, rx): + """Hits in one file as (line_no, text, discard_line_or_None).""" + try: + src = open(path, encoding="utf-8", errors="replace").read() + except OSError: + return [] + spans = discarded_ranges(src) + starts = line_starts(src) + + def line_of(idx): + lo, hi = 0, len(starts) - 1 + while lo < hi: + mid = (lo + hi + 1) // 2 + if starts[mid] <= idx: + lo = mid + else: + hi = mid - 1 + return lo + 1 + + out = [] + for m in rx.finditer(src): + ln = line_of(m.start()) + end = src.find("\n", m.start()) + text = src[starts[ln - 1]:(len(src) if end < 0 else end)] + killed = next((s for s, e in spans if s <= m.start() < e), None) + out.append((ln, text, None if killed is None else line_of(killed))) + return out + + +def walk(paths): + for p in paths: + if os.path.isfile(p): + yield p + for root, dirs, files in os.walk(p): + dirs[:] = [d for d in dirs if d not in {".git", "node_modules", "target", "out"}] + for f in sorted(files): + if f.endswith(CLJ_EXT): + yield os.path.join(root, f) + + +def main(argv): + mode, flags = "both", 0 + args = [] + for a in argv: + if a == "--live": + mode = "live" + elif a == "--dead": + mode = "dead" + elif a in ("-i", "--ignore-case"): + flags |= re.IGNORECASE + elif a in ("-h", "--help"): + print(__doc__) + return 0 + else: + args.append(a) + if not args: + print(__doc__, file=sys.stderr) + return 2 + + pattern, paths = args[0], args[1:] or ["src", "test"] + paths = [p for p in paths if os.path.exists(p)] + rx = re.compile(pattern, flags) + + live, dead = [], [] + for path in walk(paths): + for ln, text, killed in scan(path, rx): + (dead if killed else live).append((path, ln, text, killed)) + + if mode in ("live", "dead"): + rows = live if mode == "live" else dead + for path, ln, text, _ in rows: + print(f"{path}:{ln}:{text}") + return 0 if rows else 1 + + if live: + print(f"LIVE ({len(live)})") + for path, ln, text, _ in live: + print(f" {path}:{ln}:{text.strip()[:100]}") + if dead: + if live: + print() + print(f"DISCARDED BY #_ ({len(dead)}) -- present in the file, never compiled") + for path, ln, text, killed in dead: + print(f" {path}:{ln}:{text.strip()[:100]}") + print(f" discarded at {path}:{killed}") + if not live and not dead: + return 1 + return 0 + + +if __name__ == "__main__": + sys.exit(main(sys.argv[1:])) diff --git a/scripts/e2e/api-sub-loaders.js b/scripts/e2e/api-sub-loaders.js new file mode 100644 index 000000000..8068afa5d --- /dev/null +++ b/scripts/e2e/api-sub-loaders.js @@ -0,0 +1,182 @@ +// Do the five API-backed subscriptions still load the same way? +// +// ./scripts/e2e/run.sh api-sub-loaders.js +// +// P5 of the #669 branch moves ::mi5e/custom-items, ::char5e/characters, +// ::party5e/parties, ::folder5e/folders and :user onto one reg-api-sub HOF. +// Nothing in either test suite covers four of those five, and a refactor's +// failure mode is a SILENT behaviour change -- so this is a DIFFERENTIAL probe, +// not a pass/fail one. Run it on integration, run it on the merge, diff the +// TRACE blocks. Identical trace = the refactor changed nothing observable. +// +// What it watches: every request to the five endpoints -- method, path, whether +// it carried an Authorization header, and its status. Those four facts are +// exactly what the HOF took over from the five hand-written sites. +// +// It also asserts RESPONSE handling, which the first version could not: the +// seeded accounts own items, so a 200 now carries a body and the list is the +// evidence that :set-event -> db-population still works. And it asserts +// ISOLATION -- signing in as one account must not surface the other's items. +// Run it as both: E2E_USER=kaylee (default) and E2E_USER=zoe E2E_PASS=washburne7 +// +// SELFTEST=1 logs in with a wrong password. The endpoints must then NOT be +// requested at all, because every one of them is guarded on the auth token. +const { chromium } = require('playwright'); +const fs = require('fs'); + +const BASE = process.env.E2E_BASE || 'http://localhost:8890'; +const EXECUTABLE = process.env.E2E_CHROMIUM + || ['/opt/pw-browsers/chromium-1194/chrome-linux/chrome', + '/opt/pw-browsers/chromium/chrome-linux/chrome'].find(p => fs.existsSync(p)); +const USER = process.env.E2E_USER || 'kaylee'; +const PASS = process.env.SELFTEST ? 'wrong-on-purpose' : (process.env.E2E_PASS || 'serenity99'); + +// dev/e2e_boot.clj seeds two accounts, each owning its own items. One account +// cannot show isolation: a query returning EVERYBODY's items would look exactly +// like a correct one. These are the names it seeds. +const OWNED = { kaylee: ['Kaylee Seeded Alpha', 'Kaylee Seeded Beta', 'Kaylee Seeded Gamma'], + zoe: ['Zoe Seeded Only'] }; +const OTHERS = Object.entries(OWNED).filter(([u]) => u !== USER).flatMap(([, v]) => v); + +// The five subs' endpoints. There is no /api/ prefix -- the route tree is rooted +// at "/", so the API lives at /dnd/5e/... and the SPA pages at /pages/dnd/5e/... +// The discriminator is therefore the ABSENCE of /pages/. A tail-only match +// counted an HTML page load as an unauthenticated API call; the control run +// caught that, which is the point of having one. +const isApi = url => !/\/pages\//.test(new URL(url).pathname); +const WATCH = { + items: /\/items(\?|$)/, + characters: /character-summaries(\?|$)/, + parties: /\/parties(\?|$)/, + folders: /\/folders(\?|$)/, + user: /\/user(\?|$)/, +}; + +let failures = 0; +const check = (n, ok, d) => { console.log(` ${ok ? 'PASS' : 'FAIL'} ${n}${d ? ' — ' + d : ''}`); if (!ok) failures++; }; + +(async () => { + const browser = await chromium.launch({ executablePath: EXECUTABLE }); + const ctx = await browser.newContext({ viewport: { width: 1400, height: 1000 } }); + await ctx.addInitScript(() => { + try { + localStorage.setItem('orcpub:no-cookie-banner', '1'); + localStorage.setItem('whats-new-seen', JSON.stringify('summer-patch-2026')); + } catch (e) {} + }); + const page = await ctx.newPage(); + + const hits = []; // {name, method, auth, status} + let itemsBody = ''; // raw body of the items response, to tell a server-side + // miss from a render-side one when the list looks empty + const pageErrors = []; + page.on('pageerror', e => pageErrors.push(e.message)); + page.on('response', async res => { + const url = res.url(); + if (!isApi(url)) return; + for (const [name, re] of Object.entries(WATCH)) { + if (!re.test(url)) continue; + const req = res.request(); + hits.push({ name, method: req.method(), + auth: !!(req.headers()['authorization']), status: res.status() }); + if (name === 'items' && res.status() === 200) { + try { itemsBody += await res.text(); } catch (e) {} + } + break; + } + }); + + // Log in. + await page.goto(`${BASE}/pages/login-page`, { waitUntil: 'domcontentloaded' }); + await page.waitForSelector('input'); + await page.locator('input').nth(0).fill(USER); + await page.locator('input').nth(1).fill(PASS); + await page.locator('button.form-button').first().click(); + await page.waitForTimeout(6000); + + const loggedIn = await page.evaluate(() => { + try { return !!localStorage.getItem('user'); } catch (e) { return false; } + }); + console.log(`\nsigned in: ${loggedIn}${process.env.SELFTEST ? ' (SELFTEST: expected false)' : ''}`); + + // Visit the pages that mount the five subs. + for (const path of ['/pages/dnd/5e/magic-items', + '/pages/dnd/5e/characters', + '/pages/dnd/5e/parties']) { + await page.goto(BASE + path, { waitUntil: 'load', timeout: 120000 }); + await page.waitForTimeout(7000); + } + + // --- response handling + isolation ------------------------------------- + // The first version of this probe ran against an account with NO content, so + // every 200 carried an empty body and the :set-event -> db-population path + // never ran. That is exactly where a wrong :db-key hides. With seeded content + // the list itself is the assertion. + // /my-content is the HOMEBREW SOURCES page (orcbrew import, quarantine panel). + // Custom magic items render on /magic-items -- route dnd-e5-item-list-page-route, + // the page whose list is ::char5e/filtered-items, which is the chain P1 fixes. + // Asserting against my-content reported "not rendered" for items that were in + // the API response the whole time. + await page.goto(BASE + '/pages/dnd/5e/magic-items', { waitUntil: 'load', timeout: 120000 }); + await page.waitForTimeout(9000); + const listed = await page.evaluate(() => document.body.innerText); + + // The loading overlay is a COUNTER, not a boolean (reframe-subscription-patterns.md). + // P5 adds an increment/decrement pair to :user that did not exist before, so the + // thing to assert is that it still settles. + const overlayVisible = await page.evaluate(() => { + const el = document.querySelector('.loading-overlay, [class*=loading]'); + if (!el) return false; + const s = getComputedStyle(el); + return s.display !== 'none' && s.visibility !== 'hidden' && el.offsetParent !== null; + }); + + console.log('\nchecks:'); + if (process.env.SELFTEST) { + check('no endpoint was requested while unauthenticated', hits.length === 0, + `${hits.length} request(s): ${[...new Set(hits.map(h => h.name))].join(',')}`); + } else { + for (const name of Object.keys(WATCH)) { + const got = hits.filter(h => h.name === name); + check(`${name}: requested`, got.length > 0, `${got.length} request(s)`); + if (got.length) { + check(`${name}: every request carried auth`, got.every(h => h.auth)); + check(`${name}: no 401`, !got.some(h => h.status === 401), + `statuses ${[...new Set(got.map(h => h.status))].join(',')}`); + } + } + } + if (!process.env.SELFTEST) { + const mine = OWNED[USER] || []; + // Two separate assertions on purpose. The API one says the server found and + // returned the rows; the page one says the sub populated db and the list + // rendered them. Collapsing them into "the list is empty" cannot tell a + // :db-key regression from a tab that simply is not open. + const apiMissing = mine.filter(n => !itemsBody.includes(n)); + check(`${USER}'s items came back from the API`, apiMissing.length === 0, + apiMissing.length ? `missing from response: ${apiMissing.join(', ')}` + : `all ${mine.length} in the response body`); + const missing = mine.filter(n => !listed.includes(n)); + check(`${USER}'s own items are rendered`, missing.length === 0, + missing.length ? `missing from page: ${missing.join(', ')}` : `all ${mine.length} present`); + const leaked = OTHERS.filter(n => listed.includes(n) || itemsBody.includes(n)); + check("no other account's items leaked in", leaked.length === 0, + leaked.length ? `LEAKED: ${leaked.join(', ')}` : `none of ${OTHERS.length} other-owner items`); + } + check('the loading overlay settled', !overlayVisible); + check('no uncaught page errors', pageErrors.length === 0, pageErrors.slice(0, 2).join(' | ')); + + // Normalised, order-independent trace. This is the artefact to diff between runs. + const trace = {}; + for (const h of hits) { + const k = `${h.name} ${h.method} auth=${h.auth} status=${h.status}`; + trace[k] = (trace[k] || 0) + 1; + } + console.log('\nTRACE'); + Object.keys(trace).sort().forEach(k => console.log(` ${trace[k]}x ${k}`)); + console.log('END-TRACE'); + + console.log(`\n${failures ? 'FAILURES: ' + failures : 'all checks passed'}`); + await browser.close(); + process.exit(failures ? 1 : 0); +})(); diff --git a/scripts/e2e/auth-flows.js b/scripts/e2e/auth-flows.js new file mode 100644 index 000000000..99a404880 --- /dev/null +++ b/scripts/e2e/auth-flows.js @@ -0,0 +1,489 @@ +// Every state of every flow where an account is made, verified, entered or recovered. +// +// ./scripts/e2e/run.sh auth-flows.js +// +// These flows had NO coverage, and no way to get any: the verification key and the reset +// key exist only inside an email, registration itself fails when no mail can be sent, and +// every other suite sidesteps all of it by starting from a user e2e-boot seeded straight +// into Datomic. So nothing here was ever exercised, and a reset form that refused every +// ordinary password in silence reached a merge. +// +// run.sh starts scripts/e2e/lib/mail-sink.js and points the app at it, which is what makes +// the keys readable. Without E2E_MAIL_PORT this suite skips rather than pretending to pass. +const { chromium } = require('playwright'); +const { BASE, EXECUTABLE, newContext, checker } = require('./lib'); +const { startMailSink } = require('./lib/mail-sink'); + +const PORT = Number(process.env.E2E_MAIL_PORT || 0); +const GOOD = 'brass lantern rope'; +// 14 characters, passes every local rule, and the corpus has seen it thousands of +// times -- so ONLY the server can refuse it. It is the one case that proves the +// server's reason reaches the page. +const COMMON = 'letmeinletmein'; +const SHORT = 'Sw0rdfish'; + +const uniq = () => 'e2e' + Date.now().toString(36) + Math.floor(Math.random() * 1e4); +const text = (page, sel) => page.locator(sel).innerText().catch(() => ''); +const cardText = page => text(page, '.registration-content'); +const heading = page => text(page, '.auth-heading'); +const authInputs = page => page.locator('.auth-form input'); +const press = (page, label) => page.locator('.auth-tail button', { hasText: label }).click(); + +// The link in a captured mail, with quoted-printable soft breaks undone. +const linkIn = (mail, re) => (mail.raw.replace(/=\r\n/g, '').match(re) || [])[0]; + +async function registerAndVerify(page, sink, user, email, password) { + // Clear FIRST. waitFor returns a matching message already held, so with mail + // from an earlier account still in the sink this returned that one and + // re-verified somebody else -- leaving the new account uncreated and the + // failure surfacing three steps later as reset mail that never arrived. + sink.clear(); + await page.goto(`${BASE}/pages/register-page`, { waitUntil: 'networkidle' }); + await page.waitForSelector('.auth-form input'); + const b = authInputs(page); + await b.nth(0).fill(user); await b.nth(1).fill(email); await b.nth(2).fill(email); + await b.nth(3).fill(password); await b.nth(4).fill(password); + await press(page, 'JOIN'); + const mail = await sink.waitFor(new RegExp(email.replace('.', '\\.'), 'i'), 20000); + const key = (mail.raw.replace(/=\r\n/g, '').match(/verify\?key=([A-Za-z0-9-]+)/) || [])[1]; + await page.goto(`${BASE}/verify?key=${key}`, { waitUntil: 'networkidle' }); + await page.waitForTimeout(800); + return key; +} + +// A signed-in session sends the register page somewhere else, so a block that +// registers after one that logs in silently registers nobody -- which showed up +// three steps later as reset mail that never arrived. +async function signOut(ctx, page) { + await ctx.clearCookies(); + await page.goto(`${BASE}/pages/login-page`, { waitUntil: 'domcontentloaded' }); + await page.evaluate(() => { try { localStorage.removeItem('token'); } catch (e) {} }); +} + +// null when no reset mail arrives, rather than throwing: a limit that stops the +// mail is a RESULT this suite asserts about, not a crash. +async function resetLinkFor(page, sink, email) { + sink.clear(); + await page.goto(`${BASE}/pages/send-password-reset-page`, { waitUntil: 'networkidle' }); + await page.waitForSelector('.auth-form input'); + await authInputs(page).first().fill(email); + await press(page, 'SUBMIT'); + const mail = await sink.waitFor(/reset/i, 20000).catch(() => null); + if (!mail) return null; + return linkIn(mail, /https?:\/\/[^\s"'<>]*reset-password[^\s"'<>]*/); +} + +(async () => { + if (!PORT) { + console.log('SKIP auth-flows.js needs E2E_MAIL_PORT; run it through scripts/e2e/run.sh'); + process.exit(0); + } + const sink = await startMailSink({ port: PORT }); + const browser = await chromium.launch({ executablePath: EXECUTABLE }); + const ctx = await newContext(browser); + const page = await ctx.newPage(); + const { check, failures } = checker(); + const errors = []; + page.on('pageerror', e => errors.push(String(e))); + + const user = uniq(), email = `${user}@example.com`; + + console.log('registration'); + await registerAndVerify(page, sink, user, email, GOOD); + check((await heading(page)).toLowerCase().includes('registration is complete'), + 'a new account registers and verifies', await heading(page)); + + // The same address and name a second time must be refused, per field. + await page.goto(`${BASE}/pages/register-page`, { waitUntil: 'networkidle' }); + await page.waitForSelector('.auth-form input'); + let b = authInputs(page); + await b.nth(0).fill(user); await b.nth(1).fill(email); await b.nth(2).fill(email); + await b.nth(3).fill(GOOD); await b.nth(4).fill(GOOD); + await press(page, 'JOIN'); + await page.waitForTimeout(2500); + const taken = await cardText(page); + check(/taken|already/i.test(taken), 'a username and email already in use are refused', + taken.replace(/\n/g, ' | ').slice(0, 120)); + + // Short password: the rules must say what is wrong, not just refuse. + await page.goto(`${BASE}/pages/register-page`, { waitUntil: 'networkidle' }); + await page.waitForSelector('.auth-form input'); + b = authInputs(page); + await b.nth(0).fill(uniq()); const e2 = uniq() + '@example.com'; + await b.nth(1).fill(e2); await b.nth(2).fill(e2); + await b.nth(3).fill(SHORT); await b.nth(4).fill(SHORT); + await press(page, 'JOIN'); + await page.waitForTimeout(1200); + check(/at least 12 characters/i.test(await cardText(page)), + 'a short password is refused WITH its reason'); + + // The chips and the tip. Five rules, each visible as its own chip before + // anything is pressed, and a line pointing forward once nothing is wrong. + await page.goto(`${BASE}/pages/register-page`, { waitUntil: 'networkidle' }); + await page.waitForSelector('.auth-form input'); + b = authInputs(page); + await b.nth(0).fill('kaylee'); + const chipState = async () => { + await page.waitForTimeout(400); + return page.locator('.pw-chip').evaluateAll(els => Object.fromEntries( + els.map(e => [e.textContent.trim(), e.classList.contains('is-ok')]))); + }; + await b.nth(3).fill('aaaaaaaaaaaa'); + let c = await chipState(); + check(c['12 or more'] === true && c['no triples'] === false && c['some variety'] === false, + 'twelve of one letter: length passes, triples and variety do not', JSON.stringify(c)); + await b.nth(3).fill('abcdefghijkl'); + c = await chipState(); + check(c['no key runs'] === false, 'an alphabet run fails its own chip', JSON.stringify(c)); + await b.nth(3).fill('kayleekaylee12'); + c = await chipState(); + check(c['not your name'] === false, 'a password holding the username fails its own chip', + JSON.stringify(c)); + await b.nth(3).fill('brass lantern rope candle bell'); + c = await chipState(); + check(Object.values(c).every(Boolean) && Object.keys(c).length === 5, + 'a real passphrase satisfies all five', JSON.stringify(c)); + const tip = await text(page, '.pw-note'); + check(tip.length > 0, 'and the meter says what to do next rather than only what is wrong', tip); + + console.log('login'); + await page.goto(`${BASE}/pages/login-page`, { waitUntil: 'networkidle' }); + await page.waitForSelector('input'); + await page.locator('input').nth(0).fill(user); + await page.locator('input').nth(1).fill('not the password'); + await page.locator('button.form-button').click(); + await page.waitForTimeout(2500); + const wrongPass = await cardText(page); + + await page.goto(`${BASE}/pages/login-page`, { waitUntil: 'networkidle' }); + await page.waitForSelector('input'); + await page.locator('input').nth(0).fill('nobody' + uniq()); + await page.locator('input').nth(1).fill('not the password'); + await page.locator('button.form-button').click(); + await page.waitForTimeout(2500); + const noSuchUser = await cardText(page); + + // The oracle: a wrong password and a username that does not exist must be + // indistinguishable, or the login form is a membership test. + check(wrongPass === noSuchUser, + 'a wrong password and an unknown username answer identically', + wrongPass === noSuchUser ? '' : `"${wrongPass.slice(0,60)}" vs "${noSuchUser.slice(0,60)}"`); + + console.log('password reset'); + const link = await resetLinkFor(page, sink, email); + check(!!link, 'a reset link is emailed for a known address', link || 'no link in the mail'); + + // An address nobody registered must answer the same way and send nothing. + sink.clear(); + await page.goto(`${BASE}/pages/send-password-reset-page`, { waitUntil: 'networkidle' }); + await page.waitForSelector('.auth-form input'); + await authInputs(page).first().fill(uniq() + '@example.com'); + await press(page, 'SUBMIT'); + await page.waitForTimeout(2500); + check((await heading(page)).toLowerCase().includes('check your email'), + 'an unknown address is answered the same way'); + check(sink.messages.length === 0, 'and no mail is sent for it', + sink.messages.length ? `${sink.messages.length} sent` : ''); + + await page.goto(link, { waitUntil: 'networkidle' }); + await page.waitForSelector('.auth-form input'); + check((await heading(page)).toLowerCase().includes('choose a new password'), + 'the link opens the reset form'); + check(await page.locator('.pw-slot').count() === 4, + 'the strength meter is on the reset form too'); + + // Short: refused by a rule the browser knows, so it must say so without a round trip. + let r = authInputs(page); + await r.nth(0).fill(SHORT); await r.nth(1).fill(SHORT); + await press(page, 'SUBMIT'); + await page.waitForTimeout(800); + check(/at least 12 characters/i.test(await cardText(page)), + 'a short new password says what is wrong'); + + // Common: passes every local rule, so ONLY the server can refuse it. This is the + // case that was answering 400 with an empty body and a generic apology. + r = authInputs(page); + await r.nth(0).fill(COMMON); await r.nth(1).fill(COMMON); + await press(page, 'SUBMIT'); + await page.waitForTimeout(3500); + const common = await cardText(page); + check(/too common/i.test(common), + "the server's reason for a common password reaches the page", + common.replace(/\n/g, ' | ').slice(0, 140)); + + // Mismatch, while masked. + r = authInputs(page); + await r.nth(0).fill(GOOD + ' one'); await r.nth(1).fill(GOOD + ' two'); + await press(page, 'SUBMIT'); + await page.waitForTimeout(800); + check(/do not match/i.test(await cardText(page)), 'a mismatched confirmation is caught'); + + // And the whole point: a good password goes through and then LETS YOU IN. + const fresh = GOOD + ' candle'; + r = authInputs(page); + await r.nth(0).fill(fresh); await r.nth(1).fill(fresh); + await press(page, 'SUBMIT'); + await page.waitForTimeout(3500); + check(/password/i.test(await heading(page)) === false || /changed|reset/i.test(await cardText(page)), + 'the reset completes', (await heading(page))); + + await page.goto(`${BASE}/pages/login-page`, { waitUntil: 'networkidle' }); + await page.waitForSelector('input'); + await page.locator('input').nth(0).fill(user); + await page.locator('input').nth(1).fill(fresh); + await page.locator('button.form-button').click(); + await page.waitForTimeout(3500); + check(!page.url().includes('login-page'), 'and the new password logs in', page.url().replace(BASE, '')); + + // The link is one use only. + await page.goto(link, { waitUntil: 'networkidle' }); + await page.waitForTimeout(1500); + const reused = (await cardText(page)).toLowerCase(); + check(/no longer works|already been used|expired/.test(reused), + 'a reset link cannot be used twice', reused.replace(/\n/g, ' | ').slice(0, 120)); + + // ---- the limits, none of which had ever been exercised ---- + console.log('limits'); + + // Five failures against one username inside the window, then it is turned away + // whatever is typed -- including, on the sixth, the RIGHT password. + const victim = uniq(), vEmail = victim + '@example.com'; + await signOut(ctx, page); + await registerAndVerify(page, sink, victim, vEmail, GOOD); + for (let i = 0; i < 5; i++) { + await page.goto(`${BASE}/pages/login-page`, { waitUntil: 'networkidle' }); + await page.waitForSelector('input'); + await page.locator('input').nth(0).fill(victim); + await page.locator('input').nth(1).fill('wrong one ' + i); + await page.locator('button.form-button').click(); + await page.waitForTimeout(1200); + } + // A sixth WRONG guess is turned away by the limit rather than by the lookup. + await page.goto(`${BASE}/pages/login-page`, { waitUntil: 'networkidle' }); + await page.waitForSelector('input'); + await page.locator('input').nth(0).fill(victim); + await page.locator('input').nth(1).fill('wrong one 5'); + await page.locator('button.form-button').click(); + await page.waitForTimeout(2500); + const throttled = await cardText(page); + check(page.url().includes('login-page') && /too many|attempt/i.test(throttled), + 'a sixth wrong guess at one account is turned away by the limit', + throttled.replace(/\n/g, ' | ').slice(0, 110)); + + // And the owner still gets in. Locking the account on five wrong guesses would + // hand anybody a way to lock anybody out, so the limit deliberately sits on the + // FAILURE path only -- this is the assertion that keeps it there. + await page.goto(`${BASE}/pages/login-page`, { waitUntil: 'networkidle' }); + await page.waitForSelector('input'); + await page.locator('input').nth(0).fill(victim); + await page.locator('input').nth(1).fill(GOOD); + await page.locator('button.form-button').click(); + await page.waitForTimeout(3500); + check(!page.url().includes('login-page'), + 'and the real password still gets the owner in, so nobody can lock them out', + page.url().replace(BASE, '')); + + // Three reset requests for one address inside the hour; the fourth sends no mail + // but must still answer identically, or the limit is itself an oracle. + const r2 = uniq(), r2Email = r2 + '@example.com'; + await signOut(ctx, page); + await registerAndVerify(page, sink, r2, r2Email, GOOD); + check((await heading(page)).toLowerCase().includes('registration is complete'), + 'a second account registers after a session was signed in', await heading(page)); + const sent = []; + for (let i = 0; i < 3; i++) sent.push(Boolean(await resetLinkFor(page, sink, r2Email))); + check(sent.every(Boolean), `all ${sent.length} reset requests inside the limit are sent`, + `sent = [${sent.join(', ')}]`); + sink.clear(); + await page.goto(`${BASE}/pages/send-password-reset-page`, { waitUntil: 'networkidle' }); + await page.waitForSelector('.auth-form input'); + await authInputs(page).first().fill(r2Email); + await press(page, 'SUBMIT'); + await page.waitForTimeout(2500); + check((await heading(page)).toLowerCase().includes('check your email'), + 'a throttled reset still answers the same way'); + check(sink.messages.length === 0, 'and sends no fourth mail in the hour', + sink.messages.length ? `${sink.messages.length} sent` : ''); + + // ---- the card at phone width ---- + const phone = await ctx.newPage(); + await phone.setViewportSize({ width: 390, height: 844 }); + await phone.goto(`${BASE}/pages/register-page`, { waitUntil: 'networkidle' }); + await phone.waitForTimeout(1200); + const docW = await phone.evaluate(() => document.documentElement.scrollWidth); + check(docW <= 392, 'the register card does not scroll sideways on a phone', `scrollWidth ${docW}`); + const solid = await phone.evaluate(() => { + const root = document.querySelector('.registration-content'); + let n = 0; + for (const el of root.querySelectorAll('*')) { + if (![...el.childNodes].some(x => x.nodeType === 3 && x.textContent.trim())) continue; + const st = getComputedStyle(el); + // The three deliberate ones: .notch and .lift are single-line label chips + // positioned to the pixel, and .auth-heading is display type at 1.2. + if (el.classList.contains('notch') || el.classList.contains('lift') + || el.classList.contains('auth-heading')) continue; + if (parseFloat(st.lineHeight) / parseFloat(st.fontSize) < 1.25) n++; + } + return n; + }); + check(solid === 0, 'no text on the card is set solid', solid ? `${solid} elements under 1.25` : ''); + await phone.close(); + + // ---- the notices, and who is allowed to move an account ---- + console.log('notices'); + + const nUser = uniq(), nEmail = nUser + '@example.com'; + await signOut(ctx, page); + await registerAndVerify(page, sink, nUser, nEmail, GOOD); + + // A completed reset must tell the account that it happened. + const nLink = await resetLinkFor(page, sink, nEmail); + await page.goto(nLink, { waitUntil: 'networkidle' }); + await page.waitForSelector('.auth-form input'); + sink.clear(); + const np = GOOD + ' kettle'; + let nb = authInputs(page); + await nb.nth(0).fill(np); await nb.nth(1).fill(np); + await press(page, 'SUBMIT'); + await page.waitForTimeout(3000); + let notice = await sink.waitFor(/password changed/i, 15000).catch(() => null); + check(Boolean(notice), 'a completed password reset tells the account it happened', + notice ? notice.headers.subject : 'no mail arrived'); + if (notice) { + check(notice.headers.to.includes(nEmail), 'and it goes to the account address', notice.headers.to); + check(!/key=|token=/.test(notice.raw), + 'and carries no working credential, so it does not teach people to click one'); + } + + // Moving the account to another address needs the password, not just a session. + await page.goto(`${BASE}/pages/login-page`, { waitUntil: 'networkidle' }); + await page.waitForSelector('input'); + await page.locator('input').nth(0).fill(nUser); + await page.locator('input').nth(1).fill(np); + await page.locator('button.form-button').click(); + await page.waitForTimeout(3500); + + // Driven through the form a person uses, not a hand-rolled request: the + // endpoint takes transit over PUT and getting either wrong looks like a pass. + const moveTo = uniq() + '@example.com'; + const attempt = async (pw) => { + await page.goto(`${BASE}/pages/my-account`, { waitUntil: 'networkidle' }); + await page.waitForTimeout(1500); + const change = page.locator('button', { hasText: /^Change$|Change email|Change again/ }).first(); + if (await change.count()) await change.click().catch(() => {}); + await page.waitForTimeout(500); + await page.locator('input[placeholder="New email address"]').fill(moveTo); + await page.locator('input[placeholder="Confirm new email address"]').fill(moveTo); + await page.locator('input[placeholder="Your current password"]').fill(pw); + await page.locator('button.form-button', { hasText: 'Save' }).first().click(); + await page.waitForTimeout(3000); + return page.locator('.registration-content, body').first().innerText(); + }; + + sink.clear(); + const wrongText = await attempt('not the password'); + check(/password is not right/i.test(wrongText), + 'a session alone can no longer move the account to another address', + wrongText.replace(/\n/g, ' | ').slice(0, 110)); + check(sink.messages.length === 0, 'and nothing is emailed for a refused attempt', + `${sink.messages.length} sent`); + + sink.clear(); + await attempt(np); + const toOld = await sink.waitFor(new RegExp(nEmail.replace('.', '\\.'), 'i'), 15000).catch(() => null); + check(Boolean(toOld), 'the real password moves it, and the address LOSING the account is told', + toOld ? toOld.headers.subject : 'never told'); + if (toOld) check(toOld.raw.includes(moveTo), 'and told where it is going'); + + // ---- a reset takes back the sessions that existed before it ---- + console.log('withdrawal'); + + const wUser = uniq(), wEmail = wUser + '@example.com'; + await signOut(ctx, page); + await registerAndVerify(page, sink, wUser, wEmail, GOOD); + + // Sign in and keep the token, standing in for another device still logged in. + await page.goto(`${BASE}/pages/login-page`, { waitUntil: 'networkidle' }); + await page.waitForSelector('input'); + await page.locator('input').nth(0).fill(wUser); + await page.locator('input').nth(1).fill(GOOD); + await page.locator('button.form-button').click(); + await page.waitForTimeout(3500); + const otherDevice = await page.evaluate(() => { + const c = window.cljs.core; + return c.get_in(window.re_frame.db.app_db.state, + c.PersistentVector.fromArray([c.keyword(null, 'user-data'), c.keyword(null, 'token')], true)); + }); + const useToken = (t) => page.evaluate(async ([b, tok]) => + (await fetch(b + '/user', { headers: { Authorization: `Token ${tok}` } })).status, [BASE, t]); + check(await useToken(otherDevice) === 200, 'a signed-in session works before the reset'); + + // Reset the password from somewhere else entirely. + await signOut(ctx, page); + const wLink = await resetLinkFor(page, sink, wEmail); + await page.goto(wLink, { waitUntil: 'networkidle' }); + await page.waitForSelector('.auth-form input'); + const wNew = GOOD + ' lantern'; + const wb = authInputs(page); + await wb.nth(0).fill(wNew); await wb.nth(1).fill(wNew); + await press(page, 'SUBMIT'); + await page.waitForTimeout(3500); + + check(await useToken(otherDevice) === 401, + 'and is refused afterwards, so a reset takes the account back', + `HTTP ${await useToken(otherDevice)}`); + + // And signing in again works normally -- this withdraws tokens, it does not + // limit how many places somebody may be signed in. + await page.goto(`${BASE}/pages/login-page`, { waitUntil: 'networkidle' }); + await page.waitForSelector('input'); + await page.locator('input').nth(0).fill(wUser); + await page.locator('input').nth(1).fill(wNew); + await page.locator('button.form-button').click(); + await page.waitForTimeout(3500); + const reissued = await page.evaluate(() => { + const c = window.cljs.core; + return c.get_in(window.re_frame.db.app_db.state, + c.PersistentVector.fromArray([c.keyword(null, 'user-data'), c.keyword(null, 'token')], true)); + }); + check(Boolean(reissued) && await useToken(reissued) === 200, + 'while a fresh sign-in works, on as many devices as you like'); + + // Sign out everywhere, without changing the password. + await page.goto(`${BASE}/pages/login-page`, { waitUntil: 'networkidle' }); + await page.waitForSelector('input'); + await page.locator('input').nth(0).fill(wUser); + await page.locator('input').nth(1).fill(wNew); + await page.locator('button.form-button').click(); + await page.waitForTimeout(3500); + const deviceA = await page.evaluate(() => { + const c = window.cljs.core; + return c.get_in(window.re_frame.db.app_db.state, + c.PersistentVector.fromArray([c.keyword(null, 'user-data'), c.keyword(null, 'token')], true)); + }); + const bye = await page.evaluate(async ([b, t]) => + (await fetch(b + '/user/sessions', { method: 'DELETE', headers: { Authorization: `Token ${t}` } })).status, + [BASE, deviceA]); + check(bye === 200, 'sign out everywhere answers', `HTTP ${bye}`); + check(await useToken(deviceA) === 401, + 'and the session that asked for it is signed out too', `HTTP ${await useToken(deviceA)}`); + check(await useToken(reissued) === 401, 'along with every other session'); + + // The password did NOT change: it still signs in. + await page.goto(`${BASE}/pages/login-page`, { waitUntil: 'networkidle' }); + await page.waitForSelector('input'); + await page.locator('input').nth(0).fill(wUser); + await page.locator('input').nth(1).fill(wNew); + await page.locator('button.form-button').click(); + await page.waitForTimeout(3500); + check(!page.url().includes('login-page'), + 'and the password still works, because signing out is not changing it', + page.url().replace(BASE, '')); + + check(errors.length === 0, 'no page errors anywhere in the run', errors.slice(0, 3).join(' | ')); + + await browser.close(); + await sink.stop(); + console.log(failures() ? `\n${failures()} FAILED` : '\nall passed'); + process.exit(failures() ? 1 : 0); +})(); diff --git a/scripts/e2e/export-character-pdf.js b/scripts/e2e/export-character-pdf.js new file mode 100644 index 000000000..3b4830f21 --- /dev/null +++ b/scripts/e2e/export-character-pdf.js @@ -0,0 +1,402 @@ +// Browser end-to-end check of the PDF export. +// +// ./scripts/e2e/run.sh export-character-pdf.js +// +// Builds characters through the real builder and exports each one, so the whole +// path runs: pdf_spec assembling the field map in the browser, the form POST, +// then routes.clj and pdf.clj. The scenarios differ in spellcasting classes, +// which is what decides how many pages the sheet needs. +// +// The PDFs land in $E2E_OUT; run.sh then inspects them with PDFBox, which is the +// only place field names can be read (they sit in compressed object streams). +// +// Console output is collected throughout and any error or warning fails the run. + +const { chromium } = require('playwright'); +const fs = require('fs'); +const path = require('path'); + +const BASE = process.env.E2E_BASE || 'http://localhost:8890'; +const OUT = process.env.E2E_OUT || '/tmp/e2e-pdf'; +// The bundled Chromium build can be older than this playwright release expects, +// so point at what is installed rather than letting it look for a download. +const EXECUTABLE = process.env.E2E_CHROMIUM + || ['/opt/pw-browsers/chromium-1194/chrome-linux/chrome', + '/opt/pw-browsers/chromium/chrome-linux/chrome'].find(p => fs.existsSync(p)); + +const noise = [ + /favicon/i, + // No outbound network in the sandbox, so the webfont never loads. That is the + // environment, not the app; the page renders in its fallback stack. + /fonts\.googleapis\.com/i, + /ERR_CONNECTION_RESET/i, +]; + +// A character worth exporting, and what its sheet should need. pdf_spec emits a +// spellcasting section per class that HAS SPELLS, so a caster with none chosen +// produces no spell page -- hence `spells` on the casting scenarios. +const SCENARIOS = [ + { name: 'barbarian', classes: [['Barbarian', '20']], spells: 0, minPages: 3, + note: 'no spellcasting, so no spell pages' }, + { name: 'wizard-20', classes: [['Wizard', '20']], spells: 8, minPages: 4, + note: 'one caster, spells spread across its list so several level boxes fill' }, + { name: 'multiclass', classes: [['Wizard', '5'], ['Cleric', '5'], ['Sorcerer', '5']], + spells: 3, minPages: 6, abilities: true, + note: 'three casting classes, each picked from its own block: a spellcasting ' + + 'section per caster, and a class whose list outgrows one page takes another' }, +]; + +function record(page, sink, label) { + page.on('console', msg => { + const type = msg.type(); + if (type !== 'error' && type !== 'warning') return; + const text = msg.text(); + if (noise.some(re => re.test(text))) return; + sink.push(`${label}: console.${type}: ${text}`); + }); + page.on('pageerror', err => sink.push(`${label}: pageerror: ${err.message}`)); + page.on('requestfailed', req => { + if (noise.some(re => re.test(req.url()))) return; + sink.push(`${label}: request failed: ${req.url()} (${req.failure()?.errorText})`); + }); +} + +// The builder re-renders on every change and shows no loading indicator, so the +// only signal that a change has landed is the DOM going quiet. Waiting for that +// beats a fixed sleep both ways: it returns as soon as the render finishes, and +// it still waits on a machine slower than the sleep was tuned for. +async function settle(page, quiet = 300, cap = 8000) { + await page.evaluate(([quiet, cap]) => new Promise(resolve => { + let timer = setTimeout(done, quiet); + const ceiling = setTimeout(done, cap); + const observer = new MutationObserver(() => { + clearTimeout(timer); + timer = setTimeout(done, quiet); + }); + function done() { + clearTimeout(timer); + clearTimeout(ceiling); + observer.disconnect(); + resolve(); + } + observer.observe(document.body, { childList: true, subtree: true, attributes: true }); + }), [quiet, cap]).catch(() => {}); +} + +// Several nodes carry the same label and some are hidden -- the mobile layout +// renders its own copy -- so .first() often lands on one nothing can click. +async function firstVisible(scope, selector) { + const all = scope.locator(selector); + const count = await all.count(); + for (let i = 0; i < count; i++) { + const el = all.nth(i); + if (await el.isVisible()) return el; + } + return null; +} + +async function clickVisible(scope, selector) { + const el = await firstVisible(scope, selector); + if (el) await el.click(); + return Boolean(el); +} + +// The builder's step tabs sit in the left options panel; the same words appear in +// the top navigation, so pick by position rather than by text alone. +async function openBuilderTab(page, label) { + // The tab row is near the top of the options panel, and these helpers scroll + // down to reach fields, so return there before looking for it. + await page.evaluate(() => window.scrollTo(0, 0)); + await page.waitForTimeout(400); + const all = page.locator(`text=${label}`); + for (let i = 0; i < await all.count(); i++) { + const el = all.nth(i); + if (!(await el.isVisible())) continue; + const box = await el.boundingBox(); + if (box && box.x < 650 && box.y > 400) { await el.click(); return true; } + } + return false; +} + +// Class rows gain extra dropdowns as levels rise -- a Wizard at 5 picks an +// Arcane Tradition -- so the class and level selects cannot be found by position. +// Classify them by what they contain instead: a class select offers class names, +// a level select offers only numbers. +const CLASS_NAMES = ['Barbarian', 'Bard', 'Cleric', 'Druid', 'Fighter', 'Monk', + 'Paladin', 'Ranger', 'Rogue', 'Sorcerer', 'Warlock', 'Wizard']; + +async function classifySelects(page) { + const selects = page.locator('select'); + const classSelects = []; + const levelSelects = []; + for (let i = 0; i < await selects.count(); i++) { + const el = selects.nth(i); + if (!(await el.isVisible())) continue; + const options = (await el.locator('option').allTextContents()).map(o => o.trim()); + // A second class row omits the classes already taken, so no single name is + // guaranteed -- count how many of them appear instead. + const classish = options.filter(o => CLASS_NAMES.includes(o)).length; + if (classish >= 3) classSelects.push(el); + else if (options.length > 1 && options.every(o => /^\d+$/.test(o))) levelSelects.push(el); + } + return { classSelects, levelSelects }; +} + +// classifySelects reads the DOM as it stands, and a click that adds a class row +// only produces its selects a render later -- a quiet DOM is not enough, because +// re-frame leaves a gap before it re-renders. Poll for the selects themselves. +async function waitForSelects(page, wanted, timeout = 15000) { + const deadline = Date.now() + timeout; + let sets = await classifySelects(page); + while (!wanted(sets) && Date.now() < deadline) { + await page.waitForTimeout(150); + sets = await classifySelects(page); + } + return sets; +} + +// Multiclassing into a caster requires the relevant mental score at 13, and the +// default array leaves WIS at 10 and CHA at 8 -- so every caster is refused. +// Manual Entry exposes the six base scores as editable inputs. +async function raiseAbilities(page, check) { + check(await openBuilderTab(page, 'Ability Scores'), 'the builder has an Ability Scores step'); + await settle(page); + check(await clickVisible(page, 'text=Manual Entry'), 'abilities can be entered by hand'); + await settle(page); + + // The first six number inputs are the base scores, in STR DEX CON INT WIS CHA + // order; the rest are the computed totals and are read-only. + const inputs = page.locator('input[type=number]'); + let set = 0; + for (let i = 0; i < await inputs.count() && set < 6; i++) { + const el = inputs.nth(i); + if (!(await el.isVisible())) continue; + await el.fill('15'); + set += 1; + await page.waitForTimeout(200); + } + check(set === 6, `set ${set} ability scores to 15`); + await settle(page); +} + +async function setClasses(page, classes, check) { + await openBuilderTab(page, 'Class / Level'); + await settle(page); + + for (let i = 0; i < classes.length; i++) { + if (i > 0) { + // Adds the first class not already taken; its dropdown is then re-pointed. + const added = await clickVisible(page, 'text=Add Levels in Another Class'); + check(added, `added a slot for class ${i + 1}`); + } + const [className, level] = classes[i]; + let { classSelects } = await waitForSelects(page, s => s.classSelects.length > i); + check(classSelects.length > i, `class ${i + 1} has a class dropdown`); + if (classSelects.length <= i) return; + + // Multiclass options carry their prerequisite in the label -- "Cleric + // (Requires WIS 13 or higher)" -- so match on the leading class name. + const classOptions = (await classSelects[i].locator('option').allTextContents()) + .map(o => o.trim()); + const optionIndex = classOptions.findIndex(o => o.startsWith(className)); + check(optionIndex >= 0, + `${className} is offered for class ${i + 1}` + + (optionIndex >= 0 ? '' : ` (offered: ${classOptions.join(', ')})`)); + await classSelects[i].selectOption({ index: optionIndex, timeout: 10000 }); + + // Changing the class re-renders the row and its level dropdown. + const { levelSelects } = await waitForSelects(page, s => s.levelSelects.length > i); + check(levelSelects.length > i, `class ${i + 1} has a level dropdown`); + if (levelSelects.length <= i) return; + const levelOptions = (await levelSelects[i].locator('option').allTextContents()) + .map(o => o.trim()); + check(levelOptions.includes(level), + `level ${level} is offered for class ${i + 1}` + + (levelOptions.includes(level) ? '' : ` (offered up to ${levelOptions.slice(-1)})`)); + await levelSelects[i].selectOption({ label: level, timeout: 10000 }); + await settle(page); + } +} + +// Every caster gets its own " Cantrips Known" / " Spells Known" +// block, each a div.p-5.m-b-20.m-b-0-last holding one heading and its rows. +// Walking the page top-down fills only the blocks it reaches first, which leaves +// later casters with no spells -- and pdf_spec emits a spellcasting section only +// for a class that HAS spells, so those casters get no page at all. Picking from +// each block is what makes a multiclass sheet carry one section per caster. +// +// Spell rows are not real checkboxes -- the square is styled markup -- so the row +// label is what gets clicked. Its class distinguishes it from, say, the "Light +// Theme" toggle, which a text match on "Light" would otherwise hit. +async function blockTitles(page) { + return page.evaluate(() => + Array.from(document.querySelectorAll('span.m-l-5.f-s-18.f-w-b.flex-grow-1')) + .map(el => el.textContent.trim())); +} + +// Picks one row from the named block, skipping labels already taken. Addressing +// the block by its heading rather than by index survives the re-render each +// selection causes, which reorders and re-length the lists. +// `spread` walks the list end to end instead of taking the first few, so the +// higher spell levels get chosen too and more than one level box is filled. +async function pickInBlock(page, title, taken, spread) { + return page.evaluate(([title, taken, spread]) => { + const head = Array.from(document.querySelectorAll('span.m-l-5.f-s-18.f-w-b.flex-grow-1')) + .find(h => h.textContent.trim() === title); + if (!head) return null; + const block = head.parentElement.parentElement; + const rows = Array.from(block.querySelectorAll('span.f-w-b.f-s-1.flex-grow-1')) + .filter(r => !taken.includes(r.textContent.trim())); + if (!rows.length) return null; + const row = rows[Math.min(rows.length - 1, Math.floor(spread * rows.length))]; + const label = row.textContent.trim(); + row.click(); + return label; + }, [title, taken, spread]); +} + +async function selectSpells(page, perBlock, check) { + check(await openBuilderTab(page, 'Spells'), 'the builder has a Spells step'); + await settle(page); + + const titles = await blockTitles(page); + check(titles.length > 0, `the Spells step offers ${titles.length} block(s)`); + + const filled = []; + for (const title of titles) { + const taken = []; + for (let i = 0; i < perBlock; i++) { + const label = await pickInBlock(page, title, taken, i / perBlock); + if (!label) break; + taken.push(label); + await settle(page, 150, 3000); + } + filled.push(`${title.replace(' Known', '')} ${taken.length}`); + } + check(filled.every(f => !f.endsWith(' 0')), + `chose spells in every block - ${filled.join(', ')}`); +} + +async function exportPdf(page, context, check) { + let pdfBytes = null; + // Chromium hands a PDF response to its internal viewer, so response.body() + // returns the viewer's wrapper HTML rather than the document. Intercepting the + // route runs the page's own request and yields the real bytes. + const handler = async route => { + const res = await route.fetch(); + pdfBytes = await res.body(); + await route.fulfill({ response: res }); + }; + await context.route('**/character.pdf', handler); + + check(await clickVisible(page, 'text=Export'), 'the builder offers an Export control'); + await settle(page); + + // "Create PDF" stays disabled until a sheet is chosen: print-button-enabled in + // views.cljs gates on print-character-sheet-style? being set. + const styles = await firstVisible(page, 'select'); + check(styles !== null, 'the panel offers a character sheet dropdown'); + await styles.selectOption({ index: 1 }); + await page.waitForTimeout(500); + + const createPdf = await firstVisible(page, 'button:has-text("Create PDF")'); + check(createPdf !== null, 'the export panel offers a clickable "Create PDF"'); + check(await createPdf.isEnabled(), 'and it is enabled once a sheet is chosen'); + + // The sticky header overlays whatever playwright scrolls to. + await createPdf.evaluate(el => el.scrollIntoView({ block: 'center' })); + await page.waitForTimeout(500); + + const [response] = await Promise.all([ + context.waitForEvent('response', + { predicate: r => r.url().includes('character.pdf'), timeout: 90000 }), + createPdf.click({ timeout: 10000 }).catch(() => createPdf.click({ force: true })), + ]); + check(response.status() === 200, `the export answers 200 (got ${response.status()})`); + // the route handler fills pdfBytes on its own schedule, which the response + // event does not wait for -- so wait for the bytes, not for a duration + for (let i = 0; i < 200 && pdfBytes === null; i++) await page.waitForTimeout(50); + await context.unroute('**/character.pdf', handler); + return pdfBytes; +} + +// Exported so a throwaway probe script can drive the builder the same way this +// does, rather than keeping its own copy that drifts. run.sh takes a script name, +// so a probe is `./scripts/e2e/run.sh probe.js`. +module.exports = { settle, firstVisible, clickVisible, openBuilderTab, + classifySelects, waitForSelects, raiseAbilities, setClasses, selectSpells, + blockTitles, pickInBlock, + exportPdf, SCENARIOS, BASE, OUT, EXECUTABLE }; + +if (require.main !== module) return; + +(async () => { + fs.mkdirSync(OUT, { recursive: true }); + if (!EXECUTABLE) { + console.error('no chromium found under /opt/pw-browsers; set E2E_CHROMIUM'); + process.exit(1); + } + const problems = []; + const failures = []; + const check = (ok, message) => { + console.log(` ${ok ? 'ok ' : 'FAIL'} ${message}`); + if (!ok) failures.push(message); + }; + + const browser = await chromium.launch({ executablePath: EXECUTABLE }); + const context = await browser.newContext({ acceptDownloads: true }); + // The cookie banner and the What's New panel (since 2026-09-06) sit over the builder and take + // every click. Mark both seen before any page loads, as api-sub-loaders.js does. + await context.addInitScript(() => { + try { + localStorage.setItem('orcpub:no-cookie-banner', '1'); + localStorage.setItem('whats-new-seen', JSON.stringify('summer-patch-2026')); + } catch (e) {} + }); + + for (const scenario of SCENARIOS) { + console.log(`\n${scenario.name}: ${scenario.classes.map(c => c.join(' ')).join(' / ')}`); + console.log(` (${scenario.note})`); + const page = await context.newPage(); + record(page, problems, scenario.name); + try { + await page.goto(`${BASE}/pages/dnd/5e/character-builder`, + { waitUntil: 'networkidle', timeout: 60000 }); + await settle(page); + await clickVisible(page, 'text=Got it!'); // cookie banner covers the controls + + if (scenario.abilities) await raiseAbilities(page, check); + await setClasses(page, scenario.classes, check); + if (scenario.spells > 0) await selectSpells(page, scenario.spells, check); + await page.screenshot({ path: path.join(OUT, `${scenario.name}-built.png`) }); + + const bytes = await exportPdf(page, context, check); + check(bytes !== null && bytes.length > 50000, + `the export came back (${bytes ? Math.round(bytes.length / 1024) : 0} KB)`); + if (bytes) { + check(bytes.slice(0, 5).toString() === '%PDF-', 'and is a PDF'); + fs.writeFileSync(path.join(OUT, `${scenario.name}.pdf`), bytes); + // run.sh inspects each PDF with PDFBox; the expected page count travels + // alongside so it can be asserted there. + fs.writeFileSync(path.join(OUT, `${scenario.name}.min-pages`), + String(scenario.minPages)); + } + } catch (err) { + failures.push(`${scenario.name} threw: ${err.message.split('\n')[0]}`); + await page.screenshot({ path: path.join(OUT, `${scenario.name}-failure.png`) }) + .catch(() => {}); + } + await page.close(); + } + + await browser.close(); + + console.log('\nconsole output while signed out:'); + console.log(problems.length ? problems.map(p => ` ${p}`).join('\n') : ' none'); + if (failures.length) { + console.log('\nfailures:'); + failures.forEach(f => console.log(` ${f}`)); + } + console.log(`\n${failures.length} check(s) failed, ${problems.length} console problem(s)`); + process.exit(failures.length + problems.length === 0 ? 0 : 1); +})(); diff --git a/scripts/e2e/lib.js b/scripts/e2e/lib.js new file mode 100644 index 000000000..c28f16ac5 --- /dev/null +++ b/scripts/e2e/lib.js @@ -0,0 +1,100 @@ +// Helpers the scripts/e2e suites share: a browser context with the overlays out of the way, logging in, +// watching share requests, pressing and reading the share buttons, and the homebrew the share suites seed. +const BASE = process.env.E2E_BASE || 'http://localhost:8890'; +const EXECUTABLE = process.env.E2E_CHROMIUM || undefined; + +// The homebrew language e2e-boot's E2E_HOMEBREW_CHARACTER_ID character knows. The server never has it; +// a suite puts it in the owner's local library. +const HOMEBREW = { + source: 'E2E Tongues', + language: 'E2E Cant', + library: description => + `{"E2E Tongues" {:orcpub.dnd.e5/languages {:e2e-cant {:name "E2E Cant" :key :e2e-cant :option-pack "E2E Tongues" :description "${description}"}}}}`, +}; + +// A context whose pages skip the cookie banner and What's New panel, keep what Copy link writes in +// window.__copied, and start with `library` in local storage when given. +async function newContext(browser, library) { + const ctx = await browser.newContext(); + await ctx.addInitScript(lib => { + try { + localStorage.setItem('orcpub:no-cookie-banner', '1'); + localStorage.setItem('whats-new-seen', JSON.stringify('summer-patch-2026')); + if (lib && !localStorage.getItem('plugins')) localStorage.setItem('plugins', lib); + } catch (e) {} + Object.defineProperty(navigator, 'clipboard', { + configurable: true, + value: { writeText: text => { window.__copied = text; return Promise.resolve(); } }, + }); + }, library || null); + return ctx; +} + +async function login(page, user, pass) { + await page.goto(`${BASE}/pages/login-page`, { waitUntil: 'domcontentloaded' }); + await page.waitForSelector('input'); + await page.locator('input').nth(0).fill(user); + await page.locator('input').nth(1).fill(pass); + await page.locator('button.form-button').click(); + await page.waitForTimeout(3500); +} + +// Every request to a share route this page makes, as "METHOD status". +function watchShares(page) { + const seen = []; + page.on('response', r => { + if (new URL(r.url()).pathname.includes('/shares/')) seen.push(`${r.request().method()} ${r.status()}`); + }); + return seen; +} + +async function waitForButton(page, text) { + await page.waitForFunction( + t => [...document.querySelectorAll('button')].some(b => b.textContent.includes(t) && !b.disabled), + text, { timeout: 60000 }); +} + +async function copyLink(page) { + await waitForButton(page, 'Copy link'); + await page.evaluate(() => { window.__copied = ''; }); + await page.locator('button', { hasText: 'Copy link' }).first().click(); + await page.waitForTimeout(300); + return page.evaluate(() => window.__copied || ''); +} + +// The sources in the page's shared-homebrew overlay, and the seeded language's description there. +async function sharedHomebrew(page) { + return page.evaluate(src => { + const c = window.cljs.core; + const shared = c.get(window.re_frame.db.app_db.state, c.keyword(null, 'shared-plugins')); + const path = c.PersistentVector.fromArray( + [src, c.keyword('orcpub.dnd.e5', 'languages'), c.keyword(null, 'e2e-cant'), c.keyword(null, 'description')], true); + return { + sources: shared ? c.clj__GT_js(c.vec(c.keys(shared))) : [], + description: shared ? c.get_in(shared, path) : null, + }; + }, HOMEBREW.source); +} + +// The status a signed-in page gets from character `id`'s share-token route: 200 when it is shared, 404 +// when not, 410 when its last link expired and the owner has not acted on that. +async function tokenStatus(page, id) { + return page.evaluate(async url => { + const c = window.cljs.core; + const login = c.get_in(window.re_frame.db.app_db.state, + c.PersistentVector.fromArray([c.keyword(null, 'user-data'), c.keyword(null, 'token')], true)); + return (await fetch(url, { headers: { Authorization: `Token ${login}` } })).status; + }, `${BASE}/dnd/5e/characters/${id}/share-token`); +} + +function checker() { + let failures = 0; + const check = (ok, label, detail) => { + if (!ok) failures++; + console.log(` ${ok ? 'PASS' : 'FAIL'} ${label}${detail ? ' -- ' + detail : ''}`); + }; + return { check, failures: () => failures }; +} + +module.exports = { BASE, EXECUTABLE, HOMEBREW, newContext, login, watchShares, waitForButton, copyLink, + sharedHomebrew, tokenStatus, checker }; diff --git a/scripts/e2e/lib/mail-sink.js b/scripts/e2e/lib/mail-sink.js new file mode 100644 index 000000000..cbeb32a45 --- /dev/null +++ b/scripts/e2e/lib/mail-sink.js @@ -0,0 +1,120 @@ +// A throwaway SMTP server that accepts anything and writes what it was given to +// a JSON file. Dependency-free on purpose: it is under a hundred lines of the +// protocol and adding a package to run the tests is how test setups rot. +// +// It exists because the e2e harness had no mail server, so registration, +// verification and password reset could not be driven end to end AT ALL -- the +// verification key and the reset key only ever exist inside an email. That is +// why those flows were never covered, and why a broken reset form reached a +// merge. Point the app at this and the keys become readable: +// +// EMAIL_SERVER_URL=127.0.0.1 EMAIL_SERVER_PORT=2525 +// +// Usage: +// const sink = await startMailSink({ port: 2525, file: '/tmp/mail.json' }); +// ... drive the app ... +// const msg = await sink.waitFor(/verification/i); +// sink.stop(); + +const net = require('net'); +const fs = require('fs'); + +function parseMessage(raw) { + const split = raw.indexOf('\r\n\r\n'); + const head = split === -1 ? raw : raw.slice(0, split); + const body = split === -1 ? '' : raw.slice(split + 4); + const headers = {}; + // Unfold continuation lines before splitting on the colon. + for (const line of head.replace(/\r\n[ \t]+/g, ' ').split('\r\n')) { + const i = line.indexOf(':'); + if (i > 0) headers[line.slice(0, i).toLowerCase().trim()] = line.slice(i + 1).trim(); + } + return { headers, body, raw }; +} + +function startMailSink({ port = 2525, file = null } = {}) { + const messages = []; + const waiters = []; + + const deliver = (msg) => { + messages.push(msg); + if (file) fs.writeFileSync(file, JSON.stringify(messages, null, 2)); + for (let i = waiters.length - 1; i >= 0; i--) { + if (waiters[i].match(msg)) { waiters[i].resolve(msg); waiters.splice(i, 1); } + } + }; + + const server = net.createServer((sock) => { + let buf = ''; + let inData = false; + let data = ''; + const say = (s) => sock.write(s + '\r\n'); + say('220 mail-sink ready'); + + sock.on('data', (chunk) => { + buf += chunk.toString('utf8'); + let nl; + while ((nl = buf.indexOf('\r\n')) !== -1) { + const line = buf.slice(0, nl); + buf = buf.slice(nl + 2); + + if (inData) { + // A lone dot ends the message; a leading dot on any other line is stuffed. + if (line === '.') { + inData = false; + deliver(parseMessage(data)); + data = ''; + say('250 2.0.0 Ok: queued'); + } else { + data += (line.startsWith('..') ? line.slice(1) : line) + '\r\n'; + } + continue; + } + + const verb = line.split(/\s+/)[0].toUpperCase(); + switch (verb) { + case 'EHLO': + // No STARTTLS and no AUTH advertised: the client then sends neither, + // which is what keeps this short. + say('250-mail-sink'); + say('250 8BITMIME'); + break; + case 'HELO': say('250 mail-sink'); break; + case 'MAIL': + case 'RCPT': say('250 2.1.0 Ok'); break; + case 'AUTH': say('235 2.7.0 Accepted'); break; + case 'RSET': say('250 2.0.0 Ok'); break; + case 'NOOP': say('250 2.0.0 Ok'); break; + case 'DATA': inData = true; say('354 End data with .'); break; + case 'QUIT': say('221 2.0.0 Bye'); sock.end(); break; + default: say('250 2.0.0 Ok'); break; + } + } + }); + sock.on('error', () => {}); + }); + + return new Promise((resolve, reject) => { + server.once('error', reject); + server.listen(port, '127.0.0.1', () => resolve({ + port, + messages, + clear: () => { messages.length = 0; if (file) fs.writeFileSync(file, '[]'); }, + // Resolves with the first message already held or next delivered that matches. + waitFor: (re, timeoutMs = 15000) => new Promise((res, rej) => { + const match = (m) => re.test(m.raw); + const already = messages.find(match); + if (already) return res(already); + const w = { match, resolve: res }; + waiters.push(w); + setTimeout(() => { + const i = waiters.indexOf(w); + if (i !== -1) { waiters.splice(i, 1); rej(new Error(`no mail matching ${re} in ${timeoutMs}ms`)); } + }, timeoutMs); + }), + stop: () => new Promise((res) => server.close(res)), + })); + }); +} + +module.exports = { startMailSink }; diff --git a/scripts/e2e/move-homebrew-keeps-dependents.js b/scripts/e2e/move-homebrew-keeps-dependents.js new file mode 100644 index 000000000..e00aead5f --- /dev/null +++ b/scripts/e2e/move-homebrew-keeps-dependents.js @@ -0,0 +1,97 @@ +// Does moving a class into a source that already has its key take the class's subclasses with it? +// +// ./scripts/e2e/run.sh move-homebrew-keeps-dependents.js +// +// Two sources each hold a class keyed :e2e-artificer, each with a subclass of its own. Moving Alpha's +// class into Beta through My Content's Move / copy bar has to rename it, since Beta already holds that +// key. Alpha's subclass must follow the renamed class, Beta's must stay on Beta's class, and the moved +// class must remember its old key. Before the fix (Greptile, PR #34) the rename left Alpha's subclass on +// :e2e-artificer, where it attached to Beta's class. The rules themselves are in orcbrew_validation_test. +// Needs no account: My Content is the browser's local library. +const { chromium } = require('playwright'); +const { BASE, EXECUTABLE, newContext, checker } = require('./lib'); + +const ALPHA = 'E2E Alpha'; +const BETA = 'E2E Beta'; +const LIBRARY = + `{"${ALPHA}" {:orcpub.dnd.e5/classes {:e2e-artificer {:name "E2E Artificer" :key :e2e-artificer :option-pack "${ALPHA}"}}` + + ` :orcpub.dnd.e5/subclasses {:e2e-alchemist {:name "E2E Alchemist" :key :e2e-alchemist :class :e2e-artificer :option-pack "${ALPHA}"}}}` + + ` "${BETA}" {:orcpub.dnd.e5/classes {:e2e-artificer {:name "E2E Artificer" :key :e2e-artificer :option-pack "${BETA}"}}` + + ` :orcpub.dnd.e5/subclasses {:e2e-armorer {:name "E2E Armorer" :key :e2e-armorer :class :e2e-artificer :option-pack "${BETA}"}}}}`; + +// Every class and subclass in the page's library, as {source, type, key, name, class, formerKeys}. +const library = page => page.evaluate(() => { + const c = window.cljs.core; + const kw = (ns, n) => c.keyword(ns, n); + const plugins = c.get(window.re_frame.db.app_db.state, kw(null, 'plugins')); + const rows = []; + c.run_BANG_(entry => { + const source = c.key(entry), plugin = c.val(entry); + for (const type of ['classes', 'subclasses']) { + c.run_BANG_(e => { const k = c.key(e), item = c.val(e); rows.push({ + source, type, key: c.name(k), + name: c.get(item, kw(null, 'name')), + class: (cls => cls ? c.name(cls) : null)(c.get(item, kw(null, 'class'))), + formerKeys: c.clj__GT_js(c.map(c.name, c.get(item, kw(null, 'former-keys')))), + }); }, c.get(plugin, kw('orcpub.dnd.e5', type))); + } + }, plugins); + return rows; +}); + +(async () => { + const { check, failures } = checker(); + const browser = await chromium.launch({ executablePath: EXECUTABLE }); + const ctx = await newContext(browser, LIBRARY); + const page = await ctx.newPage(); + const errors = []; + page.on('pageerror', e => errors.push(String(e))); + + await page.goto(`${BASE}/dnd/5e/my-content`, { waitUntil: 'networkidle', timeout: 120000 }); + await page.waitForTimeout(2000); + const before = await library(page); + check(before.filter(r => r.type === 'classes').length === 2, 'both sources load their class', JSON.stringify(before)); + + await page.locator('button.b-move').click(); + // Sources and the content types inside them render collapsed, each with its own "expand" link. + const row = page.locator('.mc-selrow', { hasText: 'E2E Artificer' }).first(); + for (let i = 0; i < 10 && !(await row.isVisible().catch(() => false)); i++) { + const expand = page.locator('span.underline', { hasText: /^expand$/ }).first(); + if (!(await expand.isVisible().catch(() => false))) break; + await expand.click(); + await page.waitForTimeout(300); + } + await page.locator('.mc-selrow', { hasText: 'E2E Artificer' }).first().click(); + await page.locator('select').filter({ has: page.locator(`option[value="${BETA}"]`) }).first().selectOption(BETA); + await page.locator('button', { hasText: 'Move here' }).click(); + await page.waitForTimeout(1500); + + const after = await library(page); + const classes = after.filter(r => r.type === 'classes'); + const moved = classes.find(r => r.source === BETA && r.key !== 'e2e-artificer'); + const betaOwn = classes.find(r => r.source === BETA && r.key === 'e2e-artificer'); + const alchemist = after.find(r => r.key === 'e2e-alchemist'); + const armorer = after.find(r => r.key === 'e2e-armorer'); + + check(!classes.some(r => r.source === ALPHA), 'the class left Alpha', JSON.stringify(classes)); + check(!!moved, 'it arrived in Beta under a new key', JSON.stringify(classes)); + check(!!betaOwn && betaOwn.name === 'E2E Artificer', "Beta's own class is untouched", JSON.stringify(betaOwn)); + check(!!moved && alchemist && alchemist.class === moved.key, + "Alpha's subclass follows the moved class", JSON.stringify({ alchemist, moved: moved && moved.key })); + check(!!armorer && armorer.class === 'e2e-artificer', "Beta's subclass stays on Beta's class", JSON.stringify(armorer)); + check(!!moved && (moved.formerKeys || []).includes('e2e-artificer'), + 'the moved class remembers its old key', JSON.stringify(moved)); + + // The move persists like any edit: a reload reads it back from local storage. + await page.reload({ waitUntil: 'networkidle' }); + await page.waitForTimeout(2000); + const reloaded = await library(page); + const alchemistAgain = reloaded.find(r => r.key === 'e2e-alchemist'); + check(!!moved && alchemistAgain && alchemistAgain.class === moved.key, 'and it survives a reload', + JSON.stringify(alchemistAgain)); + check(errors.length === 0, 'no uncaught page errors', errors.join(' | ')); + + await browser.close(); + console.log(failures() ? `\nFAILURES: ${failures()}` : '\nall checks passed'); + process.exit(failures() ? 1 : 0); +})(); diff --git a/scripts/e2e/package-lock.json b/scripts/e2e/package-lock.json new file mode 100644 index 000000000..5cb5da3f7 --- /dev/null +++ b/scripts/e2e/package-lock.json @@ -0,0 +1,40 @@ +{ + "name": "orcpub-e2e", + "lockfileVersion": 3, + "requires": true, + "packages": { + "": { + "name": "orcpub-e2e", + "dependencies": { + "playwright": "^1.62.0" + } + }, + "node_modules/playwright": { + "version": "1.63.0", + "resolved": "https://registry.npmjs.org/playwright/-/playwright-1.63.0.tgz", + "integrity": "sha512-+7ziBLidS4NaNCdt57SUDT+wYmmd5fmiQejUic/kb+YsYSCPyOOE9sebzMjNmQrsnNpDJqd4WHvV/8lfKfUDUg==", + "license": "Apache-2.0", + "dependencies": { + "playwright-core": "1.63.0" + }, + "bin": { + "playwright": "cli.js" + }, + "engines": { + "node": ">=20" + } + }, + "node_modules/playwright-core": { + "version": "1.63.0", + "resolved": "https://registry.npmjs.org/playwright-core/-/playwright-core-1.63.0.tgz", + "integrity": "sha512-rYCsBF/M5HjUch52bbtVONEFjv6Xu8sm8h72dNlR5bzIE1fvC/bxgspzkjSfU+MweEMmPM8KJebG6nnyxo5mCg==", + "license": "Apache-2.0", + "bin": { + "playwright-core": "cli.js" + }, + "engines": { + "node": ">=20" + } + } + } +} diff --git a/scripts/e2e/package.json b/scripts/e2e/package.json new file mode 100644 index 000000000..8641f3ef0 --- /dev/null +++ b/scripts/e2e/package.json @@ -0,0 +1,11 @@ +{ + "name": "orcpub-e2e", + "private": true, + "description": "Browser end-to-end checks. Chromium is expected to be already present; PLAYWRIGHT_SKIP_BROWSER_DOWNLOAD=1 keeps npm from fetching another copy.", + "scripts": { + "test": "node run.js" + }, + "dependencies": { + "playwright": "^1.62.0" + } +} diff --git a/scripts/e2e/party-keeps-shared-homebrew.js b/scripts/e2e/party-keeps-shared-homebrew.js new file mode 100644 index 000000000..a332b6af8 --- /dev/null +++ b/scripts/e2e/party-keeps-shared-homebrew.js @@ -0,0 +1,91 @@ +// Does a character added to a party from a share link keep showing its homebrew on the party page? +// +// ./scripts/e2e/run.sh party-keeps-shared-homebrew.js +// +// kaylee shares the seeded character that knows a homebrew language (E2E_HOMEBREW_CHARACTER_ID). zoe, +// signed in and without that homebrew, opens the link and presses Add to Party, which makes a new party. +// On zoe's parties page the party must list the share token for that character, and opening its row must +// load the homebrew. After kaylee presses New link, zoe's party must list no token and opening the row +// must load nothing. That only characters can join a party is covered by party_test.clj. +const { chromium } = require('playwright'); +const { BASE, EXECUTABLE, HOMEBREW, newContext, login, waitForButton, copyLink, sharedHomebrew, + checker } = require('./lib'); + +const ID = process.env.E2E_HOMEBREW_CHARACTER_ID; +const NAME = 'Wren Holloway'; + +// The share token zoe's parties list for the character, from the parties the page loaded. +const listedToken = page => page.evaluate(id => { + const c = window.cljs.core; + const parties = c.get(window.re_frame.db.app_db.state, c.keyword('orcpub.dnd.e5.character', 'parties')); + const tokens = []; + c.run_BANG_(party => c.run_BANG_(ch => { + if (c.get(ch, c.keyword(null, 'db/id')) === parseInt(id) || c.get(ch, c.keyword('db', 'id')) === parseInt(id)) { + tokens.push(c.get(ch, c.keyword('orcpub.party-share', 'token')) || null); + } + }, c.get(party, c.keyword('orcpub.dnd.e5.party', 'character-ids'))), parties || c.PersistentVector.EMPTY); + return tokens; +}, ID); + +async function openPartyRow(page) { + await page.goto(`${BASE}/pages/dnd/5e/parties`, { waitUntil: 'networkidle', timeout: 120000 }); + await page.waitForTimeout(2000); + const tokens = await listedToken(page); + await page.locator(`text=${NAME}`).first().click(); + await page.waitForTimeout(4000); + return { tokens, ...(await sharedHomebrew(page)) }; +} + +(async () => { + if (!ID) { + console.error('No E2E_HOMEBREW_CHARACTER_ID: run this through scripts/e2e/run.sh, which reads it from e2e-boot.'); + process.exit(2); + } + const { check, failures } = checker(); + const browser = await chromium.launch({ executablePath: EXECUTABLE }); + + console.log('kaylee shares her character:'); + const kaylee = await newContext(browser, HOMEBREW.library('Spoken only by test suites.')); + const owner = await kaylee.newPage(); + owner.on('dialog', d => d.accept()); + await login(owner, 'kaylee', 'serenity99'); + await owner.goto(`${BASE}/pages/dnd/5e/characters/${ID}`, { waitUntil: 'networkidle', timeout: 120000 }); + await waitForButton(owner, 'Share link'); + await owner.locator('button', { hasText: 'Share link' }).first().click(); + const link = await copyLink(owner); + check(/#s=[A-Za-z0-9_-]{22}$/.test(link), 'a short link', link); + + console.log('\nzoe opens it and adds the character to a party:'); + const zoe = await newContext(browser, null); + const page = await zoe.newPage(); + const errors = []; + page.on('pageerror', e => errors.push(e.message)); + await login(page, 'zoe', 'washburne7'); + await page.goto(link, { waitUntil: 'networkidle', timeout: 120000 }); + await page.waitForTimeout(4000); + check((await sharedHomebrew(page)).sources.includes(HOMEBREW.source), 'the link loaded the homebrew'); + const add = page.locator('button.form-button', { hasText: /^ADD$/ }); + check(await add.count() > 0, 'the page offers Add to Party'); + await add.first().click(); + await page.waitForTimeout(2500); + + console.log('\nzoe opens the party page:'); + const kept = await openPartyRow(page); + check(kept.tokens.length === 1 && kept.tokens[0] === link.split('#s=')[1], 'the party keeps the link\'s token', JSON.stringify(kept.tokens)); + check(kept.sources.includes(HOMEBREW.source), 'opening the row loads the homebrew', JSON.stringify(kept.sources)); + + console.log('\nkaylee presses New link, and zoe opens the party page again:'); + await copyLink(owner); + await owner.locator('button', { hasText: 'New link' }).first().click(); + await owner.waitForTimeout(3000); + const revoked = await openPartyRow(page); + check(revoked.tokens.length === 1 && revoked.tokens[0] === null, 'the party lists no token', JSON.stringify(revoked.tokens)); + check(revoked.sources.length === 0, 'opening the row loads nothing', JSON.stringify(revoked.sources)); + check(errors.length === 0, 'no uncaught page errors on zoe\'s pages', errors.join(' | ')); + + await kaylee.close(); + await zoe.close(); + await browser.close(); + console.log(failures() ? `\nFAILURES: ${failures()}` : '\nall checks passed'); + process.exit(failures() ? 1 : 0); +})(); diff --git a/scripts/e2e/run.sh b/scripts/e2e/run.sh new file mode 100755 index 000000000..50b167496 --- /dev/null +++ b/scripts/e2e/run.sh @@ -0,0 +1,197 @@ +#!/usr/bin/env bash +# Browser end-to-end checks against a real server and a real database. +# +# ./scripts/e2e/run.sh +# +# Boots the app on an in-memory Datomic, seeds two verified users, drives +# Chromium through the named suite in scripts/e2e/, then tears the server down. +# Run it with no suite to list them. +# +# The database is datomic:mem://, which only exists inside the JVM that +# created it -- that is why dev/e2e_boot.clj starts the server AND seeds the +# user in one process rather than shelling out to `lein run -m user`. +set -uo pipefail +cd "$(dirname "$0")/../.." + +# No default suite. A default named run.js let two unrelated suites claim the same file, and +# neither run said which one it had picked. Checked before the server boots, so a typo is free. +SUITE="${1:-}" +if [ -z "$SUITE" ] || [ ! -f "scripts/e2e/$SUITE" ]; then + [ -n "$SUITE" ] && echo "no such suite: scripts/e2e/$SUITE" + echo "usage: ./scripts/e2e/run.sh " + echo "suites:" + for f in scripts/e2e/*.js; do + b=$(basename "$f"); [ "$b" = "lib.js" ] && continue + echo " $b" + done + exit 2 +fi + +PORT="${E2E_PORT:-8890}" +LOG="${E2E_LOG:-/tmp/e2e-server.log}" + +# Resolved from scripts/e2e, which is where its package.json and node_modules +# live -- checking from the project root reported it missing while every suite +# could load it perfectly well. +if ! (cd "$(dirname "$0")" && node -e "require('playwright')") 2>/dev/null; then + echo "playwright is not installed. Run:" + echo " (cd scripts/e2e && PLAYWRIGHT_SKIP_BROWSER_DOWNLOAD=1 npm install)" + exit 1 +fi + +# Every suite launches whatever E2E_CHROMIUM names, and export-character-pdf.js otherwise only looks +# in the web sandbox's /opt/pw-browsers. Resolve it with the probes' finder so any machine works. +if [ -z "${E2E_CHROMIUM:-}" ]; then + E2E_CHROMIUM="$(node -e "process.stdout.write(require('./test/browser/lib/find-chrome').resolveBrowser() || '')")" + export E2E_CHROMIUM +fi + +# Staleness, which is the trap this whole block exists for. +# +# The SERVER is `lein run`: compiled from source at every boot, so it is never +# stale. The bundle and the stylesheet are artifacts ON DISK and nothing here +# rebuilt them. That asymmetry is what makes it invisible -- edit a .cljs, boot +# a server that is genuinely fresh, and the page still runs the OLD code while +# every signal says it is new. A suite then passes or fails against code that is +# not the code you changed, which is worse than either outcome on its own. +# +# E2E_SKIP_BUILD=1 keeps the artifacts as they are, and says so loudly. +BUNDLE=resources/public/js/compiled/orcpub.js +SHEET=resources/public/css/compiled/styles.css + +# Sources newer than an artifact, a few of them, for the message. +newer_sources() { + local artifact="$1"; shift + find "$@" -name '*.clj*' -newer "$artifact" 2>/dev/null | head -5 +} + +# The folders the bundle compiles from, read from its own build config (:watch-dirs), never kept +# by hand here: a hand-kept list missed web/cljs, and startup edits were tested against an old +# bundle with no warning. Unreadable config stops the run rather than skip the check. +cljs_dirs() { + local cfg=prod.cljs.edn + grep -q CLOSURE_UNCOMPILED_DEFINES "$BUNDLE" 2>/dev/null && cfg=dev.cljs.edn + sed -n 's/.*:watch-dirs *\[\([^]]*\)\].*/\1/p' "$cfg" | tr -d '"' +} + +if [ ! -f "$BUNDLE" ]; then + echo "Building the ClojureScript bundle (first run only)..." + lein fig:prod || exit 1 +elif ! CLJS_DIRS="$(cljs_dirs)" || [ -z "$CLJS_DIRS" ]; then + echo "Could not read :watch-dirs from the build config, so cannot tell whether the bundle is stale." + exit 1 +elif [ -n "$(newer_sources "$BUNDLE" $CLJS_DIRS)" ]; then + echo "ClojureScript changed since the bundle was built (sources: $CLJS_DIRS):" + newer_sources "$BUNDLE" $CLJS_DIRS | sed 's/^/ /' + if [ -n "${E2E_SKIP_BUILD:-}" ]; then + echo " E2E_SKIP_BUILD is set, so this run tests the OLD client code." + else + # Rebuild the kind that is already there. A dev bundle and a prod bundle + # are not interchangeable here: the CSP branch below keys off which one it + # is, so silently swapping them would trade this trap for another. + if grep -q CLOSURE_UNCOMPILED_DEFINES "$BUNDLE" 2>/dev/null; then + echo " Rebuilding (development bundle)..."; lein fig:build || exit 1 + else + echo " Rebuilding (production bundle)..."; lein fig:prod || exit 1 + fi + fi +fi + +# styles.css is generated by garden and deliberately untracked, so a fresh +# checkout has none. Without it the app serves an unstyled page and every +# scenario reports a 404 as a console problem. +# +# Only the styles namespaces are watched. Garden output can in principle depend +# on anything it requires, but a change outside styles/ that moves the CSS is +# rare enough not to be worth rebuilding the stylesheet on every server edit. +if [ ! -f "$SHEET" ]; then + echo "Compiling the stylesheet (first run only)..." + lein garden once || exit 1 +elif [ -n "$(newer_sources "$SHEET" src/clj/orcpub/styles)" ]; then + echo "Styles changed since the stylesheet was compiled:" + newer_sources "$SHEET" src/clj/orcpub/styles | sed 's/^/ /' + if [ -n "${E2E_SKIP_BUILD:-}" ]; then + echo " E2E_SKIP_BUILD is set, so this run tests the OLD stylesheet." + else + echo " Recompiling..."; lein garden once || exit 1 + fi +fi + +# A development bundle (lein fig:build) loads its namespaces with document.write, which the +# strict policy's 'strict-dynamic' blocks, so every page fails before a check runs. Say so and +# run without CSP; lein fig:prod builds the bundle the strict policy is for. +if [ -z "${CSP_POLICY:-}" ] && grep -q CLOSURE_UNCOMPILED_DEFINES resources/public/js/compiled/orcpub.js 2>/dev/null; then + export CSP_POLICY=none + echo "Development bundle on disk, so CSP is off for this run. Build with lein fig:prod to test under the strict policy." +fi + +# Mail. Without a server to send to, registration FAILS outright -- the +# verification email goes out inside the same try as the transaction -- and the +# verification and reset keys exist nowhere but in that mail. That is why the +# signup, verify and reset flows had no coverage at all. The suites point at +# scripts/e2e/lib/mail-sink.js, which accepts anything and hands the message +# back, so a suite can follow the link a real person would click. +MAIL_PORT="${E2E_MAIL_PORT:-2525}" +export E2E_MAIL_PORT="$MAIL_PORT" + +echo "Starting server on :${PORT}..." +DATOMIC_URL="datomic:mem://orcpub-e2e" \ +ORCPUB_ENV=dev \ +SIGNATURE="${SIGNATURE:-e2e-test-signature}" \ +EMAIL_SERVER_URL=127.0.0.1 \ +EMAIL_SERVER_PORT="$MAIL_PORT" \ +EMAIL_FROM_ADDRESS="${EMAIL_FROM_ADDRESS:-e2e@orcpub.invalid}" \ +PORT="$PORT" \ + setsid lein with-profile init-db run -m e2e-boot > "$LOG" 2>&1 & +SERVER_PID=$! +# lein forks a JVM, and killing only the wrapper leaves that child holding the +# port -- the next run then fails to bind and silently tests the stale server. +# setsid puts both in their own process group so the whole group goes at once. +trap 'kill -- -$SERVER_PID 2>/dev/null' EXIT + +for _ in $(seq 1 90); do + curl -sf -o /dev/null "http://localhost:${PORT}/" && break + sleep 2 +done +if ! curl -sf -o /dev/null "http://localhost:${PORT}/"; then + echo "Server never came up. Log:"; tail -30 "$LOG"; exit 1 +fi +if grep -q BindException "$LOG" 2>/dev/null; then + echo "Port ${PORT} was already in use, so this would have tested a stale server." + exit 1 +fi + +# The server answers before e2e-boot has finished seeding, so wait until it says it has. +for _ in $(seq 1 60); do grep -q E2E-READY "$LOG" && break; sleep 1; done +# The seeded character that carries one of kaylee's custom items, for suites that open it. +E2E_CHARACTER_ID=$(grep -o "E2E-CHARACTER 200 [0-9]*" "$LOG" | awk '{print $3}') +export E2E_CHARACTER_ID +# Another of kaylee's characters, which also uses homebrew, for the share-link suite. +E2E_HOMEBREW_CHARACTER_ID=$(grep -o "E2E-HOMEBREW-CHARACTER 200 [0-9]*" "$LOG" | awk '{print $3}') +export E2E_HOMEBREW_CHARACTER_ID +# And one whose last share link expired, for the expired-note suite. +E2E_EXPIRED_CHARACTER_ID=$(grep -o "E2E-EXPIRED-CHARACTER 200 [0-9]*" "$LOG" | awk '{print $3}') +export E2E_EXPIRED_CHARACTER_ID + +E2E_BASE="http://localhost:${PORT}" node "scripts/e2e/$SUITE" +NODE_RC=$? + +# The browser cannot read PDF field names -- they sit in compressed object +# streams -- so each exported file is inspected here, where PDFBox is available. +# Only the PDF suite writes there; any other suite would inspect a previous run's files. +OUT="${E2E_OUT:-/tmp/e2e-pdf}" +if [ "$NODE_RC" -eq 0 ] && [ "$SUITE" = "export-character-pdf.js" ]; then + for pdf in "$OUT"/*.pdf; do + [ -f "$pdf" ] || continue + # export-character-pdf.js leaves the expected page count beside each PDF. + MIN_PAGES="" + [ -f "${pdf%.pdf}.min-pages" ] && MIN_PAGES=$(cat "${pdf%.pdf}.min-pages") + echo + echo "Inspecting $(basename "$pdf")..." + if ! lein with-profile init-db run -m clojure.main dev/inspect_export.clj \ + "$pdf" $MIN_PAGES 2>&1 | grep -Ev "JAVA_TOOL|^WARNING|WARN "; then + NODE_RC=1 + fi + done +fi +exit "$NODE_RC" diff --git a/scripts/e2e/share-from-character-list.js b/scripts/e2e/share-from-character-list.js new file mode 100644 index 000000000..bacd46a53 --- /dev/null +++ b/scripts/e2e/share-from-character-list.js @@ -0,0 +1,66 @@ +// Can a character be shared from the character list, where there is no room for the share line? +// +// ./scripts/e2e/run.sh share-from-character-list.js +// +// kaylee's homebrew character (E2E_HOMEBREW_CHARACTER_ID) has never been shared. Its row in the character list +// has one Copy link button: a single press must make the share, send the homebrew to the server, copy a short +// link and say the character is now shared. Pressing it again copies the same link, and the character page +// then shows the character as shared, with that link. +const { chromium } = require('playwright'); +const { BASE, EXECUTABLE, HOMEBREW, newContext, login, watchShares, waitForButton, copyLink, tokenStatus, + checker } = require('./lib'); + +const ID = process.env.E2E_HOMEBREW_CHARACTER_ID; + +(async () => { + if (!ID) { + console.error('No E2E_HOMEBREW_CHARACTER_ID: run this through scripts/e2e/run.sh, which reads it from e2e-boot.'); + process.exit(2); + } + const { check, failures } = checker(); + const browser = await chromium.launch({ executablePath: EXECUTABLE }); + const ctx = await newContext(browser, HOMEBREW.library('Spoken only by test suites.')); + const page = await ctx.newPage(); + const errors = []; + page.on('pageerror', e => errors.push(e.message)); + const requests = watchShares(page); + await login(page, 'kaylee', 'serenity99'); + + console.log('kaylee opens the character list:'); + await page.goto(`${BASE}/pages/dnd/5e/characters`, { waitUntil: 'networkidle', timeout: 120000 }); + check(await tokenStatus(page, ID) === 404, 'the character starts unshared'); + await page.evaluate(id => { + const c = window.cljs.core; + window.re_frame.core.dispatch(c.PersistentVector.fromArray( + [c.keyword(null, 'toggle-character-expanded'), parseInt(id)], true)); + }, ID); + // The message is a toast that closes itself, so watch for it from before the press. + const noticed = page.waitForFunction(() => document.body && document.body.innerText.includes('now shared'), null, + { timeout: 30000 }).then(() => true, () => false); + await waitForButton(page, 'Copy link'); + await page.evaluate(() => { window.__copied = ''; }); + await page.locator('button', { hasText: 'Copy link' }).first().click(); + await page.waitForFunction(() => window.__copied, null, { timeout: 30000 }).catch(() => {}); + const link = await page.evaluate(() => window.__copied || ''); + check(/#s=[A-Za-z0-9_-]{22}$/.test(link), 'one press makes a short link and copies it', link); + check(requests.includes('PUT 200'), 'the homebrew went to the server', requests.join(', ')); + check(await noticed, 'the page says the character is now shared'); + check(await tokenStatus(page, ID) === 200, 'the server has the share'); + + console.log('\nshe presses it again:'); + await page.waitForTimeout(2500); + const again = await copyLink(page); + check(again === link, 'the same link is copied', again); + + console.log('\nshe opens the character page:'); + await page.goto(`${BASE}/pages/dnd/5e/characters/${ID}`, { waitUntil: 'networkidle', timeout: 120000 }); + await waitForButton(page, 'Stop sharing').catch(() => {}); + check(await page.evaluate(() => [...document.querySelectorAll('.share-pill')].some(p => p.textContent.trim() === 'Shared')), + 'the page shows it as shared'); + check(await copyLink(page) === link, 'with the same link'); + check(errors.length === 0, 'no uncaught page errors', errors.join(' | ')); + + await browser.close(); + console.log(failures() ? `\nFAILURES: ${failures()}` : '\nall checks passed'); + process.exit(failures() ? 1 : 0); +})(); diff --git a/scripts/e2e/share-link-carries-homebrew.js b/scripts/e2e/share-link-carries-homebrew.js new file mode 100644 index 000000000..735ac4c59 --- /dev/null +++ b/scripts/e2e/share-link-carries-homebrew.js @@ -0,0 +1,148 @@ +// Does a short share link carry a character's homebrew to someone who does not have it, and keep +// carrying the current homebrew? +// +// ./scripts/e2e/run.sh share-link-carries-homebrew.js +// +// e2e-boot seeds a character for kaylee that knows a homebrew language, and run.sh passes its id on as +// E2E_HOMEBREW_CHARACTER_ID. kaylee's browser gets that language in its local library, where an import +// would leave it. Signed in, her character page must store nothing until she presses Share link, which +// sends the homebrew to the server and makes a link that carries only a token; the character list must +// then give the same link. The link is opened logged out and as zoe, neither of whom has the homebrew, +// and both must load it; a wrong token must load nothing and say so. Then kaylee changes the language's +// description and opens her character again: the link must stay the same and show the new description. +// Then she presses New link: the old link loads nothing and the new one works. Last, she presses Stop +// sharing: that link loads nothing too, and opening her character again stores nothing. +const { chromium } = require('playwright'); +const { BASE, EXECUTABLE, HOMEBREW, newContext, login, watchShares, waitForButton, copyLink, + sharedHomebrew, tokenStatus, checker } = require('./lib'); + +const ID = process.env.E2E_HOMEBREW_CHARACTER_ID; +const FIRST = 'Spoken only by test suites.'; +const SECOND = 'Spoken only by test suites, and now with verbs.'; + +// kaylee, signed in, on her character's page, with the language described as `description`. +async function ownerSession(browser, description) { + const ctx = await newContext(browser, HOMEBREW.library(description)); + const page = await ctx.newPage(); + page.on('dialog', d => d.accept()); + const requests = watchShares(page); + await login(page, 'kaylee', 'serenity99'); + await page.goto(`${BASE}/pages/dnd/5e/characters/${ID}`, { waitUntil: 'networkidle', timeout: 120000 }); + return { ctx, page, requests }; +} + +async function open(browser, link, who) { + const ctx = await newContext(browser, null); + const page = await ctx.newPage(); + const errors = []; + page.on('pageerror', e => errors.push(e.message)); + const requests = watchShares(page); + if (who) await login(page, who.user, who.pass); + await page.goto(link, { waitUntil: 'domcontentloaded', timeout: 120000 }); + // A failure notice is a toast that closes itself, so watch for it from the start. + const noticed = page.waitForFunction(() => document.body && document.body.innerText.includes('could not be loaded'), + null, { timeout: 15000 }).then(() => 1, () => 0); + await page.waitForLoadState('networkidle', { timeout: 120000 }); + await page.waitForTimeout(5000); + const loaded = await sharedHomebrew(page); + const banner = await page.locator('text=Shared with 1 homebrew piece').count(); + const failedNotice = await noticed; + await ctx.close(); + return { ...loaded, banner, failedNotice, requests, errors }; +} + +(async () => { + if (!ID) { + console.error('No E2E_HOMEBREW_CHARACTER_ID: run this through scripts/e2e/run.sh, which reads it from e2e-boot.'); + process.exit(2); + } + const { check, failures } = checker(); + const browser = await chromium.launch({ executablePath: EXECUTABLE }); + + console.log('kaylee makes the link:'); + const first = await ownerSession(browser, FIRST); + const beforeSharing = await tokenStatus(first.page, ID); + check(beforeSharing === 404 && first.requests.length === 0, 'nothing is stored before Share link is pressed', + `token route ${beforeSharing}; ${first.requests.join(', ') || 'no uploads'}`); + await waitForButton(first.page, 'Share link'); + await first.page.locator('button', { hasText: 'Share link' }).first().click(); + const link = await copyLink(first.page); + const match = link.match(/#s=([A-Za-z0-9_-]{22})$/); + check(Boolean(match), 'the link carries a share token', link); + check(link.length < 120, 'the link is short', `${link.length} characters`); + check(first.requests.includes('PUT 200'), 'the homebrew went to the server', first.requests.join(', ')); + + // The character list's Copy link is the same button, which once packed whatever character was open in + // the builder; in this fresh browser that is none, so its link must match the character page's. + console.log('\nkaylee copies it from the character list:'); + await first.page.goto(`${BASE}/pages/dnd/5e/characters`, { waitUntil: 'networkidle', timeout: 120000 }); + await first.page.evaluate(id => { + const c = window.cljs.core; + window.re_frame.core.dispatch(c.PersistentVector.fromArray( + [c.keyword(null, 'toggle-character-expanded'), parseInt(id)], true)); + }, ID); + let listLink; + try { listLink = await copyLink(first.page); } catch (e) { listLink = `(no Copy link in the row: ${e.message.split('\n')[0]})`; } + check(Boolean(match) && listLink === link, "the list row's link is the character page's link", listLink); + await first.ctx.close(); + + if (match) { + for (const who of [null, { user: 'zoe', pass: 'washburne7' }]) { + console.log(`\n${who ? who.user : 'logged out'} opens it:`); + const r = await open(browser, link, who); + check(r.sources.includes(HOMEBREW.source), 'the homebrew loaded', JSON.stringify(r.sources)); + check(r.description === FIRST, 'with its description', r.description); + check(r.banner > 0, 'the page says what was shared'); + check(!r.requests.some(x => x.startsWith('PUT')), 'nothing was uploaded', r.requests.join(', ')); + check(r.errors.length === 0, 'no uncaught page errors', r.errors.join(' | ')); + } + + console.log('\nthe same link with a wrong token:'); + const wrong = await open(browser, link.replace(/#s=[A-Za-z0-9_-]{22}$/, '#s=' + 'A'.repeat(22)), null); + check(wrong.sources.length === 0, 'loads nothing', JSON.stringify(wrong.sources)); + check(wrong.failedNotice > 0, 'says the homebrew could not be loaded'); + + console.log('\nkaylee changes the description and opens her character again:'); + const second = await ownerSession(browser, SECOND); + check(await copyLink(second.page) === link, 'the link stays the same'); + check(second.requests.includes('PUT 200'), 'the changed homebrew went to the server', second.requests.join(', ')); + await second.ctx.close(); + const updated = await open(browser, link, null); + check(updated.description === SECOND, 'the same link shows the new description', updated.description); + + console.log('\nkaylee makes a new link:'); + const third = await ownerSession(browser, SECOND); + await copyLink(third.page); + await third.page.locator('button', { hasText: 'New link' }).first().click(); + let fresh = link; + for (let i = 0; i < 20 && fresh === link; i++) { await third.page.waitForTimeout(1000); fresh = await copyLink(third.page); } + check(/#s=[A-Za-z0-9_-]{22}$/.test(fresh) && fresh !== link, 'New link gives a different link', fresh); + await third.ctx.close(); + const old = await open(browser, link, null); + check(old.sources.length === 0 && old.failedNotice > 0, 'the old link loads nothing, and says so', JSON.stringify(old.sources)); + const renewed = await open(browser, fresh, null); + check(renewed.description === SECOND, 'the new link loads the homebrew', renewed.description); + + console.log('\nkaylee stops sharing:'); + const fourth = await ownerSession(browser, SECOND); + await waitForButton(fourth.page, 'Stop sharing'); + await fourth.page.locator('button', { hasText: 'Stop sharing' }).first().click(); + await waitForButton(fourth.page, 'Share link'); + check(await tokenStatus(fourth.page, ID) === 404, 'the page offers Share link again, and the server keeps no share'); + await fourth.ctx.close(); + const stopped = await open(browser, fresh, null); + check(stopped.sources.length === 0 && stopped.failedNotice > 0, 'the last link loads nothing, and says so', + JSON.stringify(stopped.sources)); + const fifth = await ownerSession(browser, SECOND); + await waitForButton(fifth.page, 'Share link'); + await fifth.page.waitForTimeout(2000); + const afterStopping = await tokenStatus(fifth.page, ID); + check(!fifth.requests.some(x => x.startsWith('PUT')) && afterStopping === 404, + 'opening the character again stores nothing', `token route ${afterStopping}; ${fifth.requests.join(', ') || 'no uploads'}`); + await fifth.ctx.close(); + } + + await browser.close(); + console.log(failures() ? `\nFAILURES: ${failures()}` : '\nall checks passed'); + process.exit(failures() ? 1 : 0); +})(); diff --git a/scripts/e2e/share-link-expired-note.js b/scripts/e2e/share-link-expired-note.js new file mode 100644 index 000000000..bbc6c3b47 --- /dev/null +++ b/scripts/e2e/share-link-expired-note.js @@ -0,0 +1,83 @@ +// Does the owner's page say that a share link expired, and keep saying so until she acts on it? +// +// ./scripts/e2e/run.sh share-link-expired-note.js +// +// e2e-boot seeds a character for kaylee that knows the homebrew language and whose last share link expired, +// as the pruner leaves it: a note of the character and the date. run.sh passes its id on as +// E2E_EXPIRED_CHARACTER_ID. The share line's status must say the link expired, with Share link beside it, on +// her character page and again after a reload. The character list row has only its Copy link button, so +// the status stays on the page. Dismiss must remove it for good without storing a share. +const { chromium } = require('playwright'); +const { BASE, EXECUTABLE, HOMEBREW, newContext, login, watchShares, waitForButton, tokenStatus, + checker } = require('./lib'); + +const ID = process.env.E2E_EXPIRED_CHARACTER_ID; +const NOTE = 'Link expired'; + +// The status text once it shows, or null when it does not within 30 seconds. innerText is what is on +// screen, so a hidden status does not count; the comparison ignores case in case a container sets +// capitals. +async function note(page) { + const shown = await page.waitForFunction(t => document.body && document.body.innerText.toLowerCase().includes(t), + NOTE.toLowerCase(), { timeout: 30000 }).then(() => true, () => false); + if (!shown) return null; + return page.evaluate(t => [...document.querySelectorAll('span')].map(s => s.textContent).find(x => x.includes(t)), NOTE); +} + +const noteShowing = page => page.evaluate(t => document.body.innerText.toLowerCase().includes(t), NOTE.toLowerCase()); + +(async () => { + if (!ID) { + console.error('No E2E_EXPIRED_CHARACTER_ID: run this through scripts/e2e/run.sh, which reads it from e2e-boot.'); + process.exit(2); + } + const { check, failures } = checker(); + const browser = await chromium.launch({ executablePath: EXECUTABLE }); + const ctx = await newContext(browser, HOMEBREW.library('Spoken only by test suites.')); + const page = await ctx.newPage(); + const errors = []; + page.on('pageerror', e => errors.push(e.message)); + const requests = watchShares(page); + await login(page, 'kaylee', 'serenity99'); + const characterPage = `${BASE}/pages/dnd/5e/characters/${ID}`; + + console.log('kaylee opens the character:'); + await page.goto(characterPage, { waitUntil: 'networkidle', timeout: 120000 }); + const text = await note(page); + check(Boolean(text), 'the page says the link expired', text); + check(/\b20\d\d\b/.test(text || ''), 'with the date', text); + await waitForButton(page, 'Share link'); + check(await tokenStatus(page, ID) === 410, 'Share link is offered beside it, and the server still has the note'); + + console.log('\nshe reloads and does nothing:'); + await page.reload({ waitUntil: 'networkidle', timeout: 120000 }); + check(Boolean(await note(page)), 'the note is still there'); + + console.log('\nshe opens the character list:'); + await page.goto(`${BASE}/pages/dnd/5e/characters`, { waitUntil: 'networkidle', timeout: 120000 }); + await page.evaluate(id => { + const c = window.cljs.core; + window.re_frame.core.dispatch(c.PersistentVector.fromArray( + [c.keyword(null, 'toggle-character-expanded'), parseInt(id)], true)); + }, ID); + await waitForButton(page, 'Copy link'); + check(!(await noteShowing(page)), "the character's row has its Copy link button and leaves the status to the page"); + + console.log('\nshe dismisses it:'); + await page.goto(characterPage, { waitUntil: 'networkidle', timeout: 120000 }); + await waitForButton(page, 'Dismiss'); + await page.locator('button', { hasText: 'Dismiss' }).first().click(); + await page.waitForFunction(t => !document.body.innerText.includes(t), NOTE, { timeout: 15000 }).catch(() => {}); + check(!(await noteShowing(page)), 'the note goes'); + await page.reload({ waitUntil: 'networkidle', timeout: 120000 }); + await waitForButton(page, 'Share link'); + await page.waitForTimeout(2000); + check(!(await noteShowing(page)), 'and stays gone after a reload'); + check(await tokenStatus(page, ID) === 404, 'the server keeps no share and no note'); + check(!requests.some(x => x.startsWith('PUT')), 'nothing was stored', requests.join(', ') || 'no uploads'); + check(errors.length === 0, 'no uncaught page errors', errors.join(' | ')); + + await browser.close(); + console.log(failures() ? `\nFAILURES: ${failures()}` : '\nall checks passed'); + process.exit(failures() ? 1 : 0); +})(); diff --git a/scripts/e2e/shared-character-shows-its-items.js b/scripts/e2e/shared-character-shows-its-items.js new file mode 100644 index 000000000..4a165ed37 --- /dev/null +++ b/scripts/e2e/shared-character-shows-its-items.js @@ -0,0 +1,107 @@ +// Does a character that uses its owner's custom items show them to someone else? +// +// ./scripts/e2e/run.sh shared-character-shows-its-items.js +// +// e2e-boot seeds a fighter owned by kaylee with one of her custom items equipped and prints its +// id; run.sh passes that on as E2E_CHARACTER_ID. The page is opened three times: logged out, as +// zoe, and as kaylee. The server sends the equipped item with the character, so each viewer should +// see it in the Equipment tab's Other Magic Items table. For logged-out and zoe it comes from the +// copy kept for that character; kaylee's comes from her own list, and no copy is kept for her. +const { chromium } = require('playwright'); + +const BASE = process.env.E2E_BASE || 'http://localhost:8890'; +const ID = process.env.E2E_CHARACTER_ID; +const EXECUTABLE = process.env.E2E_CHROMIUM || undefined; +const ITEM = 'Kaylee Seeded Alpha'; + +const kept = (page, id) => page.evaluate(id => { + const c = window.cljs.core, s = window.re_frame.db.app_db.state; + const nameKw = c.keyword('orcpub.dnd.e5.magic-items', 'name'); + const names = items => (items == null ? [] : c.clj__GT_js(c.vec(c.map(i => c.get(i, nameKw), items)))); + const overlay = c.deref(window.re_frame.core.subscribe( + c.PersistentVector.fromArray([c.keyword('orcpub.dnd.e5.magic-items', 'shared-custom-items')], true))); + return { + keptForThisCharacter: names(c.get(c.get(s, c.keyword(null, 'character-custom-items')), parseInt(id))), + overlay: names(overlay), + }; +}, id); + +async function view(browser, who) { + const ctx = await browser.newContext(); + // The cookie banner and What's New panel would take the clicks. + await ctx.addInitScript(() => { + try { + localStorage.setItem('orcpub:no-cookie-banner', '1'); + localStorage.setItem('whats-new-seen', JSON.stringify('summer-patch-2026')); + } catch (e) {} + }); + const page = await ctx.newPage(); + const errors = []; + page.on('pageerror', e => errors.push(e.message)); + let read = null; + page.on('response', async r => { + if (new URL(r.url()).pathname !== `/dnd/5e/characters/${ID}`) return; + try { + const body = await r.text(); + read = { status: r.status(), carriesItem: body.includes(ITEM), carriesEmail: body.includes('@example.com') }; + } catch (e) {} + }); + if (who) { + await page.goto(`${BASE}/pages/login-page`, { waitUntil: 'domcontentloaded' }); + await page.waitForSelector('input'); + await page.locator('input').nth(0).fill(who.user); + await page.locator('input').nth(1).fill(who.pass); + await page.locator('button.form-button').click(); + await page.waitForTimeout(3500); + } + await page.goto(`${BASE}/pages/dnd/5e/characters/${ID}`, { waitUntil: 'networkidle', timeout: 120000 }); + await page.waitForTimeout(4000); + const state = await kept(page, ID); + // The tab labels are lowercase text styled as capitals; switch the way the tab's click does. + await page.evaluate(() => { + const c = window.cljs.core; + window.re_frame.core.dispatch(c.PersistentVector.fromArray( + [c.keyword('orcpub.dnd.e5.character', 'set-selected-display-tab'), 'equipment'], true)); + }); + await page.waitForTimeout(2000); + const table = page.locator('tbody.other-magic-items'); + const tableText = (await table.count()) ? (await table.first().innerText()).replace(/\s+/g, ' ') : ''; + await ctx.close(); + return { name: who ? who.user : 'logged out', read, tableText, errors, ...state }; +} + +(async () => { + if (!ID) { + console.error('No E2E_CHARACTER_ID: run this through scripts/e2e/run.sh, which reads it from e2e-boot.'); + process.exit(2); + } + const browser = await chromium.launch({ executablePath: EXECUTABLE }); + const results = [ + await view(browser, null), + await view(browser, { user: 'zoe', pass: 'washburne7' }), + await view(browser, { user: 'kaylee', pass: 'serenity99' }), + ]; + await browser.close(); + + let failures = 0; + const check = (ok, label, detail) => { + if (!ok) failures++; + console.log(` ${ok ? 'PASS' : 'FAIL'} ${label}${detail ? ' -- ' + detail : ''}`); + }; + for (const r of results) { + console.log(`\n${r.name}:`); + check(r.read && r.read.status === 200, 'the character read succeeded'); + check(r.read && r.read.carriesItem, 'the character read carried the equipped item'); + check(r.read && !r.read.carriesEmail, 'the character read carried no email address'); + check(r.tableText.includes(ITEM), "the item is in the sheet's Other Magic Items table", r.tableText); + check(r.errors.length === 0, 'no uncaught page errors', r.errors.join(' | ')); + if (r.name === 'kaylee') { + check(r.keptForThisCharacter.length === 0, 'her own character keeps no server copy', JSON.stringify(r.keptForThisCharacter)); + } else { + check(JSON.stringify(r.keptForThisCharacter) === JSON.stringify([ITEM]), 'the item is kept for this character', JSON.stringify(r.keptForThisCharacter)); + check(r.overlay.includes(ITEM), "the item is in the sheet's overlay", JSON.stringify(r.overlay)); + } + } + console.log(failures ? `\nFAILURES: ${failures}` : '\nall checks passed'); + process.exit(failures ? 1 : 0); +})(); diff --git a/scripts/e2e/template-on-open.js b/scripts/e2e/template-on-open.js new file mode 100644 index 000000000..913c46639 --- /dev/null +++ b/scripts/e2e/template-on-open.js @@ -0,0 +1,104 @@ +// When is the character template built, and what does that cost? +// +// ./scripts/e2e/run.sh template-on-open.js +// +// The template cache feeds the save, and the heal of a renamed pick waits for it (offered-keys). +// Pages with no character must not build it. Opening a character must start it, so the heal lands +// without a save, and must not make the builder noticeably slower. Prints timings so a change can +// be compared against the run before it. +const { chromium } = require('playwright'); +const { BASE, EXECUTABLE, checker } = require('./lib'); + +const OLD_KEY = 'half-elf-phb'; +const NEW_KEY = 'half-elf-ua'; +const PLUGINS = `{"Heal Pak" {:orcpub.dnd.e5/races {:${NEW_KEY} {:key :${NEW_KEY} :name "Half-Elf (UA)" :option-pack "Heal Pak" :former-key :${OLD_KEY}}}}}`; +const CACHE = '[:orcpub.dnd.e5.autosave-fx/cached-template]'; +const OFFERED = '[:orcpub.dnd.e5.content-reconciliation/offered-keys]'; +const RACE = '[:character :orcpub.entity/options :race :orcpub.entity/key]'; + +async function context(browser) { + const ctx = await browser.newContext({ viewport: { width: 1280, height: 900 } }); + await ctx.addInitScript(plugins => { + try { + localStorage.setItem('plugins', plugins); + localStorage.setItem('orcpub:no-cookie-banner', '1'); + localStorage.setItem('whats-new-seen', '"summer-patch-2026"'); + } catch (e) {} + window.__blocked = 0; + try { + new PerformanceObserver(l => l.getEntries().forEach(e => { window.__blocked += e.duration; })) + .observe({ type: 'longtask', buffered: true }); + } catch (e) {} + }, PLUGINS); + return ctx; +} + +const appReady = page => page.waitForFunction(() => { + try { return cljs.core.contains_QMARK_(cljs.core.deref(re_frame.db.app_db), cljs.reader.read_string(':plugins')); } + catch (e) { return false; } +}, null, { timeout: 60000 }); + +const isSet = (page, p) => page.evaluate(p => { + const v = cljs.core.get_in(cljs.core.deref(re_frame.db.app_db), cljs.reader.read_string(p)); + return v != null; +}, p); + +const dbAt = (page, p) => page.evaluate(p => + cljs.core.pr_str(cljs.core.get_in(cljs.core.deref(re_frame.db.app_db), cljs.reader.read_string(p))), p); + +async function until(page, fn, ms) { + const t0 = Date.now(); + while (Date.now() - t0 < ms) { if (await fn()) return Date.now() - t0; await page.waitForTimeout(100); } + return null; +} + +(async () => { + const { check, failures } = checker(); + const browser = await chromium.launch({ executablePath: EXECUTABLE }); + + // Pages with no character build nothing. + for (const [label, url] of [['login', '/pages/login-page'], ['character list', '/pages/dnd/5e/characters']]) { + const ctx = await context(browser); + const page = await ctx.newPage(); + await page.goto(`${BASE}${url}`, { waitUntil: 'domcontentloaded' }); + await appReady(page); + await page.waitForTimeout(4000); + check(!(await isSet(page, CACHE)) && !(await isSet(page, OFFERED)), `${label}: no template built`); + console.log(` ${label}: main thread blocked ${Math.round(await page.evaluate(() => window.__blocked))} ms`); + await ctx.close(); + } + + // Opening a character with a renamed pick: when does it heal, and what does the builder cost? + const runs = []; + for (let i = 0; i < 3; i++) { + const ctx = await context(browser); + const page = await ctx.newPage(); + const t0 = Date.now(); + await page.goto(`${BASE}/pages/dnd/5e/character-builder`, { waitUntil: 'domcontentloaded' }); + await appReady(page); + await page.waitForFunction(() => /Ability Scores/.test(document.body.innerText), null, { timeout: 60000 }); + const builderMs = Date.now() - t0; + const blockedBefore = await page.evaluate(() => window.__blocked); + await page.evaluate(oldKey => { + const rd = cljs.reader.read_string; + const db = cljs.core.deref(re_frame.db.app_db); + const broken = cljs.core.assoc_in(cljs.core.get(db, rd(':character')), + rd('[:orcpub.entity/options :race]'), rd(`{:orcpub.entity/key :${oldKey}}`)); + re_frame.core.dispatch_sync(cljs.core.conj(rd('[:orcpub.dnd.e5.character/open-character]'), broken)); + }, OLD_KEY); + const healMs = await until(page, async () => (await dbAt(page, RACE)) === `:${NEW_KEY}`, 15000); + const cacheMs = await until(page, () => isSet(page, CACHE), 1000); + await page.waitForTimeout(500); + const openBlocked = Math.round((await page.evaluate(() => window.__blocked)) - blockedBefore); + runs.push({ builderMs, healMs, openBlocked }); + console.log(` run ${i + 1}: builder ready ${builderMs} ms, heal after open ${healMs === null ? 'NONE in 15s' : healMs + ' ms'}, ` + + `cache ${cacheMs === null ? 'absent' : 'present'}, main thread blocked after open ${openBlocked} ms`); + await ctx.close(); + } + check(runs.every(r => r.healMs !== null), 'opening a character heals its renamed pick without a save', + runs.map(r => r.healMs).join(', ')); + + await browser.close(); + console.log(failures() ? `\nFAILURES: ${failures()}` : '\nall checks passed'); + process.exit(failures() ? 1 : 0); +})(); diff --git a/scripts/fold-branch-changelog.sh b/scripts/fold-branch-changelog.sh new file mode 100755 index 000000000..063abd00a --- /dev/null +++ b/scripts/fold-branch-changelog.sh @@ -0,0 +1,84 @@ +#!/usr/bin/env bash +# Fold docs/branch-changelog.md into a release section of the root CHANGELOG.md, +# then remove it. Run this at merge-to-integration time (see docs/CONTRIBUTING.md). +# +# Usage: +# scripts/fold-branch-changelog.sh "" [""] +# +# the release the entries belong to, e.g. "Summer Patch" +# (matched against a "## [] ..." header). +# optional heading for the folded block; defaults to the +# current git branch name. +# +# It automates the MECHANICAL fold (extract entries -> insert -> git rm). Editorial +# curation (release name, prose tidy-ups) is still yours — review CHANGELOG.md after. +set -euo pipefail + +RELEASE="${1:-}" +if [ -z "$RELEASE" ]; then + echo "Usage: scripts/fold-branch-changelog.sh \"\" [\"\"]" >&2 + exit 2 +fi + +BC="docs/branch-changelog.md" +CL="CHANGELOG.md" +TITLE="${2:-$(git rev-parse --abbrev-ref HEAD 2>/dev/null || echo "branch")}" + +[ -f "$CL" ] || { echo "No $CL found." >&2; exit 1; } +if [ ! -f "$BC" ]; then + echo "No $BC to fold — nothing to do." + exit 0 +fi + +if ! grep -q "^## \[${RELEASE}\]" "$CL"; then + echo "Release section '## [${RELEASE}]' not found in $CL." >&2 + echo "Add the section header first, or pass the exact release name." >&2 + exit 1 +fi + +# Build the folded block from the branch changelog: +# - strip the HTML comment block and the top-level "# ..." title +# - drop "## Why this branch exists" AND its body (reviewer context; never reaches +# CHANGELOG.md). A "## Highlights" section, if the branch earned one, is a normal +# "## Section" and survives as "**Highlights**". +# - demote every other "## Section" to "**Section**" +# - wrap it all under "### " +BLOCK="$(mktemp)" +{ + echo "### ${TITLE}" + echo + awk ' + //) inc=0; next } # skip HTML comment + /^## Why this branch exists[[:space:]]*$/ { skipwhy=1; next } # drop the Why header AND + skipwhy && /^(## |### )/ { skipwhy=0 } # its body, up to the next + skipwhy { next } # section (any "---" too) + /^## / { sub(/^## /, ""); print "**" $0 "**"; next } # demote section headers + /^# / { next } # drop the doc title + { print } + ' "$BC" | sed '/./,$!d' # trim leading blank lines +} > "$BLOCK" + +# Insert the block at the END of the "## [RELEASE]" section (just before the next +# top-level "## " header, or at EOF if it is the last section). +awk -v release="## [${RELEASE}]" -v blockfile="$BLOCK" ' + index($0, release) == 1 { insection = 1; print; next } + insection == 1 && /^## / { + while ((getline line < blockfile) > 0) print line + close(blockfile) + print "" + insection = 0 + } + { print } + END { + if (insection == 1) { + while ((getline line < blockfile) > 0) print line + close(blockfile) + } + } +' "$CL" > "${CL}.tmp" && mv "${CL}.tmp" "$CL" + +rm -f "$BLOCK" +git rm -q "$BC" + +echo "Folded $BC into '## [${RELEASE}]' as '### ${TITLE}' and removed the branch changelog." +echo "Review $CL, then commit." diff --git a/scripts/lint-changelog.py b/scripts/lint-changelog.py new file mode 100755 index 000000000..9d0eeaf8c --- /dev/null +++ b/scripts/lint-changelog.py @@ -0,0 +1,228 @@ +#!/usr/bin/env python3 +"""Check the newest CHANGELOG.md release against the house style. + +The rules are the mechanical half of docs/branch-changelog.template.md. They are +the ones a machine can judge; whether a bullet is worth writing at all is still a +person's call. + +Every "## [Release]" section is checked. The older releases already comply, so +there is nothing to grandfather -- and a rule with no exemptions is one nobody has +to remember the shape of. + +Run it locally the same way CI does: + + scripts/lint-changelog.py # check CHANGELOG.md + scripts/lint-changelog.py --stats # ... and print the length distribution + scripts/lint-changelog.py path/to/file.md # check some other file + +Exit status is 0 when clean and 1 when anything is found, which is the whole +contract with CI: a step fails because the command it ran returned non-zero. +""" + +import re +import sys +import statistics + +DEFAULT_CHANGELOG = "CHANGELOG.md" + +# A bullet may be detailed; it may not be a paragraph. The cap is deliberately +# generous: the median bullet in the Summer Patch is 34 words, so 80 is more than +# twice a typical entry and only catches genuine prose. +MAX_BULLET_WORDS = 80 + +# Prose is allowed in exactly one place, and only that much of it. +MAX_HIGHLIGHT_SENTENCES = 3 + +# Words that say a change was good without saying what it does. +JARGON = re.compile( + r"\b(seamless\w*|robust\w*|comprehensive\w*|powerful\w*|streamlin\w*" + r"|leverag\w*|delve|utilize|holistic|cutting-edge|game-chang\w*)\b", + re.I, +) + +BLOCK_LABELS = ("**Added**", "**Fixed**", "**Changed**", "**Highlights**") + + +def releases(lines): + """(start, end, heading) for every release section, newest first.""" + heads = [i for i, l in enumerate(lines) if l.startswith("## [")] + return [ + (s, heads[n + 1] if n + 1 < len(heads) else len(lines), lines[s].strip()) + for n, s in enumerate(heads) + ] + + +def bullets(lines, start, end): + """Each bullet as (line-number, joined-text). + + Folds in both kinds of continuation: a wrapped line, and a further indented + paragraph after a blank line -- markdown renders that second paragraph as part + of the list item, so it counts toward the bullet's length rather than reading + as prose that escaped a bullet. + """ + out, i = [], start + while i < end: + if lines[i].startswith("- "): + text, j = lines[i], i + 1 + while j < end: + if lines[j].startswith((" ", "\t")) and lines[j].strip(): + text += " " + lines[j].strip() + j += 1 + elif not lines[j].strip(): + # A blank line only ends the bullet if what follows is not + # another indented paragraph belonging to it. + k = j + while k < end and not lines[k].strip(): + k += 1 + if k < end and lines[k].startswith((" ", "\t")) and lines[k].strip(): + j = k + continue + break + else: + break + out.append((i + 1, text[2:].strip())) + i = j + else: + i += 1 + return out + + +def highlight_blocks(lines, start, end): + """Each Highlights block as (line-number, section-name, joined-prose).""" + out, section, i = [], "(release)", start + while i < end: + stripped = lines[i].strip() + if stripped.startswith("###"): + section = stripped.lstrip("#").strip() + if stripped in ("**Highlights**", "### Highlights"): + j, body = i + 1, [] + while j < end and not ( + lines[j].strip().startswith(BLOCK_LABELS) + or lines[j].startswith(("###", "## ", "- ")) + ): + if lines[j].strip(): + body.append(lines[j].strip()) + j += 1 + out.append((i + 1, section, " ".join(body))) + i = j + continue + i += 1 + return out + + +def prose_blocks(lines, start, end): + """Runs of 2+ standalone prose lines: paragraphs outside a bullet or Highlights. + + A line is skipped when it is a bullet, an indented continuation of one, a + heading, a bold block label, an HTML comment, a table row, or inside a fenced + code block or a Highlights section -- so only genuine paragraphs are left. + """ + out, current = [], [] + in_bullet = in_code = in_highlights = False + for i in range(start, end): + raw = lines[i] + s = raw.strip() + if s.startswith("```"): + in_code = not in_code + continue + if in_code: + continue + if s.startswith("###") or s in BLOCK_LABELS: + in_highlights = "highlight" in s.lower() + in_bullet = False + continue + if not s: + if current: + out.append(current) + current = [] + nxt = next((lines[k] for k in range(i + 1, end) if lines[k].strip()), "") + if not (in_bullet and nxt.startswith((" ", "\t"))): + in_bullet = False + continue + if s.startswith("- "): + in_bullet = True + continue + if in_bullet and raw.startswith((" ", "\t")): + continue # continuation paragraph -- counted by bullets(), not prose + if s.startswith(("#", "