From 94441fa9e117acc7bd0f16d793b91230fb42d880 Mon Sep 17 00:00:00 2001 From: Michael Johnson Date: Sat, 17 Oct 2026 05:42:05 -0400 Subject: [PATCH] feat(docs): merge choice combos into a single block --- dbuild/choice_merge.py | 289 +++++++++++++++++++++++++++ dbuild/docs.py | 316 +++++++++++++++++------------- dbuild/templates/README.j2 | 9 +- dbuild/templates/README.mkdocs.j2 | 81 ++++++-- tests/test_choice_merge.py | 107 ++++++++++ tests/test_docs.py | 24 ++- 6 files changed, 673 insertions(+), 153 deletions(-) create mode 100644 dbuild/choice_merge.py create mode 100644 tests/test_choice_merge.py diff --git a/dbuild/choice_merge.py b/dbuild/choice_merge.py new file mode 100644 index 0000000..0e292ba --- /dev/null +++ b/dbuild/choice_merge.py @@ -0,0 +1,289 @@ +"""One page for every way of answering a stack's choices. + +The docs used to write a full set of files per combination; ten on/off +choices made 2048 of them. Here the combinations' markdown is merged into one +superset of lines, each line tagged with the options that keep it, and the +site's deploy-choices.js shows the lines the visitor's picks keep. + +The merge is proven, not assumed: every combination is rebuilt from the +superset and compared with its own render. The few that differ (all ten apps +off: ``services: {}``) are kept whole and shown instead. + +A blank line between optional parts of a file (two services) belongs to +neither: it is a separator, shown when something shows on both sides of it. +deploy-choices.js implements ``visible`` again; keep the two the same. +""" + +from __future__ import annotations + +import difflib +import html +import json +import re +from dataclasses import dataclass, field + +# Combinations kept whole because the merge cannot express them. Past this, +# the merge is not worth it and the caller falls back. +MAX_EXCEPTIONS = 8 + +_FENCE = re.compile(r"^(\s*)(```+|~~~+)") + + +@dataclass +class Line: + text: str + # choice index -> options that keep the line; a choice absent keeps it + # under any option. + allowed: dict[int, set[str]] = field(default_factory=dict) + sep: bool = False + code: bool = False + + def kept(self, picks: tuple[str, ...]) -> bool: + return all(picks[i] in opts for i, opts in self.allowed.items()) + + +def _mark_code(lines: list[Line]) -> None: + fence = None + for ln in lines: + m = _FENCE.match(ln.text) + if fence is None and m: + fence = m.group(2) + elif fence is not None and m and m.group(2).startswith(fence) and not ln.text.strip()[len(m.group(2)):]: + fence = None + else: + ln.code = fence is not None + + +def visible(lines: list[Line], picks: tuple[str, ...]) -> list[bool]: + """Which lines the picks show. A separator shows when each side, up to + the nearest line every pick shows, has a line shown (or is that line), + and not right after another blank.""" + base = [not ln.sep and ln.kept(picks) for ln in lines] + + def side(k: int, step: int) -> bool: + saw = False + j = k + step + while 0 <= j < len(lines): + ln = lines[j] + if not ln.sep and ln.text.strip(): + if not ln.allowed: + return not saw + saw = True + if base[j]: + return True + j += step + return False + + out = [base[k] or (ln.sep and side(k, -1) and side(k, 1)) for k, ln in enumerate(lines)] + last_blank = True + for k, ln in enumerate(lines): + if not out[k]: + continue + blank = not ln.text.strip() + if ln.sep and last_blank: + out[k] = False + continue + last_blank = blank + return out + + +def _norm(lines: list[str], code: list[bool]) -> list[str]: + """What a render can tell apart. Prose blank lines only separate + paragraphs (the page keeps them all); a code block's leading and + trailing blank lines are dropped by the highlighter.""" + out: list[str] = [] + block: list[str] = [] + for t, c in zip(lines, code, strict=True): + if c: + block.append(t) + continue + if block: + out.extend(_trim_blank(block)) + block = [] + if t.strip(): + out.append(t) + out.extend(_trim_blank(block)) + return out + + +def _trim_blank(block: list[str]) -> list[str]: + a, b = 0, len(block) + while a < b and not block[a].strip(): + a += 1 + while b > a and not block[b - 1].strip(): + b -= 1 + return block[a:b] + + +def _code_flags(text: list[str]) -> list[bool]: + ls = [Line(t) for t in text] + _mark_code(ls) + return [ln.code for ln in ls] + + +def _indent(chunk: list[str]) -> int: + for t in chunk: + if t.strip(): + return len(t) - len(t.lstrip()) + return 0 + + +def _slid(ops: list[tuple], a: list[str], b: list[str]) -> list[tuple]: + """difflib's opcodes with each pure delete or insert slid, among the + places it could equally be, to where it starts least indented. Services + all end in `restart: always`; unslid, dropping one took the line from + its neighbour, and the merge tagged it with the wrong choice.""" + out = [] + for n, (tag, a1, a2, b1, b2) in enumerate(ops): + if tag in ("delete", "insert"): + seq, lo, hi = (a, a1, a2) if tag == "delete" else (b, b1, b2) + prev_eq = ops[n - 1][0] == "equal" if n else False + next_eq = ops[n + 1][0] == "equal" if n + 1 < len(ops) else False + room_up = (ops[n - 1][2] - ops[n - 1][1]) if prev_eq else 0 + room_down = (ops[n + 1][2] - ops[n + 1][1]) if next_eq else 0 + best, shift = None, 0 + for d in range(-room_up, room_down + 1): + # Moving by d swaps the lines it uncovers for the ones it + # covers: they must read the same. + ok = all(seq[lo + k] == seq[hi + k] for k in (range(d, 0) if d < 0 else range(d))) + if ok and (best is None or _indent(seq[lo + d:hi + d]) < best): + best, shift = _indent(seq[lo + d:hi + d]), d + if tag == "delete": + out.append((tag, a1 + shift, a2 + shift, b1, b2)) + else: + out.append((tag, a1 + shift, a2 + shift, b1 + shift, b2 + shift)) + else: + out.append((tag, a1, a2, b1, b2)) + return out + + +def merge(variants: dict[tuple[str, ...], list[str]], default: tuple[str, ...], + options: list[list[str]]) -> tuple[list[Line], list[tuple[str, ...]]] | None: + """The superset of lines for ``variants`` (picks -> lines), and the picks + it cannot rebuild. None when there are more of those than MAX_EXCEPTIONS. + + Built from the default and each one-choice deviation from it; the other + combinations only check the result. + """ + lines = [Line(t) for t in variants[default]] + for i, opts in enumerate(options): + for o in opts: + if o == default[i]: + continue + picks = (*default[:i], o, *default[i + 1:]) + target = variants.get(picks) + if target is None: + return None + shown = [k for k, ln in enumerate(lines) if ln.kept(picks)] + seen = [lines[k].text for k in shown] + sm = difflib.SequenceMatcher(None, seen, target, autojunk=False) + # Back to front, so an insert does not shift the indices still to do. + for tag, a1, a2, b1, b2 in reversed(_slid(sm.get_opcodes(), seen, target)): + if tag == "equal": + continue + for k in shown[a1:a2]: + lines[k].allowed.setdefault(i, set(opts)).discard(o) + if b2 > b1: + at = shown[a2 - 1] + 1 if a2 > a1 else (shown[a1] if a1 < len(shown) else len(lines)) + lines[at:at] = [Line(t, {i: {o}}) for t in target[b1:b2]] + for ln in lines: + ln.allowed = {i: s for i, s in ln.allowed.items() if s != set(options[i])} + _mark_code(lines) + want = {p: _norm(t, _code_flags(t)) for p, t in variants.items()} + + def misses(limit: int) -> list[tuple[str, ...]]: + out = [] + for p in variants: + shown = visible(lines, p) + got = _norm([ln.text for ln, v in zip(lines, shown, strict=True) if v], + [ln.code for ln, v in zip(lines, shown, strict=True) if v]) + if got != want[p]: + out.append(p) + if len(out) > limit: + break + return out + + # A blank line in code that only some picks have is a separator (between + # two services) or simply part of what an option adds (an .env empty but + # for one option). Separators first; then each one, in turn, back to its + # options where that rebuilds more combinations. + seps = [ln for ln in lines if ln.code and not ln.text.strip() and ln.allowed] + kept = {id(ln): ln.allowed for ln in seps} + for ln in seps: + ln.sep, ln.allowed = True, {} + for ln in lines: + if not ln.code and not ln.text.strip(): + ln.allowed = {} + exceptions = misses(len(variants)) + for ln in seps: + if not exceptions: + break + ln.sep, ln.allowed = False, kept[id(ln)] + trial = misses(len(exceptions) - 1) + if len(trial) < len(exceptions): + exceptions = trial + else: + ln.sep, ln.allowed = True, {} + if len(exceptions) > MAX_EXCEPTIONS: + return None + return lines, exceptions + + +def to_markdown(lines: list[Line], choice_ids: list[str]) -> tuple[str, dict]: + """The superset as markdown for the site, and what the page script reads + (``data-when`` on the block): ``conds`` by id, ``lines`` one entry per + superset line ([cond id or "", separator?, blank?]), and ``blocks``, each code + block's line numbers in that list. A varying line outside code is a + ````; a code block is named by ``data-wb``. + """ + conds: dict[str, dict[str, list[str]]] = {} + ids: dict[str, str] = {} + + def cond_id(ln: Line) -> str: + if not ln.allowed: + return "" + c = {choice_ids[i]: sorted(s) for i, s in sorted(ln.allowed.items())} + key = json.dumps(c, sort_keys=True) + if key not in ids: + ids[key] = f"c{len(ids)}" + conds[ids[key]] = c + return ids[key] + + entries: list[list] = [] + blocks: dict[str, list[int]] = {} + out: list[str] = [] + block: list[int] | None = None + for k, ln in enumerate(lines): + cid = cond_id(ln) + entries.append([cid, 1 if ln.sep else 0, 0 if ln.text.strip() else 1]) + m = _FENCE.match(ln.text) + if block is None and m and not ln.code: + bid = f"b{len(blocks)}" + block = blocks.setdefault(bid, []) + text = ln.text.rstrip() + attr = f'data-wb="{bid}" data-l="{k}"' + text = text[:-1].rstrip() + f" {attr} }}" if text.endswith("}") and "{" in text else f"{text} {{ {attr} }}" + out.append(text) + elif block is not None and ln.code: + block.append(k) + out.append(ln.text) + elif block is not None: + # The highlighter drops a block's leading and trailing blank + # lines: they have no line on the page to show or hide. + while block and not lines[block[0]].text.strip(): + block.pop(0) + while block and not lines[block[-1]].text.strip(): + block.pop() + block = None + out.append(ln.text) + elif cid and ln.text.strip(): + indent = len(ln.text) - len(ln.text.lstrip()) + out.append(f'{ln.text[:indent]}{ln.text[indent:]}') + else: + out.append(ln.text) + return "\n".join(out), {"conds": conds, "lines": entries, "blocks": blocks} + + +def when_attr(data: dict) -> str: + """``data`` as an HTML attribute value.""" + return html.escape(json.dumps(data, separators=(",", ":")), quote=True) diff --git a/dbuild/docs.py b/dbuild/docs.py index bece8e8..1a556a6 100644 --- a/dbuild/docs.py +++ b/dbuild/docs.py @@ -20,8 +20,8 @@ import dataclasses import dbuild +from dbuild import choice_merge, log from dbuild import choices as choices_mod -from dbuild import log from dbuild.config import ( DEFAULT_COMMUNITY, AppTestConfig, @@ -323,10 +323,13 @@ def _site_placeholders(text: str, env_file: bool = False) -> str: return text -# Each combination is a full set of files on the page, so they multiply: -# crafting-apps' ten on/off choices made 2048, a 42 MB page that crashed -# browsers. Past this, the docs show the defaults only. +# Each combination listed is a full set of files on the page, so they +# multiply: crafting-apps' ten on/off choices made 2048, a 42 MB page that +# crashed browsers. Up to MAX_COMBOS they are listed; past it the site merges +# them into one block (choice_merge) and the GitHub README shows the defaults. +# Past MAX_MERGED even the merge's proof is too much work: defaults only. MAX_COMBOS = 16 +MAX_MERGED = 4096 # GitHub's alert blockquote (> [!WARNING]) -> the site's admonition. The site @@ -353,47 +356,60 @@ def _combination_count(cfg: Config) -> int: return math.prod(len(c.options) for c in cfg.metadata.choices) if cfg.metadata.choices else 0 +def _choices_mode(cfg: Config) -> str: + """"list" every combination, "merged" one block on the site, "capped" + the defaults only. "" without choices.""" + count = _combination_count(cfg) + if not count: + return "" + return "list" if count <= MAX_COMBOS else "merged" if count <= MAX_MERGED else "capped" + + def _combinations(cfg: Config) -> list[list[tuple[Any, Any]]]: """Every way of answering the choices: one option per choice, every - combination, the all-default one first. Only the default past MAX_COMBOS.""" + combination, the all-default one first. Only the default past MAX_MERGED.""" import itertools cs = cfg.metadata.choices if not cs: return [] ordered = [sorted(c.options, key=lambda o: o.id != c.default) for c in cs] - if _combination_count(cfg) > MAX_COMBOS: + if _combination_count(cfg) > MAX_MERGED: return [[(c, o[0]) for c, o in zip(cs, ordered, strict=True)]] return [list(zip(cs, picks, strict=True)) for picks in itertools.product(*ordered)] def _capped_note(cfg: Config, runtime: str) -> str: """What the docs say in place of the choices once they are past - MAX_COMBOS, or "" when every combination is listed.""" + MAX_COMBOS (the README; the site when the merge fails), or "" when every + combination is listed.""" count = _combination_count(cfg) if count <= MAX_COMBOS: return "" cs = cfg.metadata.choices where = "its jail from `appjail-director.yml`" if runtime == "appjail" else "its service from `compose.yaml`" # Every option only keeps or drops services (the default drops none). - if all(not (o.profile or o.env or o.ask or o.service_yaml or o.env_lines or o.director_dep or o.env_appjail) - for c in cs for o in c.options): + if all(_pure(o) for c in cs for o in c.options): return f"These files run every part. To leave one out, delete {where}." picks = ", ".join(f"{c.label}: {o.label}" for c in cs for o in c.options if o.id == c.default) return f"These files use the defaults ({picks}). The {count} other ways are too many to list here; change the files by hand." -def _choice_variants(cfg: Config) -> list[tuple[list[tuple[Any, Any]], str, str]]: - """(pairs, compose text, env text) for every combination that is not the - default. The files people copy, one per combination.""" - out = [] - for pairs in _combinations(cfg): - if all(o.id == c.default for c, o in pairs): - continue - opts = [o for _, o in pairs] - compose_text = choices_mod.flatten_compose(cfg.compose_text, cfg.compose_data, opts) - out.append((pairs, compose_text, choices_mod.env_with_options(cfg.example_env, pairs))) - return out +def _pure(o) -> bool: + """An option that does nothing but keep or drop services.""" + return not (o.profile or o.env or o.ask or o.service_yaml or o.env_lines or o.director_dep or o.env_appjail) + + +def _toggle(c) -> dict | None: + """{"on", "off"} option ids for a choice that only keeps a part or drops + it (crafting-apps' apps): the site shows it as one switch, not a row.""" + if len(c.options) != 2: + return None + on = next((o for o in c.options if o.id == c.default), None) + off = next((o for o in c.options if o.id != c.default), None) + if on is None or off is None or on.drop or not off.drop or not _pure(on) or not _pure(off): + return None + return {"on": on.id, "off": off.id} def _authored_director(cfg: Config) -> dict | None: @@ -478,123 +494,141 @@ def _choice_context(cfg: Config, context_env: list[dict[str, Any]]) -> dict[str, README lists ``combos`` flat. ``choices`` is the declaration itself. """ if not cfg.metadata.choices or not cfg.compose_text: - return {"choices": [], "combos": [], "tree": None, "capped": {"podman": "", "appjail": ""}} + return {"choices": [], "combos": [], "all_combos": list, "tree": None, "mode": "", "merge": None, + "capped": {"podman": "", "appjail": ""}} import re as _re import yaml # lazy, as everywhere in this module: the wheel build imports it without yaml - variants = {choices_mod.variant_suffix(pairs): (ct, et) for pairs, ct, et in _choice_variants(cfg)} engine_services = {ENGINE["service"] for ENGINE in choices_mod.ENGINES.values() if ENGINE.get("service")} authored = _authored_director(cfg) - combos = [] - for pairs in _combinations(cfg): - opts = [o for _, o in pairs] - suffix = choices_mod.variant_suffix(pairs) - is_default = suffix == "" - if is_default: - ct, et = choices_mod.flatten_compose(cfg.compose_text, cfg.compose_data, None), cfg.example_env - compose_file, env_file = "compose.yaml", "example.env" - else: - ct, et = variants[suffix] - compose_file, env_file = f"compose.{suffix}.yaml", f"example.{suffix}.env" - # The AppJail side: the template director grows each engine's jail, - # the .env takes every option's values with the host being the jail's - # name. - aj = dict(cfg.metadata.appjail) if isinstance(cfg.metadata.appjail, dict) else {} - values: dict[str, str] = {} - asks = [] - for o in opts: - if o.director_dep: - aj["depends_on"] = [*(aj.get("depends_on") or []), o.director_dep] - # A sidecar is named as its compose service, so an option that drops - # the service (grimmory's own MariaDB) drops the jail too. - dropped = {d for o in opts for d in o.drop} - if dropped and aj.get("depends_on"): - aj["depends_on"] = [d for d in aj["depends_on"] if d.get("name") not in dropped] - for o in opts: - values.update(o.env) - values.update(o.env_appjail) - for ln in o.env_lines: - k, _, rest = ln.partition("=") - values.setdefault(k, rest.partition(" #")[0].strip()) - for a in o.ask: - values.setdefault(a.name, a.default) - asks.append({"name": a.name, "label": a.label, "default": a.default, "type": a.type, "values": a.values}) - aj_env, seen = [], set() - for e in context_env: - item = dict(e) - if item["name"] in values: - item["default"] = values[item["name"]] - item.pop("placeholder", None) - seen.add(item["name"]) - aj_env.append(item) - for k, v in values.items(): - if k not in seen: - aj_env.append({"name": k, "default": v, "env_only": True}) - # What this combination runs, in words, before any YAML. Compose - # references resolve the way compose would: the .env value, else the - # ${VAR:-fallback}, else nothing. - env_values: dict[str, str] = {} - for ln in et.splitlines(): - k, sep, v = ln.partition("=") - if sep and k.strip() and not k.lstrip().startswith("#"): - env_values[k.strip()] = v.partition(" #")[0].strip() - resolve = lambda text, ev=env_values: _re.sub( # noqa: E731 - r"\$\{([A-Za-z_][A-Za-z0-9_]*)(?::-([^}]*))?\}", - lambda m: ev.get(m.group(1)) or (m.group(2) or ""), text) - parsed = (yaml.safe_load(ct) or {}).get("services") or {} - parts, jails = [], [] - for sname, svc in parsed.items(): - img = resolve(str((svc or {}).get("image", ""))) - short = img.split("/")[-1] - kind = "db" if sname in engine_services else "app" - parts.append({"name": sname, "image": short, "kind": kind}) - jails.append({"name": sname.replace("-", "_") if kind == "app" else f"{cfg.image}_{sname}".replace("-", "_"), "image": short, "kind": kind}) - if any(o.id == "external" for o in opts): - parts.append({"name": "your database", "image": "not run here", "kind": "none"}) - jails.append({"name": "your database", "image": "not run here", "kind": "none"}) - folders = [] - for svc in parsed.values(): - for v in (svc or {}).get("volumes") or []: - if isinstance(v, str): - bits = v.split(":") - src, ro = resolve(bits[0]), len(bits) > 2 and "ro" in bits[2].split(",") - else: - src, ro = resolve(str((v or {}).get("source", ""))), bool((v or {}).get("read_only")) - if ro or not src.startswith("/") or src.startswith(("/etc/", "/dev/", "/var/run/", "/usr/")): - continue - if src not in folders: - folders.append(src) - for o in opts: - for ln in o.env_lines: - k, _, v = ln.partition("=") - if k.endswith("_LOCATION") and v and v not in folders: - folders.append(v) - folders = [_re.sub(CONFIG_ROOT_RE, CONFIG_ROOT_VAR, f) for f in folders] - director_text = director_env_text = "" - if authored is not None: - sliced = _director_for_option(authored, cfg, None if is_default else opts) - director_text = _render_director_override(sliced) - director_env_text = _render_override_env(sliced, {"name": cfg.image, "env": aj_env}) - jails = [{"name": str((svc or {}).get("name", n)), "image": next((p["image"] for p in parts if p["name"] == n), ""), "kind": "app"} - for n, svc in sliced["services"].items()] - tpl = next((o for o in opts if o.jail_template_file), None) - combos.append({ - "id": suffix or "default", "default": is_default, - "selections": [{"choice": c.id, "choice_label": c.label, "option": o.id, "label": o.label, "default": o.id == c.default} for c, o in pairs], - "label": ", ".join(o.label for _, o in pairs), - "doc": " ".join(o.doc for _, o in pairs if o.doc), - "parts": parts, "jails": jails, "folders": folders, - "appjail": aj, "appjail_env": aj_env, - "director_text": director_text, "director_env_text": director_env_text, - "jail_template_file": tpl.jail_template_file if tpl else "", "jail_template": tpl.jail_template if tpl else "", - "ask": asks, - "compose_file": compose_file, "env_file": env_file, - "compose_text": ct, "example_env": et, - "compose_text_site": _site_placeholders(ct), - "example_env_site": _site_placeholders(et, env_file=True), - "zip": f"{cfg.image}-podman" + ("" if is_default else "-" + suffix.replace(".", "-")), - }) + + def build(pair_list: list) -> list[dict]: + combos = [] + for pairs in pair_list: + opts = [o for _, o in pairs] + suffix = choices_mod.variant_suffix(pairs) + is_default = suffix == "" + if is_default: + ct, et = choices_mod.flatten_compose(cfg.compose_text, cfg.compose_data, None), cfg.example_env + compose_file, env_file = "compose.yaml", "example.env" + else: + ct = choices_mod.flatten_compose(cfg.compose_text, cfg.compose_data, opts) + et = choices_mod.env_with_options(cfg.example_env, pairs) + compose_file, env_file = f"compose.{suffix}.yaml", f"example.{suffix}.env" + # The AppJail side: the template director grows each engine's jail, + # the .env takes every option's values with the host being the jail's + # name. + aj = dict(cfg.metadata.appjail) if isinstance(cfg.metadata.appjail, dict) else {} + values: dict[str, str] = {} + asks = [] + for o in opts: + if o.director_dep: + aj["depends_on"] = [*(aj.get("depends_on") or []), o.director_dep] + # A sidecar is named as its compose service, so an option that drops + # the service (grimmory's own MariaDB) drops the jail too. + dropped = {d for o in opts for d in o.drop} + if dropped and aj.get("depends_on"): + aj["depends_on"] = [d for d in aj["depends_on"] if d.get("name") not in dropped] + for o in opts: + values.update(o.env) + values.update(o.env_appjail) + for ln in o.env_lines: + k, _, rest = ln.partition("=") + values.setdefault(k, rest.partition(" #")[0].strip()) + for a in o.ask: + values.setdefault(a.name, a.default) + asks.append({"name": a.name, "label": a.label, "default": a.default, "type": a.type, "values": a.values}) + aj_env, seen = [], set() + for e in context_env: + item = dict(e) + if item["name"] in values: + item["default"] = values[item["name"]] + item.pop("placeholder", None) + seen.add(item["name"]) + aj_env.append(item) + for k, v in values.items(): + if k not in seen: + aj_env.append({"name": k, "default": v, "env_only": True}) + # What this combination runs, in words, before any YAML. Compose + # references resolve the way compose would: the .env value, else the + # ${VAR:-fallback}, else nothing. + env_values: dict[str, str] = {} + for ln in et.splitlines(): + k, sep, v = ln.partition("=") + if sep and k.strip() and not k.lstrip().startswith("#"): + env_values[k.strip()] = v.partition(" #")[0].strip() + resolve = lambda text, ev=env_values: _re.sub( # noqa: E731 + r"\$\{([A-Za-z_][A-Za-z0-9_]*)(?::-([^}]*))?\}", + lambda m: ev.get(m.group(1)) or (m.group(2) or ""), text) + parsed = (yaml.safe_load(ct) or {}).get("services") or {} + parts, jails = [], [] + for sname, svc in parsed.items(): + img = resolve(str((svc or {}).get("image", ""))) + short = img.split("/")[-1] + kind = "db" if sname in engine_services else "app" + parts.append({"name": sname, "image": short, "kind": kind}) + jails.append({"name": sname.replace("-", "_") if kind == "app" else f"{cfg.image}_{sname}".replace("-", "_"), "image": short, "kind": kind}) + if any(o.id == "external" for o in opts): + parts.append({"name": "your database", "image": "not run here", "kind": "none"}) + jails.append({"name": "your database", "image": "not run here", "kind": "none"}) + folders = [] + for svc in parsed.values(): + for v in (svc or {}).get("volumes") or []: + if isinstance(v, str): + bits = v.split(":") + src, ro = resolve(bits[0]), len(bits) > 2 and "ro" in bits[2].split(",") + else: + src, ro = resolve(str((v or {}).get("source", ""))), bool((v or {}).get("read_only")) + if ro or not src.startswith("/") or src.startswith(("/etc/", "/dev/", "/var/run/", "/usr/")): + continue + if src not in folders: + folders.append(src) + for o in opts: + for ln in o.env_lines: + k, _, v = ln.partition("=") + if k.endswith("_LOCATION") and v and v not in folders: + folders.append(v) + folders = [_re.sub(CONFIG_ROOT_RE, CONFIG_ROOT_VAR, f) for f in folders] + director_text = director_env_text = "" + if authored is not None: + sliced = _director_for_option(authored, cfg, None if is_default else opts) + director_text = _render_director_override(sliced) + director_env_text = _render_override_env(sliced, {"name": cfg.image, "env": aj_env}) + jails = [{"name": str((svc or {}).get("name", n)), "image": next((p["image"] for p in parts if p["name"] == n), ""), "kind": "app"} + for n, svc in sliced["services"].items()] + tpl = next((o for o in opts if o.jail_template_file), None) + combos.append({ + "id": suffix or "default", "default": is_default, + "selections": [{"choice": c.id, "choice_label": c.label, "option": o.id, "label": o.label, "default": o.id == c.default} for c, o in pairs], + "label": ", ".join(o.label for _, o in pairs), + "doc": " ".join(o.doc for _, o in pairs if o.doc), + "docs": [o.doc for _, o in pairs if o.doc], + "parts": parts, "jails": jails, "folders": folders, + "appjail": aj, "appjail_env": aj_env, + "director_text": director_text, "director_env_text": director_env_text, + "jail_template_file": tpl.jail_template_file if tpl else "", "jail_template": tpl.jail_template if tpl else "", + "ask": asks, + "compose_file": compose_file, "env_file": env_file, + "compose_text": ct, "example_env": et, + "compose_text_site": _site_placeholders(ct), + "example_env_site": _site_placeholders(et, env_file=True), + "zip": f"{cfg.image}-podman" + ("" if is_default else "-" + suffix.replace(".", "-")), + }) + return combos + + pair_list = _combinations(cfg) + mode = _choices_mode(cfg) + # Merged: the README and every other caller need the default alone; the + # site's template asks for the rest (all_combos), built once. + combos = build(pair_list[:1] if mode == "merged" else pair_list) + full: list[list[dict]] = [] + + def all_combos() -> list[dict]: + if not full: + full.append(combos if mode != "merged" else build(pair_list)) + return full[0] + by_key = {tuple(sel["option"] for sel in cb["selections"]): cb for cb in combos} def tree(level: int, picked: tuple) -> dict | None: @@ -610,10 +644,25 @@ def tree(level: int, picked: tuple) -> dict | None: "combo": by_key.get(key) if level == len(cs) - 1 else None}) return {"id": c.id, "label": c.label, "doc": c.doc, "options": nodes} - declared = [{"id": c.id, "label": c.label, "doc": c.doc, "default": c.default, + declared = [{"id": c.id, "label": c.label, "doc": c.doc, "default": c.default, "toggle": _toggle(c), "options": [{"id": o.id, "label": o.label, "doc": o.doc, "default": o.id == c.default} for o in c.options]} for c in cfg.metadata.choices] - return {"choices": declared, "combos": combos, "tree": tree(0, ()), + choice_ids = [c.id for c in cfg.metadata.choices] + options = [[o.id for o in sorted(c.options, key=lambda o: o.id != c.default)] for c in cfg.metadata.choices] + def merge(rendered: list[str]) -> dict | None: + """The site's one block for the combinations' renders (in + all_combos' order), or None when they do not merge.""" + keys = [tuple(sel["option"] for sel in cb["selections"]) for cb in all_combos()] + got = choice_merge.merge(dict(zip(keys, (r.split("\n") for r in rendered), strict=True)), keys[0], options) + if got is None: + return None + lines, exceptions = got + markdown, when = choice_merge.to_markdown(lines, choice_ids) + return {"markdown": markdown, "when": choice_merge.when_attr(when), + "exceptions": [{"key": ";".join(f"{c}={o}" for c, o in zip(choice_ids, k, strict=True)), + "text": rendered[keys.index(k)]} for k in exceptions]} + + return {"choices": declared, "combos": combos, "all_combos": all_combos, "tree": tree(0, ()), "mode": mode, "merge": merge, "capped": {r: _capped_note(cfg, r) for r in ("podman", "appjail")}} @@ -879,6 +928,7 @@ def _enrich_metadata(cfg: Config, community_override: str | None = None) -> dict cc = _choice_context(cfg, context["env"]) context["choices"], context["combos"], context["choice_tree"] = cc["choices"], cc["combos"], cc["tree"] context["choices_capped"] = cc["capped"] + context["choices_mode"], context["choice_merge"], context["all_combos"] = cc["mode"], cc["merge"], cc["all_combos"] # A stack whose author wrote the whole director: the README shows that, # not the one-service template, with the jail templates it ships. authored = _authored_director(cfg) diff --git a/dbuild/templates/README.j2 b/dbuild/templates/README.j2 index 5fdcb80..7eb675c 100644 --- a/dbuild/templates/README.j2 +++ b/dbuild/templates/README.j2 @@ -88,10 +88,11 @@ services: {#- option: one of a choice's options (see dbuild.choices); its director carries the service dbuild supplies and its .env the option's values. #} -{%- macro appjail_director_snippet(override_port=None, option=None) -%} +{%- macro appjail_director_snippet(override_port=None, option=None, one_zip=false) -%} {%- set appjail = option.appjail if option else appjail %} {%- set env = option.appjail_env if option else env %} -{%- set zip = name ~ "-appjail" ~ ("-" ~ option.id | replace(".", "-") if option and not option.default else "") %} +{#- one_zip: the site's merged block, one download whatever the picks. #} +{%- set zip = name ~ "-appjail" ~ ("-" ~ option.id | replace(".", "-") if option and not option.default and not one_zip else "") %} {%- set authored_dir = option.director_text if option else director_override_text %} {%- set authored_env = option.director_env_text if option else director_override_env %} {%- if authored_dir %} @@ -430,7 +431,7 @@ Before deploying, ensure your host environment is ready. See the [Quick Start Gu {{ choices_capped.podman }} {%- endif %} -{%- for cb in combos %} +{%- for cb in (combos[:1] if choices_capped.podman else combos) %} {%- if not choices_capped.podman %} #### {{ cb.label }}{% if cb.default %} (default){% endif %} @@ -508,7 +509,7 @@ Save as `compose.yaml`, then run `podman-compose up -d`. {{ choices_capped.appjail }} {%- endif %} -{%- for cb in combos %} +{%- for cb in (combos[:1] if choices_capped.appjail else combos) %} {%- if not choices_capped.appjail %} #### {{ cb.label }}{% if cb.default %} (default){% endif %} diff --git a/dbuild/templates/README.mkdocs.j2 b/dbuild/templates/README.mkdocs.j2 index 9e78374..260dff6 100644 --- a/dbuild/templates/README.mkdocs.j2 +++ b/dbuild/templates/README.mkdocs.j2 @@ -10,23 +10,57 @@ under it one block of files per combination, all but the default hidden. deploy-choices.js on the site shows the block that matches the pills and keeps Podman and AppJail on the same answers. Markdown inside the HTML - (md_in_html) so the fences keep their zip attributes. #} + (md_in_html) so the fences keep their zip attributes. + Past MAX_COMBOS ("merged"): every combination is rendered here, and + choice_merge folds them into one block whose varying lines carry the + picks that keep them; the script shows those. Combinations the merge + cannot express follow it whole. #} {%- macro choice_form(runtime) -%} -{%- if choices_capped[runtime] %} +{%- set merged = none %} +{%- if choices_mode == "merged" %} +{%- set rendered = [] %} +{%- for cb in all_combos() %}{% set _ = rendered.append(combo_files(cb, runtime, true)) %}{% endfor %} +{%- set merged = choice_merge(rendered) %} +{%- endif %} +{%- if choices_capped[runtime] and not merged %} {{ choices_capped[runtime] }} {{ combo_files(combos[0], runtime) }} {% else -%} -
-{% for c in choices %} +
+{% for c in choices if not c.toggle %} **{{ c.label }}** {{ c.doc }} {: .choice data-choice="{{ c.id }}" }
{% for o in c.options %}{% endfor %}
{% endfor %} +{%- set toggles = choices | selectattr("toggle") | list %} +{%- if toggles %} + +
{% for c in toggles %}{% endfor %}
+{%- if toggles | length > 3 %} +

+{%- endif %} +{%- endif %} + +
+{%- if merged %} + +
+ +{{ merged.markdown }} + +
+{% for ex in merged.exceptions %} + + +{% endfor %} +{%- else %} {% for cb in combos %}
@@ -36,19 +70,26 @@
{% endfor %} {%- endif %} +{%- endif %} {%- endmacro %} -{%- macro combo_files(cb, runtime) -%} +{#- merged: one item per line (images, folders, option docs), so the merge + can keep or drop each; one zip name whatever the picks. #} +{%- macro combo_files(cb, runtime, merged=false) -%} {% if runtime == "podman" %} -{{ you_get(cb, cb.parts, "container") }} -{% if cb.doc %} +{{ you_get(cb, cb.parts, "container", merged) }} +{% if merged %} +{% for d in cb.docs %} +{{ d }} +{% endfor %} +{% elif cb.doc %} {{ cb.doc }} {% endif %} {#- No .env at all (SQLite: nothing to set): no step asking to fill one in. #} {% if cb.example_env_site | trim -%} **1.** Save as `.env`{% if cb.ask %} and fill in {{ cb.ask | map(attribute='label') | join(', ') }}{% else %} and fill in what is empty{% endif %}: -```env { data-zip-bundle="{{ cb.zip }}" data-zip-filename=".env" } +```env { data-zip-bundle="{{ name ~ '-podman' if merged else cb.zip }}" data-zip-filename=".env" } {{ cb.example_env_site | trim }} ``` @@ -57,27 +98,41 @@ Save as `compose.yaml`: {%- endif %} -```yaml { data-zip-bundle="{{ cb.zip }}" data-zip-filename="compose.yaml" } +```yaml { data-zip-bundle="{{ name ~ '-podman' if merged else cb.zip }}" data-zip-filename="compose.yaml" } {{ cb.compose_text_site | trim }} ``` Then run `podman-compose up -d`. {% else %} -{{ you_get(cb, cb.jails, "jail") }} -{% if cb.doc %} +{{ you_get(cb, cb.jails, "jail", merged) }} +{% if merged %} +{% for d in cb.docs %} +{{ d }} +{% endfor %} +{% elif cb.doc %} {{ cb.doc }} {% endif %} -{{ appjail_director_snippet(port_var, cb) }} +{{ appjail_director_snippet(port_var, cb, merged) }} Save the files above, then run `appjail-director up`. {% endif %} {%- endmacro %} -{%- macro you_get(o, parts, unit) -%} +{%- macro you_get(o, parts, unit, merged=false) -%} +{%- if merged -%} +{%- for p in parts %} +{% if p.kind == "none" %}your own {{ p.name | replace("your database", "database") }}{% else %}`{{ p.image }}`{% endif %} +{%- endfor %} +{%- for f in o.folders %} +`{{ f }}` +{%- endfor %} +{: .you-get } +{%- else -%} {%- set imgs = [] -%} {%- for p in parts %}{% if p.kind == "none" %}{% set _ = imgs.append("your own " ~ p.name | replace("your database", "database")) %}{% else %}{% set _ = imgs.append("`" ~ p.image ~ "`") %}{% endif %}{% endfor -%} {{ imgs | join(" · ") }}{% if o.folders %} — data in {% for f in o.folders %}`{{ f }}`{{ ", " if not loop.last }}{% endfor %}{% endif %} {: .you-get } +{%- endif %} {%- endmacro %} {% block frontmatter -%} diff --git a/tests/test_choice_merge.py b/tests/test_choice_merge.py new file mode 100644 index 0000000..beaacde --- /dev/null +++ b/tests/test_choice_merge.py @@ -0,0 +1,107 @@ +"""dbuild.choice_merge: every combination's render folded into one block.""" + +from __future__ import annotations + +import itertools +import json +import unittest + +from dbuild import choice_merge as cm + + +def _shown(lines, picks): + vis = cm.visible(lines, picks) + return [ln.text for ln, v in zip(lines, vis, strict=True) if v] + + +def _apps(picks, names): + """A compose of the apps switched on, a blank line between services.""" + on = [n for n, p in zip(names, picks, strict=True) if p == "on"] + body = [] + for n in on: + if body: + body.append("") + body += [f" {n}:", f" image: {n}", " restart: always"] + return ["```yaml", "services:", *body, "```"] + + +class TestMerge(unittest.TestCase): + def test_toggles_merge_with_no_combination_kept_whole(self): + names = ["a", "b", "c", "d"] + opts = [["on", "off"]] * 4 + variants = {p: _apps(p, names) for p in itertools.product(*opts)} + lines, exceptions = cm.merge(variants, ("on",) * 4, opts) + # All off is "services:" and nothing: still the superset's lines. + self.assertEqual(exceptions, []) + for p, want in variants.items(): + self.assertEqual(_shown(lines, p), want, p) + + def test_identical_service_tails_do_not_cross_tag(self): + # Every service ends "restart: always": a naive diff takes the line + # from the neighbour when one is dropped. + names = ["a", "b", "c"] + opts = [["on", "off"]] * 3 + variants = {p: _apps(p, names) for p in itertools.product(*opts)} + lines, _ = cm.merge(variants, ("on",) * 3, opts) + tails = [ln.allowed for ln in lines if ln.text == " restart: always"] + self.assertEqual(tails, [{0: {"on"}}, {1: {"on"}}, {2: {"on"}}]) + + def test_options_that_add_and_replace_lines(self): + def env(p): + db = p[0] + out = ["```env", f"DB_TYPE={db}"] + if db != "sqlite": + out += [f"DB_HOST={db}", "DB_PASSWORD="] + return [*out, "```"] + opts = [["sqlite", "postgres", "mariadb"]] + variants = {p: env(p) for p in itertools.product(*opts)} + lines, exceptions = cm.merge(variants, ("sqlite",), opts) + self.assertEqual(exceptions, []) + for p, want in variants.items(): + self.assertEqual(_shown(lines, p), want) + + def test_an_empty_file_for_one_option_is_its_blank_not_a_separator(self): + def env(p): + return ["```env", "", "```"] if p[0] == "sqlite" else ["```env", "DB=pg", "", "PASS=", "```"] + opts = [["sqlite", "postgres"]] + variants = {p: env(p) for p in itertools.product(*opts)} + got = cm.merge(variants, ("sqlite",), opts) + self.assertIsNotNone(got) + self.assertEqual(got[1], []) + + def test_what_it_cannot_express_is_kept_whole(self): + # A line that moves with another choice's pick. + def f(p): + return ["X", "Y"] if p == ("a", "c") else ["Y", "X"] if p == ("b", "c") else ["Z"] + opts = [["a", "b"], ["c", "d"]] + variants = {p: f(p) for p in itertools.product(*opts)} + got = cm.merge(variants, ("a", "c"), opts) + self.assertIsNotNone(got) + for p in got[1]: + self.assertNotEqual(_shown(got[0], p), variants[p]) + + def test_too_many_kept_whole_is_none(self): + # Two choices that order each other's lines: not per-line at all. + letters = list("abcdef") + opts = [letters, letters] + variants = {p: sorted(p) for p in itertools.product(*opts)} + self.assertIsNone(cm.merge(variants, ("a", "a"), opts)) + + +class TestMarkdown(unittest.TestCase): + def test_spans_blocks_and_conditions(self): + names = ["a", "b"] + opts = [["on", "off"]] * 2 + + def render(p): + return ["intro", *[f"`{n}`" for n, x in zip(names, p, strict=True) if x == "on"], "", *_apps(p, names)] + variants = {p: render(p) for p in itertools.product(*opts)} + lines, _ = cm.merge(variants, ("on", "on"), opts) + md, data = cm.to_markdown(lines, names) + self.assertIn('`a`', md) + self.assertIn('```yaml { data-wb="b0" data-l="', md) + self.assertEqual(len(data["conds"]), 2) + self.assertEqual(len(data["lines"]), len(lines)) + # The block's line list skips the fences. + self.assertTrue(all(lines[k].code for k in data["blocks"]["b0"])) + json.loads(cm.when_attr(data).replace(""", '"')) diff --git a/tests/test_docs.py b/tests/test_docs.py index e714c7a..1b0d725 100644 --- a/tests/test_docs.py +++ b/tests/test_docs.py @@ -1055,11 +1055,12 @@ def test_under_the_cap_every_combination_is_listed(self): self.assertEqual(len(ctx["combos"]), 16) self.assertEqual(ctx["choices_capped"], {"podman": "", "appjail": ""}) - def test_past_the_cap_only_the_defaults(self): + def test_past_the_cap_the_readme_shows_the_defaults(self): ctx, outputs, errors = self._ctx(10) self.assertEqual(errors, []) - self.assertEqual(len(ctx["combos"]), 1) - self.assertTrue(ctx["combos"][0]["default"]) + self.assertEqual(ctx["choices_mode"], "merged") + self.assertEqual(len(ctx["combos"]), 1, "the README needs the default alone") + self.assertEqual(len(ctx["all_combos"]()), 1024, "the site merges them all") self.assertIn("app9:", ctx["combos"][0]["compose_text"], "the default runs every app") readme = outputs["README.md"] self.assertIn("To leave one out, delete its service from `compose.yaml`.", readme) @@ -1067,6 +1068,23 @@ def test_past_the_cap_only_the_defaults(self): "no heading per combination") self.assertLess(len(readme), 50_000) + def test_the_site_merges_them_into_one_block(self): + import jinja2 + + ctx, _, _ = self._ctx(10) + env = jinja2.Environment(loader=jinja2.FileSystemLoader(str(Path(docs.__file__).parent / "templates"))) + page = env.get_template("README.mkdocs.j2").render(dict(ctx, render_mode="mkdocs")) + self.assertEqual(page.count('class="deploy-merged"'), 2, "one per runtime") + self.assertLessEqual(page.count('class="deploy-combo"'), 2, "at most all-off, kept whole") + self.assertEqual(page.count('class="toggle"'), 20, "ten switches per runtime") + self.assertLess(len(page), 120_000) + + def test_past_the_merge_limit_defaults_only(self): + ctx, _, errors = self._ctx(13) + self.assertEqual(errors, []) + self.assertEqual(ctx["choices_mode"], "capped") + self.assertEqual(len(ctx["combos"]), 1) + class TestSiteAlerts(unittest.TestCase): def test_github_alert_becomes_an_admonition(self):