Remove size caps on published data

CHANGELOG.md

@@ -108,11 +108,15 @@ date when the release is tagged.
 
 ### Fixed
 
-- **Large libraries' search index.** The explorer loads JSON up to 64 MiB,
-  and a whole-library index over 32 MiB leaves out statement summaries, so
-  search still matches titles and Lean names; publication warns when it
-  does. ATLAS's index had grown past the old 32 MiB limit, so search and the
-  Topics outline failed on every page.
+- **No size caps on published data.** The explorer loads its own site's
+  search index, map data, pages and concept links whatever their size, and
+  its timeout now measures a stalled transfer rather than a long one. Render
+  keeps the whole concept-links layer, reads any `atlas.json`, and writes and
+  replaces a site whose `publication.json` passes 8 MiB (it lists every site
+  file). It notes a search index over 32 MiB, which every explorer page
+  downloads. ATLAS's
+  42 MB index had passed the old 32 MiB limit, so search and the Topics
+  outline failed on every page.
 - **Explorer review follow-ups.** Fit retains a selected unlinked result.
   Deferred list math cancels with its view and falls back to visible text on
   unavailable typesetting. Preview formatting keeps currency and code literal;
@@ -133,10 +137,9 @@ date when the release is tagged.
   heading; stale projection callbacks cannot steal focus. Outline, filters
   and self-hiding map controls keep native focus. Slash stays scoped; modified
   arrow keys stay native, and empty Escape no longer adds history entries (#442).
-- **Reading and index feedback.** Valid HTML character references keep title
-  math without admitting authored HTML. Publication warns when the exact
-  UTF-8 search index reaches 95% of the unchanged 32 MiB browser limit; nothing
-  is truncated or refused. Directory aliases retain statement fragments (#442).
+- **Reading feedback.** Valid HTML character references keep title math
+  without admitting authored HTML. Directory aliases retain statement
+  fragments (#442).
 - **Wiki references.** Section boundaries come from the site's converter, so
   Setext and formatted Sources headings cannot turn provenance into concept
   links. Formal dependencies, statuses and captured article bytes are unchanged (#442).

autoform_cli/_tree_snapshot.py

@@ -145,16 +145,16 @@ def __post_init__(self) -> None:
 
 @dataclass(frozen=True, slots=True)
 class OpaqueDirectoryMarker:
-    """A bounded marker that makes its containing directory opaque.
+    """A marker that makes its containing directory opaque.
 
-    The capture engine reads this small, bounded surface before it visits any
-    descendants.  A recognized marker is not kept: each later look recognizes
-    it afresh.  A presence-only marker accepts any entry kind without reading
+    The capture engine reads this surface, through ``max_bytes`` or whole when
+    that is ``None``, before it visits any descendants.  A recognized marker is
+    not kept: each later look recognizes it afresh.  A presence-only marker accepts any entry kind without reading
     it; this is for directory sentinels such as a nested checkout's ``.git``.
     """
 
     name: str
-    max_bytes: int
+    max_bytes: int | None
     recognizes: Callable[[bytes], bool]
     presence_only: bool = False
 
@@ -163,8 +163,8 @@ def __post_init__(self) -> None:
             raise ValueError("opaque marker name must be one portable path component")
         if not isinstance(self.presence_only, bool):
             raise ValueError("opaque marker presence_only must be a boolean")
-        if _invalid_limit(self.max_bytes):
-            raise ValueError("opaque marker max_bytes must be a non-negative integer")
+        if self.max_bytes is not None and _invalid_limit(self.max_bytes):
+            raise ValueError("opaque marker max_bytes must be a non-negative integer or None")
 
 
 ALL_ENTRIES = TreeSelection(
@@ -889,7 +889,7 @@ def _opaque_marker(
             if _vanished_in_selected_capture(error, selection):
                 continue
             raise
-        if size <= marker.max_bytes and marker.recognizes(data):
+        if (marker.max_bytes is None or size <= marker.max_bytes) and marker.recognizes(data):
             budget.add_file_bytes(size)
             return None
     return names

autoform_cli/assets/blueprint-navigation.js

@@ -17,7 +17,7 @@
   if (window.AutoformNavigation) return;
 
   var HISTORY_KEY = "autoformNavigation", GRAPH_SCHEMA = "autoform-dag-view/v2", INDEX_SCHEMA = "autoform-dag-search/v1";
-  var WIKI_SCHEMA = "autoform-wiki-links/v1", WIKI_LIMIT = 32 * 1024 * 1024;
+  var WIKI_SCHEMA = "autoform-wiki-links/v1";
 
   function create(options) {
     options = options || {};
@@ -42,8 +42,6 @@
     var bound = new Map(), htmlCache = new Map(), jsonCache = new Map(), cacheBytes = 0;
     var decluttered = new WeakMap();
     var timeout = bounded(options.timeout, 15000, 1000, 60000);
-    var htmlLimit = bounded(options.htmlLimit, 8 * 1024 * 1024, 1024, 32 * 1024 * 1024);
-    var jsonLimit = bounded(options.jsonLimit, 64 * 1024 * 1024, 1024, 64 * 1024 * 1024);
     var cacheLimit = bounded(options.cacheBytes, 48 * 1024 * 1024, 1024, 128 * 1024 * 1024);
     var htmlEntries = bounded(options.htmlEntries, 8, 1, 32), jsonEntries = bounded(options.jsonEntries, 16, 1, 64);
     var sitePrefix = prefix(catalogueURL, initialDataURL, initialCanonical), homeURL = link(sitePrefix + "index.html", initialURL);
@@ -124,39 +122,38 @@
     }
     function captureIndexDigest(bytes) {
       // Hash the existing exact input before decoding; retain no JS byte buffers
-      // or closure over them. Native WebCrypto's one input copy is bounded.
+      // or closure over them.
       try {
         var subtle = win.crypto && win.crypto.subtle;
-        if (!subtle || typeof subtle.digest !== "function" || bytes.byteLength > WIKI_LIMIT) return Promise.resolve(null);
+        if (!subtle || typeof subtle.digest !== "function") return Promise.resolve(null);
         return subtle.digest("SHA-256", bytes).then(digestHex, digestUnavailable);
       } catch (_error) { return Promise.resolve(null); }
     }
-    async function fetchText(value, type, parentSignal, maximum) {
+    async function fetchText(value, type, parentSignal) {
       var href = link(value, initialURL); if (!href) throw new Error("Refusing an external data request");
       var key = resource(href), cache = type === "html" ? htmlCache : jsonCache;
-      var limit = type === "html" ? htmlLimit : jsonLimit;
-      if (Number.isSafeInteger(maximum)) limit = Math.min(limit, maximum);
       var cached = cacheGet(cache, key);
-      if (cached) { if (cached.bytes > limit) throw new Error("Published data is too large"); return cached; }
-      var controller = new AbortController(), expired = false;
+      if (cached) return cached;
+      var controller = new AbortController(), expired = false, timer = null;
       function abort() { controller.abort(); }
+      // The timeout measures a stall, not the whole transfer: each streamed chunk restarts it, so size never times a stream out.
+      // The whole-body fallbacks below, for peers without streams, run under a single timer.
+      function arm() { clearTimeout(timer); timer = setTimeout(function () { expired = true; controller.abort(); }, timeout); }
       if (parentSignal) {
         if (parentSignal.aborted) throw abortError();
         parentSignal.addEventListener("abort", abort, {once: true});
       }
-      var timer = setTimeout(function () { expired = true; controller.abort(); }, timeout);
+      arm();
       try {
         var response = await (options.fetch || win.fetch.bind(win))(key, {credentials: "same-origin", signal: controller.signal});
         if (!response.ok) throw new Error("HTTP " + response.status);
         var responseURL = link(response.url || key, key); if (!responseURL) throw new Error("Data redirected outside this site");
-        if (Number(response.headers.get("content-length")) > limit) throw new Error("Published data is too large");
         var text, length, digest = null, isCatalogue = type === "json" && catalogueURL && key === resource(catalogueURL);
         if (response.body && response.body.getReader) {
           var reader = response.body.getReader(), chunks = [], total = 0;
           while (true) {
             var chunk = await reader.read(); if (chunk.done) break;
-            total += chunk.value.byteLength;
-            if (total > limit) { await reader.cancel(); throw new Error("Published data is too large"); }
+            arm(); total += chunk.value.byteLength;
             chunks.push(chunk.value);
           }
           var bytes = new Uint8Array(total), offset = 0;
@@ -166,13 +163,11 @@
           bytes = null; chunks = null;
         } else if (isCatalogue && typeof response.arrayBuffer === "function") {
           var raw = new Uint8Array(await response.arrayBuffer());
-          if (raw.byteLength > limit) throw new Error("Published data is too large");
           digest = captureIndexDigest(raw);
           text = new TextDecoder("utf-8", {fatal: true}).decode(raw); length = raw.byteLength; raw = null;
         } else {
           // Older text-only peers remain readable, but cannot prove byte identity.
           text = await response.text(); length = new TextEncoder().encode(text).byteLength;
-          if (length > limit) throw new Error("Published data is too large");
         }
         if (parentSignal && parentSignal.aborted) throw abortError();
         var entry = {text: text, url: responseURL, bytes: length, indexDigest: digest};
@@ -330,8 +325,7 @@
       if (disposed) return Promise.resolve(null);
       if (wikiLinks && wikiLinks.availability !== "unavailable") return Promise.resolve(wikiLinks);
       if (wikiPromise) return wikiPromise;
-      var generation = ++wikiGeneration, controller = wikiController = new AbortController(), digest = null, expired = false;
-      var timer = setTimeout(function () { expired = true; controller.abort(); }, timeout);
+      var generation = ++wikiGeneration, controller = wikiController = new AbortController(), digest = null;
       function finish(value) {
         if (disposed || generation !== wikiGeneration) return null;
         if (value.availability === "unavailable" && wikiURL) cacheDelete(jsonCache, resource(wikiURL));
@@ -346,17 +340,17 @@
         if (!loadedIndex || !wikiURL) return finish(unavailableWiki("index_unavailable"));
         digest = await waitFor(indexDigest || Promise.resolve(null), controller.signal);
         if (!digest) return finish(unavailableWiki("index_unverified"));
-        var loaded = await waitFor(fetchText(wikiURL, "json", controller.signal, WIKI_LIMIT), controller.signal), payload;
+        var loaded = await waitFor(fetchText(wikiURL, "json", controller.signal), controller.signal), payload;
         try { payload = JSON.parse(loaded.text); } catch (_error) { return finish(unavailableWiki("invalid_layer", digest)); }
         // Map insertion order is the original validated offset identity, even
         // if a view reorders the public nodes array after receiving the index.
         var normalized = await normalizeWikiLinks(payload, digest, Array.from(indexed.keys()), controller.signal);
         if (loadedIndex !== index) return null;
         return finish(normalized);
-      })().catch(function () {
+      })().catch(function (error) {
+        var expired = error && error.message === "Published data request timed out";
         return finish(unavailableWiki(expired ? "request_timed_out" : "request_unavailable", digest));
       }).finally(function () {
-        clearTimeout(timer);
         if (generation === wikiGeneration) { wikiPromise = null; wikiController = null; }
       });
       return wikiPromise;

autoform_cli/dag_viewer.py

@@ -41,12 +41,9 @@
 SCHEMA = "autoform-dag-view/v2"
 SEARCH_SCHEMA = "autoform-dag-search/v1"
 WIKI_LINKS_SCHEMA = "autoform-wiki-links/v1"
-#: A separate optional reference layer must fit the reader's unchanged data limit.
-MAX_WIKI_LINK_BYTES = 32 * 1024 * 1024
-_SEARCH_INDEX_BYTE_LIMIT = 64 * 1024 * 1024
-# Above this size the whole-library index leaves out statement summaries,
-# which are a quarter of a large library's index; titles and Lean names stay.
-_SEARCH_INDEX_SUMMARY_BUDGET = 32 * 1024 * 1024
+#: Every explorer page downloads the whole search index, so publication notes
+#: when it passes this size. Nothing caps how large it may grow.
+_SEARCH_INDEX_NOTE_BYTES = 32 * 1024 * 1024
 _WIKI_BLOCK_MATH = re.compile(f"(?:{RE_DOLLAR_BLOCK}|{RE_BRACKET_BLOCK}|{RE_TEX_BLOCK})", re.DOTALL)
 _WIKI_INLINE_MATH = re.compile(f"(?:{RE_SMART_DOLLAR_INLINE}|{RE_BRACKET_INLINE})", re.DOTALL)
 _WIKI_INLINE_CODE = re.compile(BACKTICK_RE, re.DOTALL)
@@ -329,10 +326,8 @@ def write_search_index(
     and every typed edge without repeating long article identifiers.
     ``sha256_out`` optionally receives the exact emitted bytes' digest without
     rereading any source or changing the index's wire representation.
-    An index over ``_SEARCH_INDEX_SUMMARY_BUDGET`` bytes is written again
-    without statement summaries, so search keeps titles and Lean names.
-    ``warnings_out`` reports that omission, and when the emitted bytes reach
-    95% of the default browser limit, without rejecting publication.
+    ``warnings_out`` notes when the emitted bytes pass
+    ``_SEARCH_INDEX_NOTE_BYTES``, without refusing or trimming anything.
     """
     positions = {node.id: position for position, node in enumerate(view.nodes)}
     nodes: list[dict[str, object]] = []
@@ -374,24 +369,13 @@ def write_search_index(
             for edge in view.edges
         ]
     sizes: list[int] = []
-    digests = len(sha256_out) if sha256_out is not None else 0
-    written = _write_json(path, payload, sha256_out=sha256_out, byte_count_out=sizes)
-    if sizes[0] > _SEARCH_INDEX_SUMMARY_BUDGET and any("summary" in entry for entry in nodes):
-        full = sizes.pop()
-        for entry in nodes:
-            entry.pop("summary", None)
-        if sha256_out is not None:
-            del sha256_out[digests:]
-        written = _write_json(path, payload, sha256_out=sha256_out, byte_count_out=sizes)
-        if warnings_out is not None:
-            warnings_out.append(
-                f"search index leaves out statement summaries: with them it would use {full:,} bytes, over the "
-                f"{_SEARCH_INDEX_SUMMARY_BUDGET:,}-byte budget; search still matches titles and Lean names"
-            )
-    if warnings_out is not None and sizes[0] * 100 >= _SEARCH_INDEX_BYTE_LIMIT * 95:
+    written = _write_json(
+        path, payload, sha256_out=sha256_out, byte_count_out=sizes if warnings_out is not None else None,
+    )
+    if warnings_out is not None and sizes[0] > _SEARCH_INDEX_NOTE_BYTES:
         warnings_out.append(
-            f"search index uses {sizes[0]:,} bytes ({sizes[0] / _SEARCH_INDEX_BYTE_LIMIT:.1%} of the "
-            f"{_SEARCH_INDEX_BYTE_LIMIT:,}-byte default browser limit); browser loading fails above that limit"
+            f"search index is {sizes[0]:,} bytes; every explorer page downloads it, so first loads slow down "
+            f"past {_SEARCH_INDEX_NOTE_BYTES:,} bytes"
         )
     return written
 
@@ -604,7 +588,6 @@ def extract_wiki_links(
     references: set[tuple[int, int]] = set()
     invalid_targets: set[int] = set()
     omitted_sources = frozenset(unavailable_source_ids)
-    reference_bytes = 2
     for source_id in paths:
         if owners[paths[source_id]] is None:
             unavailable.add(positions[source_id])
@@ -648,11 +631,6 @@ def extract_wiki_links(
                 continue
             pair = (positions[source_id], positions[target_id])
             if pair not in references:
-                reference_bytes += len(str(pair[0])) + len(str(pair[1])) + 3 + bool(references)
-                if reference_bytes > MAX_WIKI_LINK_BYTES:
-                    return WikiLinks(
-                        (), (), f"wiki reference layer exceeds {MAX_WIKI_LINK_BYTES} bytes; no references were published",
-                    )
                 references.add(pair)
     return WikiLinks(
         tuple(sorted(pair for pair in references if pair[1] not in invalid_targets)),
@@ -667,7 +645,7 @@ def write_wiki_links(
     index_sha256: str,
     warnings_out: list[str] | None = None,
 ) -> Path:
-    """Publish a SHA-bound optional layer, explicitly omitting an oversized one.
+    """Publish a SHA-bound optional layer.
 
     The search index and its dependencies remain byte-for-byte unchanged. A
     reader must verify its exact index digest before using these offsets.
@@ -685,12 +663,7 @@ def write_wiki_links(
         payload["reason"] = "some captured article sources or publication routes were unavailable or unsupported"
     if links.reason is not None:
         payload.update(availability="unavailable", reason=links.reason)
-    if len(_json_text(payload).encode("utf-8")) > MAX_WIKI_LINK_BYTES:
-        reason = f"wiki reference layer exceeds {MAX_WIKI_LINK_BYTES} bytes; no references were published"
-        payload.update(availability="unavailable", reason=reason, references=(), unavailable_sources=())
-        if warnings_out is not None:
-            warnings_out.append(reason)
-    elif links.reason is not None and warnings_out is not None:
+    if links.reason is not None and warnings_out is not None:
         warnings_out.append(f"wiki reference layer unavailable: {links.reason}")
     return _write_json(path, payload)
 

autoform_cli/dashboard.py

@@ -16,7 +16,6 @@
 from .render import (
     LIVE_SCRIPT,
     PUBLICATION_MANIFEST,
-    PUBLICATION_MANIFEST_MAX_BYTES,
     PUBLICATION_SCHEMA,
     load_publication_source,
 )
@@ -167,10 +166,7 @@ def publication_bound_live_state(
     manifest_path = Path(site_dir) / PUBLICATION_MANIFEST
 
     def runtime() -> _PublicationLiveGraph:
-        encoded = manifest_path.read_bytes()
-        if len(encoded) > PUBLICATION_MANIFEST_MAX_BYTES:
-            raise ValueError("publication manifest is too large")
-        manifest = json.loads(encoded)
+        manifest = json.loads(manifest_path.read_bytes())
         revision, graph = load_publication_source(blueprint_dir)
         if (
             not isinstance(manifest, dict)

autoform_cli/graph.py

@@ -81,7 +81,6 @@
 _PROOF_SECTION = "proof depends on"
 _SOURCES_SECTION = "sources"
 ATLAS_SCHEMA = "autoform-atlas/v1"
-_MAX_ATLAS_BYTES = 1_000_000
 IMPLEMENTATION_NOTES_DIR = ".implementation-notes"
 
 
@@ -415,13 +414,11 @@ def _atlas_assignments(
         if not stat.S_ISREG(os.lstat(path).st_mode):
             return {}, ["ignored, since it is not a regular file inside the blueprint"]
         with path.open("rb") as handle:
-            raw = handle.read(_MAX_ATLAS_BYTES + 1)
+            raw = handle.read()
     except FileNotFoundError:
         return {}, []
     except OSError as error:
         return {}, [f"ignored, since it cannot be read: {error.strerror}"]
-    if len(raw) > _MAX_ATLAS_BYTES:
-        return {}, [f"ignored, since it exceeds {_MAX_ATLAS_BYTES} bytes"]
     try:
         payload = json.loads(raw)
     except (ValueError, RecursionError) as error:

autoform_cli/lean.py

@@ -174,8 +174,6 @@
 PUBLICATION_SCHEMA = "autoform-publication/v3"
 #: Sites render may replace: each records a file inventory to check first.
 _REPLACEABLE_SCHEMAS = frozenset({"autoform-publication/v2", PUBLICATION_SCHEMA})
-#: Render never writes a larger manifest, so the next render can always read it.
-PUBLICATION_MANIFEST_MAX_BYTES = 8 * 1024 * 1024
 _IGNORED_OUTPUT_SCHEMAS = MANAGED_OUTPUT_SCHEMAS | {("packets", REVIEW_PACKET_SCHEMA)}
 
 @dataclass(frozen=True, slots=True)
@@ -651,7 +649,7 @@ def _lean_tree_selection(
             # mkdocs can copy files into site/ before it copies the manifest.
             OpaqueDirectoryMarker(
                 PUBLICATION_MANIFEST,
-                PUBLICATION_MANIFEST_MAX_BYTES,
+                None,  # it lists every site file, so it grows with the project
                 _is_complete_publication,
             ),
         ),
@@ -1168,7 +1166,6 @@ def url(self, name: str) -> str | None:
     "PACKET_SCHEMA",
     "PASSAGE_SCHEMA",
     "PUBLICATION_MANIFEST",
-    "PUBLICATION_MANIFEST_MAX_BYTES",
     "PUBLICATION_SCHEMA",
     "REVIEW_PACKET_SCHEMA",
     "SourceIndex",