Skip to content
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
17 commits
Select commit Hold shift + click to select a range
4d6e472
Complete Appendix H recovery and local segment imports
arjanz Nov 9, 2026
0e78028
Checkpoint pending Chapter 14 implementation and regression sources
arjanz Nov 10, 2026
f19d311
Isolate unavailable source records and retain local report provenance
arjanz Nov 10, 2026
fb90f1b
Require known import sources before runtime refinement
arjanz Nov 10, 2026
5a61dee
Correct tagged import references to GP equation C.37
arjanz Nov 10, 2026
04a9af5
Bind replayed assurances to the matching source registration
arjanz Nov 10, 2026
2309d23
Bound local availability backfill and preserve retryable cache recovery
arjanz Nov 10, 2026
c64a886
Require explicit trusted finality before pruning availability data
arjanz Nov 10, 2026
99ef20c
Document local availability fixes and branch review history
arjanz Nov 10, 2026
7643a31
Defer contract tests for unimplemented batch CLI reporting
arjanz Nov 10, 2026
b585ea6
Initialize ancestry test sessions before importing blocks
arjanz Nov 10, 2026
7093eb4
Require committed state before availability finalization
arjanz Nov 11, 2026
8238e30
Backfill provenance indexes for retained local reports
arjanz Nov 11, 2026
2a2294f
Retry historical availability backfill when retained data returns
arjanz Nov 11, 2026
c017529
Revert "Retry historical availability backfill when retained data ret…
arjanz Nov 11, 2026
e0f6979
Revert "Backfill provenance indexes for retained local reports"
arjanz Nov 11, 2026
5758388
Revert "Require committed state before availability finalization"
arjanz Nov 11, 2026
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
39 changes: 39 additions & 0 deletions Dockerfile.py13
Original file line number Diff line number Diff line change
@@ -0,0 +1,39 @@
# ---------- builder ----------
FROM python:3.13-slim-bookworm AS builder
#FROM pypy:3.11-slim-bookworm AS builder
WORKDIR /app

# OS deps for building Rust/Python extensions (and RocksDB features if needed)
RUN apt-get update && apt-get install -y --no-install-recommends \
build-essential pkg-config curl ca-certificates \
libffi-dev libssl-dev \
clang cmake \
zlib1g-dev libbz2-dev liblz4-dev libsnappy-dev libzstd-dev \
&& rm -rf /var/lib/apt/lists/*

# Speed up pip, prefer wheels where possible
ENV PIP_NO_CACHE_DIR=1
COPY requirements.txt .
RUN pip install --upgrade pip wheel setuptools && \
# Try wheels first; fall back to source when necessary (Rust present)
pip wheel -r requirements.txt -w /wheels --prefer-binary

# ---------- runtime ----------
#FROM pypy:3.11-slim-bookworm
FROM python:3.13-slim-bookworm
WORKDIR /app

# Runtime libs only (openssl for some deps; compression libs if rocksdict links dynamically)
RUN apt-get update && apt-get install -y --no-install-recommends \
libssl3 zlib1g libbz2-1.0 liblz4-1 libsnappy1v5 libzstd1 \
&& rm -rf /var/lib/apt/lists/*

COPY --from=builder /wheels /wheels
RUN pip install /wheels/*

# Your app
COPY ./pyjamaz ./pyjamaz
RUN python -m compileall -b ./pyjamaz && find ./pyjamaz -name "*.py" -delete

ENV PYTHONUNBUFFERED=1 PYTHONDONTWRITEBYTECODE=1 PYTHONPATH=/app
ENTRYPOINT ["python","-m","pyjamaz.cli"]
41 changes: 41 additions & 0 deletions Dockerfile.pypy
Original file line number Diff line number Diff line change
@@ -0,0 +1,41 @@
# ---------- builder ----------
FROM pypy:3.11-slim-bookworm AS builder
WORKDIR /app

# OS deps for building Rust/Python extensions (and RocksDB features if needed)
RUN apt-get update && apt-get install -y --no-install-recommends \
build-essential pkg-config curl ca-certificates \
libffi-dev libssl-dev \
clang cmake \
zlib1g-dev libbz2-dev liblz4-dev libsnappy-dev libzstd-dev \
&& rm -rf /var/lib/apt/lists/*

# Install Rust (new enough for maturin/cryptography; minimal profile)
RUN curl -sSf https://sh.rustup.rs | sh -s -- -y --profile minimal
ENV PATH="/root/.cargo/bin:${PATH}"

# Speed up pip, prefer wheels where possible
ENV PIP_NO_CACHE_DIR=1
COPY requirements.txt .
RUN pypy3 -m pip install --upgrade pip wheel setuptools && \
# Try wheels first; fall back to source when necessary (Rust present)
pypy3 -m pip wheel -r requirements.txt -w /wheels --prefer-binary

# ---------- runtime ----------
FROM pypy:3.11-slim-bookworm
WORKDIR /app

# Runtime libs only (openssl for some deps; compression libs if rocksdict links dynamically)
RUN apt-get update && apt-get install -y --no-install-recommends \
libssl3 zlib1g libbz2-1.0 liblz4-1 libsnappy1v5 libzstd1 \
&& rm -rf /var/lib/apt/lists/*

COPY --from=builder /wheels /wheels
RUN pypy3 -m pip install /wheels/*

# Your app
COPY ./pyjamaz ./pyjamaz
RUN pypy3 -m compileall -b ./pyjamaz && find ./pyjamaz -name "*.py" -delete

ENV PYTHONUNBUFFERED=1 PYTHONDONTWRITEBYTECODE=1 PYTHONPATH=/app
ENTRYPOINT ["pypy3","-m","pyjamaz.cli"]
237 changes: 237 additions & 0 deletions docs/reports/chapter-14-appendix-h-analysis-2026-11-09.md

Large diffs are not rendered by default.

8 changes: 8 additions & 0 deletions docs/stfs/psi.md
Original file line number Diff line number Diff line change
@@ -1 +1,9 @@
Offender evidence is checked against the prior punish-set (GP 0.8.0 equations
10.6--10.7). The culprit and fault lists must each have unique keys (10.9),
but the same validator may appear in both lists in one block.

The posterior punish-set contains that key once (10.18). The header offenders
marker concatenates culprit keys followed by fault keys, preserving the key's
two occurrences and the order of each list (10.19).

::: pyjamaz.state.components.Disputes
105 changes: 105 additions & 0 deletions docs/stfs/rho.md
Original file line number Diff line number Diff line change
@@ -1 +1,106 @@
::: pyjamaz.state.components.Assurances

Each guaranteed work report must have
`auth_gas_used <= GAS_INVOKE` (50,000,000). An excess is rejected with
`authorizer_gas_too_high` before the report is installed. The budget applies
independently to each report; exactly the budget is permitted. The encoded
field remains an unsigned 64-bit gas value.
Error code 28 is a PyJAMaz extension to the official reports STF vocabulary;
the fuzzer protocol transmits its name as an error string.

GP B.1 invokes Is-Authorized with `G_I`, and A.48 ensures consumed gas cannot
exceed the supplied budget. PyJAMaz applies this derived sanity bound during
report admission. Accumulation gas is checked separately under 11.33.

Refinement-load totals are checked independently for each guaranteed report,
including all digests whose refinement result is an error:

- Refinement gas used must be at most the profile's `G_R`.
- Imports and exports must each total at most 3,072 (`W_M` and `W_X`).
- Extrinsics must total at most 128 (`T`).
- `4,488 * total_imports + total_extrinsic_size` must be at most 13,791,360
(`W_I`, named in PR #563). The import footprint `W_F` is computed from
GP-0.8.0-eq:14.7 using the logical segment size, independently of
erasure-shard dimensions.

Violations return `refine_load_too_high` (PyJAMaz error extension 29). These
are report-admission sanity checks derived from package limits in 14.5-14.9,
the item-to-digest mapping in 14.10, report construction in 14.13, and the
invocation gas bound in B.5/A.48. The footprint is only the report-visible
lower bound of 14.6: payload, configuration and authorization-token lengths
are absent from the report. The refinement-load check does not validate serialized bundle
length or package/digest export-count equality.

For Chapter 14, the specification order is the changes, fixes and stated
deviations in the [v0.8.0 conformance release](https://github.com/jam-conformance/jam-test-vectors/releases/tag/v0.8.0),
pinned to commit `d3c089c9a1d5352ad353626d7a2db8589f137b2a`; the additional
selected [graypaper PR #563](https://github.com/gavofyork/graypaper/pull/563)
exception for inclusive gas limits and the existing footprint/bundle bounds;
then GP 0.8.0 for the remaining behavior. Networking delegated by the GP follows
[JAMNP-S v1 at e0b7d291](https://github.com/zdave-parity/jam-np/blob/e0b7d2910afc0a7406043a4c79828edbcc357c2e/simple.md).

PyJAMaz follows [graypaper PR #563](https://github.com/gavofyork/graypaper/pull/563)
for inclusive gas limits. Before authorization or refinement, the report producer
checks that declared budgets across all items total at most `G_R` for refinement
and `G_A` for accumulation. This updates the strict inequalities in
GP-0.8.0-eq:14.9. GP-0.8.0-eq:B.5 / GP-0.8.0-eq:A.48 bound consumption by
those budgets, so reported refinement gas may equal `G_R`.

Report admission rejects
`package_spec.length > MAXIMUM_SIZE_AUDITABLE_BUNDLE` with
`bundle_length_too_high` (PyJAMaz error extension 30). The conservative
serialized-bundle ceiling is 13,904,950 bytes, inclusive (`W_B` in PR #563),
derived as follows:

| Component | Maximum bytes | GP reference |
| --- | ---: | --- |
| Input footprint (`W_I`) | 13,791,360 | 14.6-14.8 |
| Authorization service and code hash | 36 | C.31 |
| Context including eight prerequisites | 425 | C.26, 14.4 |
| Token/configuration prefixes and item count | 9 | C.5, C.7, C.31 |
| Sixteen items' fixed fields and blob/vector prefixes | 992 | C.32 |
| 3,072 import references and proof-length prefixes | 107,520 | C.37, 14.15-14.16 |
| 128 extrinsic manifests | 4,608 | C.32 |

The footprint already allows 4,104 bytes of data and twelve 32-byte proof
hashes per import through `W_F = 4,488`; those bytes are not counted again.
The additional bytes cover metadata omitted from the footprint, including
one prefix per import's proof path. The prefix allowances are conservative
maxima under 14.4-14.8 and C.5. This ceiling includes metadata beyond `W_I`,
as specified by PR #563. It uses package-wide maxima independently of reported
load counters.

Equation 14.18 records the unpadded serialized bundle length defined by
14.16. Erasure padding, shard sizes, exported segments and their paged proofs
do not increase this field. Applying the input limit as `length <= W_I` is too strict:
a single `W_I`-byte payload with empty token/configuration/manifests satisfies
GP-0.8.0-eq:14.6 yet serializes to `W_I + 268` bytes. A test also covers all
sixteen items, 3,072 imports, 128 extrinsic manifests and eight prerequisites
at the content footprint limit; its encoded bundle length is 13,904,899 bytes.

These admission checks enforce necessary bounds derived from honest package
and report construction. They do not replace report reproduction and auditing
under 17.16.

Each report must also satisfy
`package_spec.exports_count == sum(result.refine_load.exports for result in results)`.
Mismatches are rejected with `exports_count_mismatch` (PyJAMaz error extension
31), after the other guarantee checks and before installing any report. The
sum is per report and includes failed results. The refinement-load export cap
in 14.5 applies separately; equality with the digest total is necessary even
when the declared count is within the 3,072-segment limit.

Equation 14.10 copies each work-item's declared export count into its digest.
Equation 14.13 supplies exactly that many segments, replacing exports on error
with zero segments of the declared count. Equations 14.17-14.18 set the
availability count to the length of their concatenation. The report producer
therefore pads failures to `work_item.export_count`, including underproduction,
overproduction, execution failure and oversized output. This also preserves
later items' segment positions.

The canonical report vectors contain inconsistent synthetic export metadata:
14 accepted Tiny vectors (21 reports) and 16 Full vectors (23 reports) expect
acceptance despite count mismatches. They remain unchanged. The tests first
assert rejection with `exports_count_mismatch`, then report their original
acceptance expectations as known conflicts (`xfail`). Synthetic boundary tests
use coherent counts and re-sign their modified reports. See `test/fixtures/SOURCE.md`.
62 changes: 58 additions & 4 deletions pyjamaz/app.py
Original file line number Diff line number Diff line change
Expand Up @@ -669,6 +669,16 @@ async def state_transition(self, block: 'Block', produce=False) -> 'STFOutput':
self.state_storage.commit()

self.working_state.state_root = self.state_storage.state_root()
availability = getattr(getattr(self, 'runtime', None), 'availability', None)
if availability is not None and not SKIP_VALIDATE_GUARANTEES:
try:
# Chapter 14 routing is published only after the guarantee's block commits.
availability.record_committed(block, validator_pool_output.post_state.validators,
validator_archive_output.post_state.validators,
self.block_context, assurances_after_assurances_output.reported)
except Exception:
# An off-chain storage failure must not turn a committed STF into a rejection.
logging.exception('Availability metadata could not be retained; backfill will retry')
signals = tuple(self.app_context.post_commit_signals)
self.app_context.post_commit_signals.clear()
if self.app_context.pubsub:
Expand Down Expand Up @@ -1021,14 +1031,56 @@ async def produce_block(

return block

async def finalize(self, header_hash: bytes):
if header_hash != self.state_storage.finalized_block_hash:
async def finalize(self, header_hash: bytes, *, availability_finality_confirmed: bool = False):
"""Finalize storage; prune availability only after an explicit trusted confirmation.

The current producer/importer calls finalize on parents as a local convention.
Those calls do not establish the finality required by GP 14.3.1 for deleting
audit data. A future finality integration must supply the confirmation.
"""
already_finalized = False
if availability_finality_confirmed:
if self.state_storage.transaction:
raise ValueError('availability finality requires committed state')
header = self.retrieve_block_header(header_hash)
if header is None or header.hash != header_hash:
raise ValueError('availability finality requires a retained committed header')
# An explicit confirmation must not turn an orphan or pending candidate
# into the canonical availability-retention checkpoint.
cursor = self.state_storage.block_hash or self.state_storage.finalized_block_hash
visited = set()
while cursor is not None and cursor not in visited:
if cursor == self.state_storage.finalized_block_hash:
already_finalized = True
if cursor == header_hash:
break
visited.add(cursor)
header = self.retrieve_block_header(cursor)
if header is not None and header.hash != cursor:
raise ValueError('availability finality requires valid ancestry headers')
cursor = header.parent if header is not None else self.state_storage.parents.get(cursor)
if cursor != header_hash:
raise ValueError('availability finality requires the active branch')
if not already_finalized and not self.state_storage.has_state_at(header_hash):
raise ValueError('availability finality requires a retained committed header')

# Real confirmation can lag the producer's local finalization convention.
# Its canonical ancestors remain eligible without rewinding storage finality.
if header_hash != self.state_storage.finalized_block_hash and not already_finalized:
self.state_storage.finalize(header_hash)
await self.store_finalized_head(header_hash)
logging.info(f'🔒 Finalized block {format_hash(header_hash)}')
else:
DEBUG and logging.debug(f'Skipped finalization, {format_hash(header_hash)} already finalized')

if availability_finality_confirmed:
availability = getattr(getattr(self, 'runtime', None), 'availability', None)
if availability is not None and self.d3l_store is not None:
try:
availability.on_finalized(header_hash)
except Exception:
logging.exception('Availability retention update failed')

async def create_state_dump(self) -> StateDump:
return StateDump(
state_root=self.working_state.state_root,
Expand Down Expand Up @@ -1083,9 +1135,11 @@ def get_other_guarantors(self):
return guarantors

# TODO deprecated
def add_work_package(self, work_package: WorkPackage, extrinsics: List[bytes]):
def add_work_package(self, work_package: WorkPackage, extrinsics: List[bytes], *,
core_index: int | None = None, bundle: bytes | None = None):

self.runtime.refine_pipeline.add_work_package(work_package, extrinsics)
self.runtime.refine_pipeline.add_work_package(work_package, extrinsics,
core_index=core_index, bundle=bundle)

async def guarantee_work_report(self, work_report: WorkReport, timeslot: int):

Expand Down
Loading
Loading