From a6b03d603ae3936924a8c357f98c335e82d7003e Mon Sep 17 00:00:00 2001 From: Arjan Zijderveld <5286904+arjanz@users.noreply.github.com> Date: Sun, 25 Oct 2026 21:47:20 +0100 Subject: [PATCH 1/4] Implement GP-aligned refinement and fuzzer target support Share work-report computation and availability commitments between the node runtime and fuzzer. Add detached state snapshots, dependency retries, protocol negotiation, and focused regression coverage. --- pyjamaz/app.py | 219 +----------- pyjamaz/availability.py | 209 +++++++++++ pyjamaz/cli.py | 6 + pyjamaz/d3l.py | 17 + pyjamaz/erasure.py | 27 ++ pyjamaz/fuzzer/v1/session.py | 28 +- pyjamaz/fuzzer/v1/target.py | 71 +++- pyjamaz/fuzzer/v1/types.py | 64 +++- pyjamaz/merkle.py | 29 ++ pyjamaz/models/common.py | 39 +-- pyjamaz/models/state.py | 11 + pyjamaz/refinement.py | 90 +++++ pyjamaz/runtime/pipelines/refine.py | 218 +++++++----- pyjamaz/settings.py | 1 + requirements.txt | 1 + test/test_fuzzer_refinement.py | 517 ++++++++++++++++++++++++++++ test/test_runtime_refinement.py | 250 ++++++++++++++ test/test_work_report_resize.py | 90 +++++ 18 files changed, 1546 insertions(+), 341 deletions(-) create mode 100644 pyjamaz/availability.py create mode 100644 pyjamaz/erasure.py create mode 100644 pyjamaz/refinement.py create mode 100644 test/test_fuzzer_refinement.py create mode 100644 test/test_runtime_refinement.py create mode 100644 test/test_work_report_resize.py diff --git a/pyjamaz/app.py b/pyjamaz/app.py index fa3b4432..350ec386 100644 --- a/pyjamaz/app.py +++ b/pyjamaz/app.py @@ -7,10 +7,9 @@ import typing from dataclasses import dataclass, field from functools import partial -from typing import TypeVar, Optional, List, Callable, Dict, Tuple +from typing import TypeVar, Optional, List, Callable -import anyio -from bandersnatch_vrfs import ietf_vrf_sign, RingContext +from bandersnatch_vrfs import ietf_vrf_sign from jamcodec.base import JamBytes from jamcodec.mixins import Serializable @@ -19,19 +18,14 @@ from pyjamaz.constants import MESSAGE_TYPES from pyjamaz.d3l import DataAvailabilityStore -from pyjamaz.exceptions import PyjamazAppError, ProcessWorkpackageError, StateTransitionError, \ +from pyjamaz.exceptions import PyjamazAppError, StateTransitionError, \ BlockValidationErrorCode, BlockValidationError -from pyjamaz.runtime.extrinsics import BlockExtrinsicCollector, WorkpackageExtrinsicCollector +from pyjamaz.runtime.extrinsics import BlockExtrinsicCollector from pyjamaz.graypaper_constants import CORE_COUNT, EPOCH_TIMESLOTS, \ - SLOT_PERIOD, MAXIMUM_AGE_LOOKUP_ANCHOR, MAXIMUM_SIZE_ENCODED_WORK_REPORT, EC_SEGMENT_SIZE + SLOT_PERIOD, MAXIMUM_AGE_LOOKUP_ANCHOR from pyjamaz.hashing import blake2b_256_hash -from pyjamaz.hostcalls.invocation import pvm_invoke_is_authorized, pvm_invoke_refine -from pyjamaz.merkle import ConstantDepthMerkleTree -from pyjamaz.models.app import StateDump, Trace, D3LEntry -from pyjamaz.models.common import WorkPackage, WorkReport, WorkPackageBundle, WorkPackageStatus, \ - WorkPackageReportableStatus, WorkPackageReadyStatus, BlockDesc, WorkPackageReportedStatus, WorkExecResult, \ - WorkDigest, WorkPackageSpec -from pyjamaz.runtime.types import WorkPackageQueueItem +from pyjamaz.models.app import StateDump, Trace +from pyjamaz.models.common import WorkPackage, WorkReport from pyjamaz.settings import SOLO_MODE, DEBUG, SKIP_VALIDATE_GUARANTEES from pyjamaz.signing import Ed25519Keypair, BandersnatchKeypair from pyjamaz.models.context import AppContext, BlockContext @@ -46,7 +40,7 @@ from pyjamaz.models.state import JamState, ServicesState, SafroleState, EntropyState, PendingChanges from pyjamaz.models.stf_output import STFOutput from pyjamaz.transport.pubsub import PubSub, PubSubSignal -from pyjamaz.utils import vrf_input_fallback_seal, vrf_input_ticket_seal, format_hash, log_execution_time, flatten_list +from pyjamaz.utils import vrf_input_fallback_seal, vrf_input_ticket_seal, format_hash, log_execution_time from pyjamaz.validation import BlockValidation T = TypeVar('T') @@ -1087,33 +1081,6 @@ def add_work_package(self, work_package: WorkPackage, extrinsics: List[bytes]): self.runtime.refine_pipeline.add_work_package(work_package, extrinsics) - # TODO deprecated - # async def process_work_package(self, work_package: WorkPackage) -> WorkReport: - # if self.get_core_assigment() is None: - # raise ProcessWorkpackageError("Cannot process work package: no core assignment") - # - # # Prepare extrinsic data (GP-0.7.2-eq:B.6 bold_x_flat) - # extrinsics = [ - # [self.work_package_extrinsics.get(work_package, x.hash, x.len) for x in w.extrinsic] - # for w in work_package.items - # ] - # - # # Set code - # work_package.set_authorization_code(self.working_state.services) - # - # work_report = await anyio.to_thread.run_sync( - # self.work_result_computation, - # work_package, - # self.get_core_assigment(), - # self.working_state.services, - # extrinsics, - # ) - # - # # Clean up work package extrinsics - # self.work_package_extrinsics.clear(work_package) - # DEBUG and logging.debug(f"Processed work package: {format_hash(work_package.hash())}") - # return work_report - async def guarantee_work_report(self, work_report: WorkReport, timeslot: int): credential = await self.create_guarantee_signature(work_report) @@ -1200,176 +1167,6 @@ def create_assurance_for_validator_index(self, cores: List[int], validator_index ) - # async def process_refine(self, timeslot: int) -> Optional[WorkReport]: - # """ - # Process queued work packages and guarantee work reports. - # """ - # - # wp_queue_item = None - # cleanup_queue = [] - # - # # Clean up work packages - # for h, w in self.work_package_queue.items(): - # if not self.working_state.recent_history.get_recent_block(w.work_package.context.lookup_anchor): - # cleanup_queue.append(h) - # - # for h in cleanup_queue: - # del self.work_package_queue[h] - # logging.info(f"πŸ—‘οΈ Discarded outdated work package {format_hash(h)}") - # - # # Find first authorized work package - # for h, w in self.work_package_queue.items(): - # if w.status.enum_value()[0] == 'Reportable': - # if self.working_state.authorizer_pools.is_authorized(w.work_package, self.get_core_assigment()): - # wp_queue_item = self.work_package_queue[h] - # break - # - # if wp_queue_item is None: - # # No reportable work package found, return - # return None - # - # try: - # work_report = await self.process_work_package(wp_queue_item.work_package) - # # Update WorkPackage status - # wp_queue_item.status = WorkPackageStatus(Reported=WorkPackageReportedStatus( - # reported_in=BlockDesc( - # slot=self.working_state.timeslot.number, - # header_hash=self.retrieve_block_hash(self.working_state.timeslot.number) - # ), - # core=self.get_core_assigment(), - # report_hash=work_report.hash() - # )) - # if self.app_context.pubsub: - # # Send signal - # await self.app_context.pubsub.publish( - # PubSubSignal( - # topic=MESSAGE_TYPES.WORK_PACKAGE_STATUS, - # data=[wp_queue_item.status.to_json()] - # ) - # ) - # await self.guarantee_work_report(work_report, timeslot) - # return work_report - # - # except ProcessWorkpackageError as e: - # # Update WorkPackage status - # wp_queue_item.status = WorkPackageStatus(Failed=str(e)) - # if self.app_context.pubsub: - # await self.app_context.pubsub.publish( - # PubSubSignal( - # topic=MESSAGE_TYPES.WORK_PACKAGE_STATUS, - # data=[wp_queue_item.status.to_json()] - # ) - # ) - # logging.error(f"Error processing work package {format_hash(wp_queue_item.work_package.hash())}: {e}") - # return None - - def work_result_computation( - self, - work_package: WorkPackage, - core_index: int, - services_state: ServicesState, - extrinsics: List[List[bytes]], - erasure_shards: int, -) -> WorkReport: - """ - GP-0.7.2-eq:14.12 (function Ξ) | the work result computation function. - - TODO finish - """ - - segment_root_lookup_keys = {h for w in work_package.items for (h, n) in w.import_segments} - - # Collect import segments - import_segments = [self.d3l_store.retrieve_segments(r).segments for r in segment_root_lookup_keys] - - auth_output = pvm_invoke_is_authorized(work_package, core_index) - - if type(auth_output.work_exec_result.ok) is not bytes: - raise ProcessWorkpackageError("Unauthorized") - - if len(auth_output.work_exec_result.ok) > MAXIMUM_SIZE_ENCODED_WORK_REPORT: - raise ProcessWorkpackageError("Oversized auth result") - - refine_outputs: List[Tuple[WorkDigest, List[bytes]]] = [] - - total_digest_size = len(auth_output.work_exec_result.ok) - - for j in range(len(work_package.items)): - - work_item = work_package.items[j] - - export_segment_offset = sum([w.export_count for k, w in enumerate(work_package.items) if k < j]) - - refine_output = pvm_invoke_refine( - core_index=core_index, - work_item_index=j, - work_package=work_package, - authorizer_output=auth_output.work_exec_result.ok, - work_items_import_segments=import_segments, - export_segment_offset=export_segment_offset, - services_state=services_state, - extrinsics=extrinsics - ) - - if total_digest_size + len(refine_output.work_exec_result.ok or b'') > MAXIMUM_SIZE_ENCODED_WORK_REPORT: - work_exec_result = WorkExecResult(digest_oversize=True) - export_segments = [bytes(EC_SEGMENT_SIZE)] * len(refine_output.export_segments) - - elif len(refine_output.export_segments) != work_item.export_count: - work_exec_result = WorkExecResult(bad_exports=True) - export_segments = [bytes(EC_SEGMENT_SIZE)] * len(refine_output.export_segments) - - elif refine_output.work_exec_result.ok is None: - work_exec_result = refine_output.work_exec_result - export_segments = [bytes(EC_SEGMENT_SIZE)] * len(refine_output.export_segments) - - else: - work_exec_result = refine_output.work_exec_result - export_segments = refine_output.export_segments - total_digest_size += len(refine_output.work_exec_result.ok) - - work_result = WorkDigest.from_work_item( - work_item=work_package.items[j], - result=work_exec_result, - gas_used=refine_output.gas_used - ) - - refine_outputs.append((work_result, export_segments)) - - # TODO inefficient: refactor refine_outputs to work_results and all_export_segments ? - all_export_segments: list[bytes] = flatten_list([o[1] for o in refine_outputs]) - - exports_root = ConstantDepthMerkleTree(all_export_segments).root() - - # store segments - # self.d3l_store[exports_root] = all_export_segments - # Also store under work-package hash (H^+) TODO check? - # self.d3l_store[work_package.hash()] = all_export_segments - - if self.d3l_store and len(all_export_segments) > 0: - d3l_item = D3LEntry( - segments=all_export_segments, - segment_root=exports_root - ) - self.d3l_store.store_segments(d3l_item) - - - package_spec = WorkPackageSpec.create_from_work_package( - work_package, [], [], [], all_export_segments, exports_root, - erasure_shards=erasure_shards, - ) - - return WorkReport( - package_spec=package_spec, - context=work_package.context, - core_index=core_index, - authorizer_hash=work_package.authorizer_hash(), - auth_output=auth_output.work_exec_result.ok, - segment_root_lookup={}, # TODO - results=[o[0] for o in refine_outputs], - auth_gas_used=auth_output.gas_used - ) - async def process_assurances(self): """ Check for active assignments and create assurances diff --git a/pyjamaz/availability.py b/pyjamaz/availability.py new file mode 100644 index 00000000..6d0d6381 --- /dev/null +++ b/pyjamaz/availability.py @@ -0,0 +1,209 @@ +"""GP 0.8.0 bundle assembly and availability commitments (section 14).""" + +from dataclasses import dataclass, field + +from jamcodec.types import H256, Vec + +from pyjamaz.erasure import encode_chunks, erasure_data_shards +from pyjamaz.exceptions import ProcessWorkpackageError +from pyjamaz.graypaper_constants import ( + EC_SEGMENT_SIZE, GAS_ACCUMULATION, GAS_REFINE, MAXIMUM_DEPENDENCIES_WORK_REPORT, + MAXIMUM_NUMBER_EXPORTS_WORK_PACKAGE, MAXIMUM_NUMBER_EXTRINSICS_WORK_PACKAGE, + MAXIMUM_NUMBER_IMPORTS_WORK_PACKAGE, MAXIMUM_SIZE_WORK_PACKAGE, MAXIMUM_WORK_ITEMS, +) +from pyjamaz.hashing import blake2b_256_hash +from pyjamaz.merkle import ConstantDepthMerkleTree, WellBalancedMerkleTree +from pyjamaz.models.common import ImportSegment, WorkItem, WorkPackage, WorkPackageSpec + + +class DataUnavailable(ProcessWorkpackageError): + """A runtime dependency may become available on a later attempt.""" + + +def validate_work_package(package: WorkPackage) -> None: + # GP-0.8.0-eq:14.2-14.9: package domain, dependencies, bandwidth, footprint and gas. + if not 1 <= len(package.items) <= MAXIMUM_WORK_ITEMS: + raise ProcessWorkpackageError("invalid work-item count") + totals = ( + (sum(len(w.extrinsic) for w in package.items), MAXIMUM_NUMBER_EXTRINSICS_WORK_PACKAGE), + (sum(len(w.import_segments) for w in package.items), MAXIMUM_NUMBER_IMPORTS_WORK_PACKAGE), + (sum(w.export_count for w in package.items), MAXIMUM_NUMBER_EXPORTS_WORK_PACKAGE), + ) + if any(count > limit for count, limit in totals): + raise ProcessWorkpackageError("work-package manifest exceeds its limits") + dependencies = { + entry.tree_root for item in package.items for entry in item.import_segments + if entry.is_work_package_hash + } + if len(dependencies) + len(package.context.prerequisites) > MAXIMUM_DEPENDENCIES_WORK_REPORT: + raise ProcessWorkpackageError("too many work-package dependencies") + import_footprint = EC_SEGMENT_SIZE + 32 * (MAXIMUM_NUMBER_EXPORTS_WORK_PACKAGE - 1).bit_length() + footprint = len(package.authorization) + len(package.authorizer_config) + sum( + len(w.payload) + len(w.import_segments) * import_footprint + sum(x.len for x in w.extrinsic) + for w in package.items + ) + if footprint > MAXIMUM_SIZE_WORK_PACKAGE: + raise ProcessWorkpackageError("work-package footprint exceeds its limit") + if (sum(w.refine_gas_limit for w in package.items) >= GAS_REFINE + or sum(w.accumulate_gas_limit for w in package.items) >= GAS_ACCUMULATION): + raise ProcessWorkpackageError("work-package gas limits exceed core limits") + + + +@dataclass(frozen=True) +class WorkPackageHash: + """The tagged H+ import-reference variant from GP 14.3 and C.35.""" + + value: bytes + + +def import_reference(entry: ImportSegment) -> bytes | WorkPackageHash: + # GP-0.8.0-eq:C.35: the upper index bit distinguishes H+ from a segment root. + return WorkPackageHash(entry.tree_root) if entry.is_work_package_hash else entry.tree_root + + +def resolve_import_root(mappings: dict[bytes, bytes], reference: bytes | WorkPackageHash) -> bytes: + # GP-0.8.0-eq:14.15: L_l(r): H union H+ -> H, parameterized by l. + if isinstance(reference, WorkPackageHash): + try: + return mappings[reference.value] + except KeyError as exc: + raise ProcessWorkpackageError("missing segment-root mapping") from exc + return reference + + +@dataclass(frozen=True) +class ImportedSegment: + segment: bytes + justification: tuple[bytes, ...] + + +@dataclass +class BundleData: + """The data lookups assumed by X, S_l and J_l; no storage or network access.""" + + extrinsics: dict[bytes, bytes] = field(default_factory=dict) + imports: dict[tuple[bytes, int], ImportedSegment] = field(default_factory=dict) + + @classmethod + def from_parts(cls, package: WorkPackage, extrinsics: list[bytes], segments: list[bytes], + justifications: list[list[bytes]]) -> "BundleData": + """Adapt the fuzzer's flat wire sequences, checking their manifest cardinalities.""" + manifests = [x for item in package.items for x in item.extrinsic] + imports = [entry for item in package.items for entry in item.import_segments] + if len(extrinsics) != len(manifests): + raise ProcessWorkpackageError("extrinsic count does not match manifest") + if len(segments) != len(imports) or len(justifications) != len(imports): + raise ProcessWorkpackageError("import or justification count does not match manifest") + data = cls() + for manifest, blob in zip(manifests, extrinsics): + if len(blob) != manifest.len or blake2b_256_hash(blob) != manifest.hash: + raise ProcessWorkpackageError("extrinsic hash or length does not match manifest") + data.extrinsics[manifest.hash] = bytes(blob) + for entry, segment, proof in zip(imports, segments, justifications): + key = (entry.tree_root, entry.index) + imported = ImportedSegment(bytes(segment), tuple(bytes(h) for h in proof)) + if key in data.imports and data.imports[key] != imported: + raise ProcessWorkpackageError("conflicting data for duplicate imports") + data.imports[key] = imported + return data + + def extrinsic_data(self, item: WorkItem) -> list[bytes]: + # GP-0.8.0-eq:14.15: X(w) = [d | (H(d), |d|) in w_x], in manifest order. + blobs = [] + for manifest in item.extrinsic: + blob = self.extrinsics.get(manifest.hash) + if blob is None or len(blob) != manifest.len or blake2b_256_hash(blob) != manifest.hash: + raise ProcessWorkpackageError("extrinsic data is unavailable or invalid") + blobs.append(blob) + return blobs + + def _verified_imports(self, mappings: dict[bytes, bytes], item: WorkItem) -> list[ImportedSegment]: + imports = [] + for entry in item.import_segments: + imported = self.imports.get((entry.tree_root, entry.index)) + if imported is None: + raise ProcessWorkpackageError("import segment is unavailable") + proof = imported.justification + if (len(imported.segment) != EC_SEGMENT_SIZE + or entry.segment_index >= MAXIMUM_NUMBER_EXPORTS_WORK_PACKAGE + or len(proof) > (MAXIMUM_NUMBER_EXPORTS_WORK_PACKAGE - 1).bit_length() + or not ConstantDepthMerkleTree.verify( + imported.segment, entry.segment_index, proof, + resolve_import_root(mappings, import_reference(entry)), + )): + raise ProcessWorkpackageError("invalid import segment justification") + imports.append(imported) + return imports + + def import_segments(self, mappings: dict[bytes, bytes], item: WorkItem) -> list[bytes]: + # GP-0.8.0-eq:14.15: S_l(w) = [b[n] | M(b) = L_l(r), (r,n) in w_i]. + return [entry.segment for entry in self._verified_imports(mappings, item)] + + def import_justifications(self, mappings: dict[bytes, bytes], item: WorkItem) -> list[tuple[bytes, ...]]: + # GP-0.8.0-eq:14.15: J_l(w) = [var(J_0(b,n)) | M(b) = L_l(r), (r,n) in w_i]. + return [entry.justification for entry in self._verified_imports(mappings, item)] + + def assemble_bundle(self, package: WorkPackage, mappings: dict[bytes, bytes]) -> bytes: + # GP-0.8.0-eq:14.16: B(p,l) = E(p, X#(p_w), S_l#(p_w), J_l#(p_w)). + output = bytearray(package.to_jam_bytes().to_bytes()) + for item in package.items: + for blob in self.extrinsic_data(item): + output.extend(blob) + for item in package.items: + for segment in self.import_segments(mappings, item): + output.extend(segment) + for item in package.items: + for proof in self.import_justifications(mappings, item): + output.extend(Vec(H256).encode(list(proof)).to_bytes()) + return bytes(output) + + +def zero_pad(alignment: int, data: bytes) -> bytes: + # GP-0.8.0-eq:14.19: P_n(x) pads x to a multiple of n octets. + if alignment < 1: + raise ValueError("padding alignment must be positive") + return data + bytes(-len(data) % alignment) + + +def paged_proof_segments(exports: list[bytes]) -> list[bytes]: + # GP-0.8.0-eq:14.12: P(s) = [P_WG(E(var(J_6(s,i)), var(L_6(s,i))))]. + tree = ConstantDepthMerkleTree(exports) + hashes = [blake2b_256_hash(b'leaf' + segment) for segment in exports] + page_depth = max(0, (len(exports) - 1).bit_length() - 6) + pages = [] + for start in range(0, len(exports), 64): + path = tree.trace(start)[:page_depth] + encoded = Vec(H256).encode(path).to_bytes() + Vec(H256).encode(hashes[start:start + 64]).to_bytes() + pages.append(zero_pad(EC_SEGMENT_SIZE, encoded)) + return pages + + +def availability_spec(package_hash: bytes, bundle: bytes, exports: list[bytes], + validator_count: int) -> WorkPackageSpec: + # GP-0.8.0-eq:14.18: A(h,b,s,v) -> (h, |b|, u, v, M(s), |s|). + if not isinstance(package_hash, bytes) or len(package_hash) != 32: + raise ValueError("work-package hash must be 32 bytes") + if len(exports) > MAXIMUM_NUMBER_EXPORTS_WORK_PACKAGE or any(len(s) != EC_SEGMENT_SIZE for s in exports): + raise ProcessWorkpackageError("invalid exported segments") + originals = erasure_data_shards(validator_count) + alignment = 2 * originals + padded_bundle = zero_pad(alignment, bundle) + bundle_shards = encode_chunks(validator_count, len(padded_bundle) // alignment, padded_bundle) + segments = exports + paged_proof_segments(exports) + segment_shard_size = EC_SEGMENT_SIZE // originals + # H.4-H.5: transpose segments so every validator row contains one shard of each segment. + transposed = b''.join( + segment[i * segment_shard_size:(i + 1) * segment_shard_size] + for i in range(originals) for segment in segments + ) + rows = encode_chunks(validator_count, len(transposed) // alignment, transposed) + leaves = [] + for bundle_shard, row in zip(bundle_shards, rows): + chunks = [row[i:i + segment_shard_size] for i in range(0, len(row), segment_shard_size)] + leaves.append(blake2b_256_hash(bundle_shard) + WellBalancedMerkleTree(chunks).root()) + return WorkPackageSpec( + hash=package_hash, length=len(bundle), erasure_root=WellBalancedMerkleTree(leaves).root(), + erasure_shards=validator_count, exports_root=ConstantDepthMerkleTree(exports).root(), + exports_count=len(exports), + ) diff --git a/pyjamaz/cli.py b/pyjamaz/cli.py index 9a302d09..aea944ff 100644 --- a/pyjamaz/cli.py +++ b/pyjamaz/cli.py @@ -783,6 +783,12 @@ def require_response(response, field, operation): trace = Trace.from_jam_bytes(JamBytes(block_file.read_bytes())) if block_file.parent != last_parent: + if last_parent is not None: + # Each trace chain has its own handshake and Initialize. + await fuzzer_session.close() + connected = False + await fuzzer_session.connect() + connected = True # Initialize ancestry = [] genesis_file = block_file.parent / "genesis.bin" diff --git a/pyjamaz/d3l.py b/pyjamaz/d3l.py index fa873b85..c0b63049 100644 --- a/pyjamaz/d3l.py +++ b/pyjamaz/d3l.py @@ -1,5 +1,7 @@ from jamcodec.base import JamBytes +from pyjamaz.availability import DataUnavailable from pyjamaz.models.app import D3LEntry +from pyjamaz.models.common import WorkReport from pyjamaz.storage import StorageEngine @@ -10,10 +12,25 @@ def __init__(self, storage_engine: StorageEngine): def retrieve_segments(self, segment_root: bytes) -> D3LEntry: data = self.storage_engine.get(segment_root) + if data is None: + raise DataUnavailable("imported segments are unavailable") d3l_item = D3LEntry.from_jam_bytes(JamBytes(data)) return d3l_item def store_segments(self, segments: D3LEntry): self.storage_engine.put(segments.segment_root, segments.to_jam_bytes().to_bytes()) + def segment_root_for_package(self, package_hash: bytes) -> bytes: + root = self.storage_engine.get(b'package-root:' + package_hash) + if root is None: + raise DataUnavailable("segment-root mapping is unavailable") + return root + + def store_work_report(self, report: WorkReport, bundle: bytes, exports: list[bytes]) -> None: + """Retain the locally computed data and the 14.14 package-to-root correspondence.""" + spec = report.package_spec + self.store_segments(D3LEntry(spec.exports_root, exports)) + self.storage_engine.put(b'bundle:' + spec.hash, bundle) + self.storage_engine.put(b'package-root:' + spec.hash, spec.exports_root) + diff --git a/pyjamaz/erasure.py b/pyjamaz/erasure.py new file mode 100644 index 00000000..20e883ad --- /dev/null +++ b/pyjamaz/erasure.py @@ -0,0 +1,27 @@ +"""GP 0.8.0 Appendix H, backed by the JAM GF(2^16) implementation.""" + +from jam_erasure import encode_shards + +from pyjamaz.graypaper_constants import EC_SEGMENT_SIZE, valid_validator_count + + +def erasure_data_shards(validator_count: int) -> int: + # GP-0.8.0-eq:H.1: D(v) = max { d < v/3 + 2 | WG mod 2d = 0 }. + if not valid_validator_count(validator_count): + raise ValueError("invalid assurer count") + return max(d for d in range(1, validator_count // 3 + 2) if EC_SEGMENT_SIZE % (2 * d) == 0) + + +def encode_chunks(validator_count: int, piece_count: int, data: bytes) -> list[bytes]: + # GP-0.8.0-eq:H.5: C^v_k(d): B_(2k D(v)) -> [B_(2k)]_v. + originals = erasure_data_shards(validator_count) + if isinstance(piece_count, bool) or not isinstance(piece_count, int) or piece_count < 0: + raise ValueError("invalid erasure piece count") + if len(data) != 2 * piece_count * originals: + raise ValueError("data length does not match erasure dimensions") + if piece_count == 0: + return [b''] * validator_count + shards = encode_shards(data, originals, validator_count - originals) + if len(shards) != validator_count or any(len(shard) != 2 * piece_count for shard in shards): + raise RuntimeError("erasure backend returned invalid shard dimensions") + return shards diff --git a/pyjamaz/fuzzer/v1/session.py b/pyjamaz/fuzzer/v1/session.py index 3c90db33..9aab9477 100644 --- a/pyjamaz/fuzzer/v1/session.py +++ b/pyjamaz/fuzzer/v1/session.py @@ -13,6 +13,7 @@ class FuzzerSession: def __init__(self, path: str, app: 'PyjamazApp') -> None: self.path = path self.app = app + self.features = Features() self.reader: asyncio.StreamReader self.writer: asyncio.StreamWriter @@ -20,7 +21,7 @@ def msg_handshake(self) -> FuzzerMessage: return FuzzerMessage( peer_info=PeerInfoMessage( fuzz_version=FUZZER_VERSION, - features=Features(fork=False, ancestry=False), + features=Features(fork=False, ancestry=False, refine=True), app_version=Version.from_str(APP_VERSION), jam_version=Version.from_str(GP_VERSION), name = "PyJAMaz" @@ -42,6 +43,8 @@ async def _do_handshake(self) -> None: except asyncio.TimeoutError: raise RuntimeError("Target did not send PeerInfo in time") + if target_peerinfo.peer_info is None: + raise RuntimeError("Target did not send PeerInfo") if target_peerinfo.peer_info.jam_version != jam_version: raise RuntimeError( f"Protocol version mismatch: ours={GP_VERSION}, theirs={target_peerinfo.peer_info.jam_version}" @@ -52,25 +55,38 @@ async def _do_handshake(self) -> None: f"Fuzzer version mismatch: ours={FUZZER_VERSION}, theirs={target_peerinfo.peer_info.fuzz_version}" ) + self.features = our_peerinfo.peer_info.features & target_peerinfo.peer_info.features logging.info( f"[fuzzer] Connected to {target_peerinfo.peer_info.name} (v{target_peerinfo.peer_info.app_version})") async def send_request(self, req: FuzzerMessage) -> FuzzerMessage: """Send *req* and return the parsed response.""" - - # logging.debug(f"[fuzzer] Sending {req.to_json()}") + if req.refine_bundle is not None and not self.features.refine: + raise RuntimeError("Refinement was not negotiated") + responses = { + 'initialize': 'state_root', 'import_block': 'state_root', + 'get_state': 'state', 'refine_bundle': 'work_report', + } + variant = req.enum_value()[0] + if variant not in responses: + raise RuntimeError(f"Unexpected outgoing request {variant}") self.writer.write(req.fuzzer_encode()) await self.writer.drain() try: rsp = await asyncio.wait_for(FuzzerMessage.fuzzer_decode(self.reader), timeout=REQUEST_TIMEOUT) except asyncio.TimeoutError: + await self.close() raise RuntimeError(f"Target timed out when responding to {req.to_json()}") - # TODO message type sanity checks - # logging.debug(f"[fuzzer] Received {rsp.to_json()}") + except (ValueError, asyncio.IncompleteReadError): + await self.close() + raise + allowed_error = variant in ('import_block', 'refine_bundle') and rsp.error is not None + if getattr(rsp, responses[variant]) is None and not allowed_error: + await self.close() + raise RuntimeError(f"Unexpected response to {variant}: expected {responses[variant]}") return rsp - async def connect(self) -> None: self.reader, self.writer = await asyncio.open_unix_connection(self.path) await self._do_handshake() diff --git a/pyjamaz/fuzzer/v1/target.py b/pyjamaz/fuzzer/v1/target.py index 9d171340..34e45b5f 100644 --- a/pyjamaz/fuzzer/v1/target.py +++ b/pyjamaz/fuzzer/v1/target.py @@ -5,10 +5,15 @@ import typing from typing import Optional -from pyjamaz.exceptions import BlockValidationError, StateTransitionError +from pyjamaz.availability import BundleData +from pyjamaz.exceptions import BlockValidationError, ProcessWorkpackageError, StateTransitionError from pyjamaz.models.block import AncestorReference +from pyjamaz.models.state import PendingChanges, ServicesState +from pyjamaz.refinement import Refinement +from pyjamaz.state.storage import StateStorage +from pyjamaz.storage import InMemoryStorageEngine from pyjamaz.settings import APP_VERSION, GP_VERSION, FUZZER_VERSION, FUZZER_FEATURE_FORK, FUZZER_FEATURE_ANCESTRY, \ - TEST_SUITE + FUZZER_FEATURE_REFINE, TEST_SUITE from pyjamaz.fuzzer.v1.types import FuzzerMessage, PeerInfoMessage, Version, Features from pyjamaz.utils import format_hash @@ -22,6 +27,7 @@ class FuzzerTarget: def __init__(self, path: str, app: 'PyjamazApp') -> None: self.path = path self.app = app + self._refine_head_hash: Optional[bytes] = None # Wipe any stale socket first try: @@ -77,6 +83,7 @@ async def msg_initialize(self, req: FuzzerMessage) -> FuzzerMessage: # Initialize working state to finalized await self.app.initialize() + self._refine_head_hash = req.initialize.header.hash logging.info(f"πŸ’Ύ Initialized state to {format_hash(self.app.working_state.state_root)}") return FuzzerMessage(state_root=self.app.working_state.state_root) @@ -85,7 +92,8 @@ def msg_handshake(self) -> FuzzerMessage: return FuzzerMessage( peer_info=PeerInfoMessage( fuzz_version=FUZZER_VERSION, - features=Features(fork=FUZZER_FEATURE_FORK, ancestry=FUZZER_FEATURE_ANCESTRY), + features=Features(fork=FUZZER_FEATURE_FORK, ancestry=FUZZER_FEATURE_ANCESTRY, + refine=FUZZER_FEATURE_REFINE), app_version=Version.from_str(APP_VERSION), jam_version=Version.from_str(GP_VERSION), name = "PyJAMaz" @@ -97,6 +105,7 @@ async def msg_import_block(self, req: FuzzerMessage): # SIMPLE_FORKING imports against any retained parent. Finalizing # here would prune ancestry and competing branches before import. await self.app.import_block(req.import_block) + self._refine_head_hash = req.import_block.header.hash logging.info(f"βœ… Block {format_hash(req.import_block.header.hash)} imported -> state root: {format_hash(self.app.working_state.state_root)}") return FuzzerMessage(state_root=self.app.working_state.state_root) @@ -107,6 +116,34 @@ async def msg_import_block(self, req: FuzzerMessage): logging.info(f"πŸ›‘ Block {format_hash(req.import_block.header.hash)} raised error -> {e}") return FuzzerMessage(error=str(e)) + async def msg_refine_bundle(self, req: FuzzerMessage) -> FuzzerMessage: + if self._refine_head_hash is None: + raise RuntimeError("RefineBundle requires initialization") + + # Detach historical lookups from mutable branch selection and PVM caches. + engine = InMemoryStorageEngine() + for key, value in self.app.state_storage.as_list_at(self._refine_head_hash): + engine.put(key, value) + services = ServicesState(services={}) + services.set_state_storage(StateStorage(engine)) + services.pending_changes = PendingChanges() + request = req.refine_bundle + bundle = request.bundle + + def compute(): + data = BundleData.from_parts( + bundle.work_package, bundle.extrinsics, bundle.import_segments, bundle.justifications, + ) + return Refinement(services, data).compute_work_report( + bundle.work_package, request.core_index, request.segment_root_lookup, request.erasure_shards, + ) + + try: + return FuzzerMessage(work_report=await asyncio.to_thread(compute)) + except ProcessWorkpackageError as exc: + # Consensus-defined bundle/Ξ failures leave the session available. + return FuzzerMessage(error=str(exc)) + async def start(self) -> None: self.server = await asyncio.start_unix_server(self._handle_client, path=self.path) @@ -127,6 +164,13 @@ async def _handle_client(self, reader: asyncio.StreamReader, writer: asyncio.Str # Handshake: wait for their PeerInfo first. them = await FuzzerMessage.fuzzer_decode(reader) ours = self.msg_handshake() + if them.peer_info is None: + raise RuntimeError("Expected PeerInfo before requests") + if (them.peer_info.fuzz_version != ours.peer_info.fuzz_version + or them.peer_info.jam_version != ours.peer_info.jam_version): + raise RuntimeError("PeerInfo protocol version mismatch") + features = ours.peer_info.features & them.peer_info.features + initialized = False writer.write(ours.fuzzer_encode()) await writer.drain() @@ -146,20 +190,31 @@ async def _handle_client(self, reader: asyncio.StreamReader, writer: asyncio.Str break try: - rsp = await self._dispatch(req) + if req.initialize is not None: + if initialized: + raise RuntimeError("Duplicate Initialize") + elif not initialized: + raise RuntimeError("Expected Initialize before requests") + if req.peer_info is not None: + raise RuntimeError("Unexpected PeerInfo after handshake") + rsp = await self._dispatch(req, features=features) + if req.initialize is not None: + initialized = True except Exception as e: logging.error(f"[fuzzer] Handler error for {req}: {e}") break # blunt termination on malformed/unexpected messages writer.write(rsp.fuzzer_encode()) await writer.drain() + except Exception as exc: + logging.error(f"[fuzzer] Session error: {exc}") finally: writer.close() await writer.wait_closed() logging.info("[fuzzer] Session finished") - async def _dispatch(self, req: FuzzerMessage) -> FuzzerMessage: + async def _dispatch(self, req: FuzzerMessage, *, features: Optional[Features] = None) -> FuzzerMessage: """ Request - response pattern for fuzzer @@ -183,5 +238,11 @@ async def _dispatch(self, req: FuzzerMessage) -> FuzzerMessage: elif req.import_block is not None: return await self.msg_import_block(req) + elif req.refine_bundle is not None: + negotiated = features if features is not None else self.msg_handshake().peer_info.features + if not negotiated.refine: + raise RuntimeError("Refinement was not negotiated") + return await self.msg_refine_bundle(req) + else: raise RuntimeError(f"Unknown incoming message {req.to_json()}") diff --git a/pyjamaz/fuzzer/v1/types.py b/pyjamaz/fuzzer/v1/types.py index cde67507..a6311dc2 100644 --- a/pyjamaz/fuzzer/v1/types.py +++ b/pyjamaz/fuzzer/v1/types.py @@ -1,14 +1,15 @@ import asyncio from dataclasses import dataclass, field -from typing import List, Tuple +from typing import Dict, List, Tuple, Optional from jamcodec.exceptions import ScaleDecodeException from jamcodec.base import JamBytes from jamcodec.mixins import Serializable -from jamcodec.types import U8, String, Vec, Array, Bytes, Tuple as JamTuple, H256, Null, UnsignedInteger, U32 +from jamcodec.types import U8, String, Vec, Array, Bytes, Tuple as JamTuple, H256, Null, UnsignedInteger, U16, U32, Map from pyjamaz.models.block import Block, Header +from pyjamaz.models.common import WorkPackage, WorkReport HEADER_LEN = 4 MAX_MESSAGE_SIZE = 16 * 1024 * 1024 @@ -52,16 +53,25 @@ def __post_init__(self): class Features: BLOCK_ANCESTRY: int = 1 << 0 SIMPLE_FORKING: int = 1 << 1 - RESERVED: int = 1 << 31 + REFINE: int = 1 << 2 + RESERVED: int = 1 << 31 - def __init__(self, fork:bool=False, ancestry:bool=False): + def __init__(self, fork: bool=False, ancestry: bool=False, refine: bool=False): self._value = 0 & 0xFFFFFFFF self.simple_forking = fork self.block_ancestry = ancestry + self.refine = refine def _get_flag(self, mask: int) -> bool: return (self._value & mask) != 0 + def __and__(self, other: "Features") -> "Features": + if not isinstance(other, Features): + return NotImplemented + features = Features() + features._value = self._value & other._value + return features + def _set_flag(self, mask: int, enabled: bool): if enabled: self._value |= mask @@ -84,6 +94,14 @@ def simple_forking(self) -> bool: def simple_forking(self, value: bool): self._set_flag(self.SIMPLE_FORKING, value) + @property + def refine(self) -> bool: + return self._get_flag(self.REFINE) + + @refine.setter + def refine(self, value: bool): + self._set_flag(self.REFINE, value) + class FeaturesCodec(UnsignedInteger): @@ -118,15 +136,33 @@ class PeerInfoMessage(Serializable): name: str = field(metadata={'codec': String}) +@dataclass +class WorkPackageBundle(Serializable): + work_package: WorkPackage = field(metadata={'codec': WorkPackage.to_codec_def()}) + extrinsics: List[bytes] = field(metadata={'codec': Vec(Bytes)}) + import_segments: List[bytes] = field(metadata={'codec': Vec(Array(U8, 4104))}) + justifications: List[List[bytes]] = field(metadata={'codec': Vec(Vec(H256))}) + + +@dataclass +class RefineBundle(Serializable): + core_index: int = field(metadata={'codec': U16}) + segment_root_lookup: Dict[bytes, bytes] = field(metadata={'codec': Map(H256, H256)}) + erasure_shards: int = field(metadata={'codec': U16}) + bundle: WorkPackageBundle = field(metadata={'codec': WorkPackageBundle.to_codec_def()}) + + @dataclass class FuzzerMessage(Serializable): - peer_info: PeerInfoMessage = field(default=None, metadata={'codec': PeerInfoMessage.to_codec_def()}) - initialize: InitializeMessage = field(default=None, metadata={'codec': InitializeMessage.to_codec_def()}) - state_root: bytes = field(default=None, metadata={'codec': H256}) - import_block: Block = field(default=None, metadata={'codec': Block.to_codec_def()}) - get_state: bytes = field(default=None, metadata={'codec': H256}) - state: List[Tuple[bytes, bytes]] = field(default=None, metadata={'codec': Vec(JamTuple(Array(U8, 31), Bytes))}) - error: str = field(default=None, metadata={'codec': {'id': 255, 'type': String}}) + peer_info: Optional[PeerInfoMessage] = field(default=None, metadata={'codec': PeerInfoMessage.to_codec_def()}) + initialize: Optional[InitializeMessage] = field(default=None, metadata={'codec': InitializeMessage.to_codec_def()}) + state_root: Optional[bytes] = field(default=None, metadata={'codec': H256}) + import_block: Optional[Block] = field(default=None, metadata={'codec': Block.to_codec_def()}) + get_state: Optional[bytes] = field(default=None, metadata={'codec': H256}) + state: Optional[List[Tuple[bytes, bytes]]] = field(default=None, metadata={'codec': Vec(JamTuple(Array(U8, 31), Bytes))}) + refine_bundle: Optional[RefineBundle] = field(default=None, metadata={'codec': RefineBundle.to_codec_def()}) + work_report: Optional[WorkReport] = field(default=None, metadata={'codec': WorkReport.to_codec_def()}) + error: Optional[str] = field(default=None, metadata={'codec': {'id': 255, 'type': String}}) _codec_enum = True @@ -148,6 +184,10 @@ async def fuzzer_decode(cls, reader: asyncio.StreamReader) -> "FuzzerMessage": raise ValueError("Incoming message too large: %d bytes" % length) payload = await reader.readexactly(length) try: - return cls.from_jam_bytes(JamBytes(payload)) + stream = JamBytes(payload) + message = cls.from_jam_bytes(stream) + if stream.get_remaining_length() or message.to_jam_bytes().to_bytes() != payload: + raise ValueError("Noncanonical message or trailing bytes") + return message except ScaleDecodeException as e: raise ValueError(f"Malformed message: {e}") from e diff --git a/pyjamaz/merkle.py b/pyjamaz/merkle.py index 51d0cade..7db7ed19 100644 --- a/pyjamaz/merkle.py +++ b/pyjamaz/merkle.py @@ -191,6 +191,35 @@ def root(self) -> bytes: class ConstantDepthMerkleTree(BinaryMerkleTree): + def trace(self, index: int) -> List[bytes]: + """Return the E.5 proof from root to leaf, excluding padding leaves.""" + if not 0 <= index < len(self.data): + raise IndexError("segment index out of range") + nodes = self.constancy_preprocessor(self.data) + proof = [] + while len(nodes) > 1: + split = len(nodes) // 2 + if index < split: + proof.append(self.node(nodes[split:])) + nodes = nodes[:split] + else: + proof.append(self.node(nodes[:split])) + nodes = nodes[split:] + index -= split + return proof + + @staticmethod + def verify(segment: bytes, index: int, proof: List[bytes], root: bytes) -> bool: + """Verify an E.5 proof, whose sibling hashes are in root-first order.""" + if not 0 <= index < 1 << len(proof) or any(len(h) != 32 for h in proof): + return False + value = blake2b_256_hash(b'leaf' + segment) + for sibling in reversed(proof): + pair = sibling + value if index & 1 else value + sibling + value = blake2b_256_hash(b'node' + pair) + index >>= 1 + return value == root + def root(self) -> bytes: """ GP-0.7.2-eq:E.4 | constant-depth merkle tree root hash diff --git a/pyjamaz/models/common.py b/pyjamaz/models/common.py index df7eb592..c427fd3a 100644 --- a/pyjamaz/models/common.py +++ b/pyjamaz/models/common.py @@ -10,8 +10,7 @@ from jamcodec.types import H256, Array, U8, U32, Bytes, Null, U64, Vec, U16, Map, VarInt64, String, Bool, H512 from pyjamaz.exceptions import BlockValidationError -from pyjamaz.graypaper_constants import MAXIMUM_NUMBER_EXTRINSICS_WORK_PACKAGE, SIZE_TRANSFER_MEMO, VALIDATOR_COUNT, \ - valid_validator_count +from pyjamaz.graypaper_constants import MAXIMUM_NUMBER_EXTRINSICS_WORK_PACKAGE, SIZE_TRANSFER_MEMO, VALIDATOR_COUNT from pyjamaz.hashing import blake2b_256_hash # from pyjamaz.models.block import Credential, Guarantee from pyjamaz.pvm.constants import ExitCondition, ExitReason @@ -126,7 +125,8 @@ def extract(cls, data: bytes) -> "Preimage": program_authors = Vec(String).decode(metadata_bytes) except Exception: metadata_version = 0 - program_name = metadata.decode("utf-8") + # GP metadata is an opaque blob; its display name need not be UTF-8. + program_name = metadata.decode("utf-8", errors="replace") program_license = '' program_authors = [] program_version = '' @@ -281,7 +281,7 @@ def hash(self) -> bytes: def authorizer_hash(self) -> bytes: """ - GP-0.7.2-eq:14.10 (bold_p_a) | Authorizer hash. + GP-0.8.0-eq:14.11: p_a = H(p_u ++ p_f). """ return blake2b_256_hash(self.auth_code_hash + self.authorizer_config) @@ -304,6 +304,10 @@ def authorization_code(self) -> bytes: return getattr(self, '_authorization_code', None) def set_authorization_code(self, services_state: 'ServicesState') -> None: + # GP-0.8.0-eq:14.11: (metadata, code) = historical_lookup(host, lookup-slot, code-hash). + # A reused package must not retain code from another historical state. + setattr(self, '_authorization_code', None) + self.authorization_metadata = None preimage_blob = services_state.historical_preimage_lookup( service_account_id=self.auth_code_host, timeslot=self.context.lookup_anchor_slot, @@ -509,7 +513,7 @@ class WorkDigest(Serializable): @classmethod def from_work_item(cls, work_item: WorkItem, result: WorkExecResult, gas_used: int) -> "WorkDigest": """ - GP-0.7.2-eq:14.9 (function_C) | the item-to-result function + GP-0.8.0-eq:14.10: C(w, result, gas_used) -> work digest. """ # Rename to WorkDigest return cls( @@ -555,30 +559,7 @@ class WorkPackageSpec(Serializable): exports_root: bytes = field(metadata={'codec': H256}) exports_count: int = field(metadata={'codec': U16}) - @classmethod - def create_from_work_package(cls, - work_package: WorkPackage, extrinsic_data: List[bytes], imported_segments: List[bytes], - justification_data: List[bytes], exported_segments: List[bytes], exports_root: bytes, - *, erasure_shards: int, - ) -> "WorkPackageSpec": - """ - GP-0.7.2-eq:14.17 function_A | creates an availability specifier from a workpackage - # TODO finish implementation - """ - # GP-0.8.0-eq:14.17: the report carries its original assurer set size. - if not valid_validator_count(erasure_shards): - raise ValueError("invalid assurer count") - - # serialized_auditable_work_package = work_package.serialize_to_auditable() - - return WorkPackageSpec( - hash=work_package.hash(), - length=work_package.to_jam_bytes().length, - erasure_root=bytes(32), - erasure_shards=erasure_shards, - exports_root=exports_root, - exports_count=len(exported_segments), - ) + diff --git a/pyjamaz/models/state.py b/pyjamaz/models/state.py index 208f631b..84e1ea8e 100644 --- a/pyjamaz/models/state.py +++ b/pyjamaz/models/state.py @@ -549,6 +549,17 @@ def __deepcopy__(self, memo): def set_state_storage(self, state: StateStorage): setattr(self, '_state_storage', state) + def snapshot(self) -> "ServicesState": + """Detach historical lookups and pending changes before dispatching Refine.""" + from pyjamaz.storage import InMemoryStorageEngine + + if self.state_storage is None: + raise ValueError("services state has no storage") + snapshot = ServicesState(services={}) + snapshot.set_state_storage(StateStorage(InMemoryStorageEngine(dict(self.state_storage.as_list())))) + snapshot.pending_changes = deepcopy(self.pending_changes or PendingChanges()) + return snapshot + @property def state_storage(self) -> Optional[StateStorage]: return getattr(self, '_state_storage', None) diff --git a/pyjamaz/refinement.py b/pyjamaz/refinement.py new file mode 100644 index 00000000..60f12a4c --- /dev/null +++ b/pyjamaz/refinement.py @@ -0,0 +1,90 @@ +"""Work-report computation with the GP's explicit (p, c, l, v) arguments.""" + +from pyjamaz.availability import BundleData, availability_spec, validate_work_package +from pyjamaz.erasure import erasure_data_shards +from pyjamaz.exceptions import ProcessWorkpackageError +from pyjamaz.graypaper_constants import CORE_COUNT, EC_SEGMENT_SIZE, MAXIMUM_SIZE_ENCODED_WORK_REPORT +from pyjamaz.hostcalls.invocation import pvm_invoke_is_authorized, pvm_invoke_refine +from pyjamaz.models.common import WorkDigest, WorkExecResult, WorkPackage, WorkReport +from pyjamaz.models.state import ServicesState + + +class Refinement: + """Bind the historical state and data availability assumed by GP section 14. + + Each execution owns its inputs and resulting bundle/exports. The runtime + acquires and persists data; this class never reads or writes node storage. + """ + + def __init__(self, services: ServicesState, data: BundleData): + self.services = services + self.data = data + self.bundle = b'' + self.exported_segments: list[bytes] = [] + + def compute_work_report(self, package: WorkPackage, core_index: int, + mappings: dict[bytes, bytes], erasure_shards: int) -> WorkReport: + # GP-0.8.0-eq:14.13: Xi(p,c,l,v) -> WorkReport or bottom (ProcessWorkpackageError). + self.bundle = b'' + self.exported_segments = [] + validate_work_package(package) + if not 0 <= core_index < CORE_COUNT: + raise ProcessWorkpackageError("invalid core index") + try: + erasure_data_shards(erasure_shards) + except ValueError as exc: + raise ProcessWorkpackageError(str(exc)) from exc + expected_mappings = { + entry.tree_root for item in package.items for entry in item.import_segments + if entry.is_work_package_hash + } + if mappings.keys() != expected_mappings: + raise ProcessWorkpackageError("segment-root dictionary has unexpected keys") + + # GP-0.8.0-eq:14.15-14.16: obtain verified X, S_l and J_l before invoking PVM code. + bundle = self.data.assemble_bundle(package, mappings) + extrinsics = [self.data.extrinsic_data(item) for item in package.items] + imports = [self.data.import_segments(mappings, item) for item in package.items] + package.set_authorization_code(self.services) + authorization = pvm_invoke_is_authorized(package, core_index) + trace = authorization.work_exec_result.ok + if not isinstance(trace, bytes): + raise ProcessWorkpackageError("Unauthorized") + if len(trace) > MAXIMUM_SIZE_ENCODED_WORK_REPORT: + raise ProcessWorkpackageError("Oversized auth result") + + digests = [] + exports = [] + output_size = len(trace) + export_offset = 0 + for index, item in enumerate(package.items): + output = pvm_invoke_refine( + core_index=core_index, work_item_index=index, work_package=package, + authorizer_output=trace, work_items_import_segments=imports, + export_segment_offset=export_offset, services_state=self.services, extrinsics=extrinsics, + ) + # GP-0.8.0-eq:14.13, I(p,j): failure precedence and exactly m_e zero exports. + result = output.work_exec_result + item_exports = output.export_segments + if output_size + len(result.ok or b'') > MAXIMUM_SIZE_ENCODED_WORK_REPORT: + result = WorkExecResult(digest_oversize=True) + elif len(item_exports) != item.export_count: + result = WorkExecResult(bad_exports=True) + if result.ok is None: + item_exports = [bytes(EC_SEGMENT_SIZE)] * item.export_count + else: + output_size += len(result.ok) + digests.append(WorkDigest.from_work_item(item, result, output.gas_used)) + exports.extend(item_exports) + export_offset += item.export_count + + # GP-0.8.0-eq:14.17: s = A(H(p), B(p,l), concat(exports), v). + spec = availability_spec(package.hash(), bundle, exports, erasure_shards) + report = WorkReport( + package_spec=spec, context=package.context, core_index=core_index, + authorizer_hash=package.authorizer_hash(), auth_output=trace, + auth_gas_used=authorization.gas_used, segment_root_lookup=dict(mappings), results=digests, + ) + self.bundle = bundle + self.exported_segments = exports + return report diff --git a/pyjamaz/runtime/pipelines/refine.py b/pyjamaz/runtime/pipelines/refine.py index 401918fa..c9a43958 100644 --- a/pyjamaz/runtime/pipelines/refine.py +++ b/pyjamaz/runtime/pipelines/refine.py @@ -1,17 +1,22 @@ import asyncio import logging from asyncio import TaskGroup +from copy import deepcopy from typing import Dict, List import anyio from pyjamaz.app import PyjamazApp +from pyjamaz.availability import BundleData, DataUnavailable, import_reference, resolve_import_root, validate_work_package +from pyjamaz.exceptions import ProcessWorkpackageError +from pyjamaz.merkle import ConstantDepthMerkleTree +from pyjamaz.refinement import Refinement from pyjamaz.constants import MESSAGE_TYPES from pyjamaz.runtime.extrinsics import WorkpackageExtrinsicCollector from pyjamaz.models.common import WorkPackage, WorkPackageStatus, WorkPackageReportableStatus, \ WorkPackageReportedStatus, BlockDesc, Credential, Guarantee from pyjamaz.runtime.types import WorkPackageQueueItem -from pyjamaz.settings import DEBUG, GUARANTEE_SIGNATURE_WAIT_PERIOD +from pyjamaz.settings import DEBUG, GUARANTEE_SIGNATURE_WAIT_PERIOD, SOLO_MODE from pyjamaz.transport.pubsub import PubSubSignal from pyjamaz.utils import format_hash @@ -39,7 +44,7 @@ def start(self, task_group: TaskGroup): task_group.start_soon(self._guarantuee_worker) def add_work_package(self, work_package: WorkPackage, extrinsics: List[bytes]): - + work_package = deepcopy(work_package) self._pending_work_packages[work_package.hash()] = WorkPackageQueueItem( work_package=work_package, status=WorkPackageStatus( Reportable=WorkPackageReportableStatus( @@ -96,84 +101,141 @@ async def _timeslot_worker(self) -> None: finally: self._timeslot_queue.task_done() + def _segment_root_lookup(self, package: WorkPackage, state) -> dict[bytes, bytes]: + # GP-0.8.0-eq:14.14: the runtime supplies the known package-to-export-root correspondence. + known = {} + for block in state.recent_history.recent_blocks: + known.update({report.hash: report.exports_root for report in block.reported}) + for pending in self._pending_work_packages.values(): + if pending.work_report is not None: + spec = pending.work_report.package_spec + known[spec.hash] = spec.exports_root + mappings = {} + for item in package.items: + for entry in item.import_segments: + if entry.is_work_package_hash and entry.tree_root not in mappings: + if entry.tree_root in known: + mappings[entry.tree_root] = known[entry.tree_root] + elif self.app.d3l_store is not None: + mappings[entry.tree_root] = self.app.d3l_store.segment_root_for_package(entry.tree_root) + else: + raise DataUnavailable("segment-root mapping is unavailable") + return mappings + + def _bundle_data(self, package: WorkPackage, mappings: dict[bytes, bytes]) -> BundleData: + extrinsics, segments, proofs = [], [], [] + stored_trees = {} + for item in package.items: + for manifest in item.extrinsic: + blob = self._work_package_extrinsics.get(package, manifest.hash, manifest.len) + if blob is None: + raise DataUnavailable("extrinsic data is unavailable") + extrinsics.append(blob) + for entry in item.import_segments: + root = resolve_import_root(mappings, import_reference(entry)) + if self.app.d3l_store is None: + raise DataUnavailable("imported segments are unavailable") + if root not in stored_trees: + stored = self.app.d3l_store.retrieve_segments(root).segments + tree = ConstantDepthMerkleTree(stored) + if tree.root() != root: + raise ProcessWorkpackageError("stored segments do not match their root") + stored_trees[root] = (stored, tree) + stored, tree = stored_trees[root] + if entry.segment_index >= len(stored): + raise ProcessWorkpackageError("import segment index out of range") + segments.append(stored[entry.segment_index]) + proofs.append(tree.trace(entry.segment_index)) + return BundleData.from_parts(package, extrinsics, segments, proofs) + + @staticmethod + def _validator_keys(state) -> tuple[bytes, ...]: + return tuple(bytes(validator.ed25519) for validator in state.validator_pool.validators) + + @staticmethod + def _view_is_committed(state, head_hash: bytes) -> bool: + storage = state.services.state_storage + return not storage.transaction and storage.block_hash in (None, head_hash) + + @staticmethod + def _retry(item: WorkPackageQueueItem) -> None: + item.work_report = None + item.signatures.clear() + item.status = WorkPackageStatus(Reportable=WorkPackageReportableStatus(remaining_blocks=4)) + async def _refine_worker(self) -> None: while True: - wp_item = await self._refine_queue.get() - - logging.info(f'πŸ‘¨β€πŸ’» Processing work-package {format_hash(wp_item.work_package.hash())} [{wp_item.status.enum_value()[0]}]') - - # if self.get_core_assigment() is None: - # raise ProcessWorkpackageError("Cannot process work package: no core assignment") - - work_package = wp_item.work_package - - # Prepare extrinsic data (GP-0.7.2-eq:B.6 bold_x_flat) - extrinsics = [ - [self._work_package_extrinsics.get(work_package, x.hash, x.len) for x in w.extrinsic] - for w in work_package.items - ] - - # Set code - work_package.set_authorization_code(self.app.working_state.services) - - # Todo move work_result_computation to here - work_report = await anyio.to_thread.run_sync( - self.app.work_result_computation, - work_package, - self.app.get_core_assigment(), - self.app.working_state.services, - extrinsics, - # GP-0.8.0-eq:14.13: capture the assurer count before dispatch. - len(self.app.working_state.validator_pool.validators), - ) - - logging.info(f'πŸ‘¨β€πŸ’»Created work-report {format_hash(work_report.hash())}') - - wp_item.work_report = work_report - - # Update work package status - wp_item.status = WorkPackageStatus( - Reported=WorkPackageReportedStatus( - reported_in=BlockDesc( - slot=self.app.working_state.timeslot.number, - header_hash=self.app.retrieve_block_hash(self.app.working_state.timeslot.number) - ), - core=self.app.get_core_assigment(), - report_hash=work_report.hash() - ) - ) - - if self.app.pubsub: - # Send signal - await self.app.pubsub.publish( - PubSubSignal( - topic=MESSAGE_TYPES.WORK_PACKAGE_STATUS, - data=[wp_item.status.to_json()] - ) - ) - - # Clean up work package extrinsics - self._work_package_extrinsics.clear(work_package) - - # Guarantee signature - credential = await self.app.create_guarantee_signature(work_report) - - await self.add_signature(work_package.hash(), credential) - - - #TODO - # TODO exchange signature with other core members - for v_idx, assignment in enumerate(self.app.block_context.guarantor_assignments): - if assignment.validator_ed25519 != self.app.config.keys.ed25519.public_key and assignment.core_index == self.app.get_core_assigment(): - signature = await self.app.create_guarantee_signature_for_validator(work_report, v_idx) - # await asyncio.sleep(3) - await self.add_signature(work_package.hash(), signature) - - # await self.app.guarantee_work_report(work_report, self.app.current_timeslot()) - - DEBUG and logging.debug(f"Processed work package: {format_hash(work_package.hash())}") - - + item = await self._refine_queue.get() + try: + await self._refine_work_package(item) + except asyncio.CancelledError: + self._retry(item) + raise + except DataUnavailable as exc: + self._retry(item) + logging.info("Refine waits for data: %s", exc) + except ProcessWorkpackageError as exc: + self._retry(item) + item.status = WorkPackageStatus(Failed=str(exc)) + logging.info("Work package rejected: %s", exc) + except Exception as exc: + self._retry(item) + item.status = WorkPackageStatus(Failed=str(exc)) + logging.exception("Work-package refinement failed") + finally: + self._refine_queue.task_done() + + async def _refine_work_package(self, item: WorkPackageQueueItem) -> None: + # Capture every input before dispatch; the worker must not follow a live state view. + state = self.app.working_state + package = deepcopy(item.work_package) + validate_work_package(package) + core = self.app.get_core_assigment() + if core is None: + self._retry(item) + return + head_root = state.state_root + slot = state.timeslot.number + head_hash = self.app.retrieve_block_hash(slot) + if head_hash is None or not self._view_is_committed(state, head_hash): + self._retry(item) + return + validator_keys = self._validator_keys(state) + mappings = self._segment_root_lookup(package, state) + data = self._bundle_data(package, mappings) + execution = Refinement(state.services.snapshot(), data) + report = await anyio.to_thread.run_sync( + execution.compute_work_report, package, core, mappings, len(validator_keys), + ) + + # A changed branch or validator/core assignment needs a fresh guarantee view. + current = self.app.working_state + if (current.state_root != head_root or self.app.get_core_assigment() != core + or self._validator_keys(current) != validator_keys + or not self._view_is_committed(current, head_hash)): + self._retry(item) + return + if self.app.d3l_store is not None: + self.app.d3l_store.store_work_report(report, execution.bundle, execution.exported_segments) + item.work_report = report + item.status = WorkPackageStatus(Reported=WorkPackageReportedStatus( + reported_in=BlockDesc(slot=slot, header_hash=head_hash), core=core, report_hash=report.hash(), + )) + if self.app.pubsub: + await self.app.pubsub.publish(PubSubSignal( + topic=MESSAGE_TYPES.WORK_PACKAGE_STATUS, data=[item.status.to_json()], + )) + credential = await self.app.create_guarantee_signature(report) + await self.add_signature(package.hash(), credential) + + if SOLO_MODE: + for index, assignment in enumerate(self.app.block_context.guarantor_assignments): + if (assignment.validator_ed25519 != self.app.config.keys.ed25519.public_key + and assignment.core_index == core): + signature = await self.app.create_guarantee_signature_for_validator(report, index) + await self.add_signature(package.hash(), signature) + self._work_package_extrinsics.clear(item.work_package) + logging.info("Created work-report %s", format_hash(report.hash())) async def _guarantuee_worker(self) -> None: while True: diff --git a/pyjamaz/settings.py b/pyjamaz/settings.py index e4b52d53..95e48f1e 100644 --- a/pyjamaz/settings.py +++ b/pyjamaz/settings.py @@ -22,6 +22,7 @@ def _apply_jam_fuzz_spec_override(): FUZZER_VERSION = 1 FUZZER_FEATURE_FORK = True FUZZER_FEATURE_ANCESTRY = False +FUZZER_FEATURE_REFINE = True DEBUG = False PROFILING = False diff --git a/requirements.txt b/requirements.txt index e8067414..51d16e56 100644 --- a/requirements.txt +++ b/requirements.txt @@ -13,6 +13,7 @@ rocksdict~=0.3 bandersnatch-vrfs==0.6.3 jamcodec~=0.1.5 +py-jam-erasure-bindings==0.1.0 numpy==2.3.5 numba==0.63.0 diff --git a/test/test_fuzzer_refinement.py b/test/test_fuzzer_refinement.py new file mode 100644 index 00000000..c92756a5 --- /dev/null +++ b/test/test_fuzzer_refinement.py @@ -0,0 +1,517 @@ +import asyncio +from copy import deepcopy +from types import SimpleNamespace +from tempfile import TemporaryDirectory + +import pytest +from asyncclick.testing import CliRunner +from jam_erasure import encode_shards +from jamcodec.base import JamBytes +from jamcodec.types import Bytes, H256, U32, Vec + +from pyjamaz import refinement as refinement_module +from pyjamaz.app import PyjamazApp +from pyjamaz.availability import BundleData, availability_spec, paged_proof_segments +from pyjamaz.erasure import erasure_data_shards +from pyjamaz.refinement import Refinement +from pyjamaz.exceptions import BlockValidationError +from pyjamaz.fuzzer.v1.session import FuzzerSession +from pyjamaz.fuzzer.v1.target import FuzzerTarget +from pyjamaz.fuzzer.v1.types import FuzzerMessage, InitializeMessage, RefineBundle, WorkPackageBundle +from pyjamaz.hashing import blake2b_256_hash as hash_blob +from pyjamaz.merkle import ConstantDepthMerkleTree, PatriciaMerkleTrie +from pyjamaz.models.app import StateDump, Trace, TraceGenesis +from pyjamaz.models.block import Block, Extrinsic, Header +from pyjamaz.models.common import ImportSegment, RefinementContext, WorkExecResult, WorkItem, WorkItemExtrinsic, WorkPackage +from pyjamaz.models.context import AppContext +from pyjamaz.models.state import PendingChanges, ServicesState +from pyjamaz.pvm.constants import Opcode +from pyjamaz.pvm.types import PVMCode +from pyjamaz.state.base import state_key_constructor_preimage, state_key_constructor_preimage_availability +from pyjamaz.state.storage import StateStorage +from pyjamaz.storage import InMemoryStorageEngine + + +MARKER = b'm' * 31 + + +@pytest.fixture +def refine_request(): + segments = [bytes([i]) * 4104 for i in range(4)] + tree = ConstantDepthMerkleTree(segments) + reference = b'r' * 32 + item = WorkItem(0, bytes(32), 1000, 1000, 2, b'payload', [ + ImportSegment.from_segment_root(tree.root(), 1), + ImportSegment.from_work_package_hash(reference, 2), + ], [WorkItemExtrinsic.from_blob(b'abc'), WorkItemExtrinsic.from_blob(b'')]) + package = WorkPackage(0, bytes(32), RefinementContext( + bytes(32), bytes(32), bytes(32), bytes(32), 0, [], + ), b'', b'', [item]) + bundle = WorkPackageBundle(package, [b'abc', b''], segments[1:3], [tree.trace(1), tree.trace(2)]) + return RefineBundle(1, {reference: tree.root()}, 6, bundle) + + +@pytest.fixture +def target(): + engine = InMemoryStorageEngine() + app = PyjamazApp.__new__(PyjamazApp) + app.state_db = engine.namespace(b'state') + app.block_db = engine.namespace(b'block') + app.state_storage = StateStorage(app.state_db) + app.app_context = AppContext(state_storage=app.state_storage) + app.working_state = None + app.pubsub = app.d3l_store = None + + def retrieve(): + services = ServicesState(services={}) + services.set_state_storage(app.state_storage) + services.pending_changes = PendingChanges() + return SimpleNamespace(state_root=app.state_storage.state_root(), services=services) + + app.retrieve_jam_state = retrieve + app.import_block = app._import_block + + async def transition(block, produce=False): + await app.initialize(block.header) + app.state_storage.put(MARKER, block.header.timeslot.to_bytes(4, 'little')) + if block.header.author_index: + raise BlockValidationError('invalid block') + app.state_storage.commit() + app.working_state = retrieve() + + app.state_transition = transition + # macOS Unix-domain socket paths cannot exceed 104 bytes. + with TemporaryDirectory(prefix='refine-', dir='/tmp') as directory: + yield FuzzerTarget(f'{directory}/target.sock', app) + + +async def initialize(target): + header = Header.default() + response = await target.msg_initialize(FuzzerMessage(initialize=InitializeMessage( + header, [(MARKER, bytes(4))], [], + ))) + assert response.error is None + return header + + +@pytest.fixture +def vm(monkeypatch): + calls = [] + auth = SimpleNamespace(work_exec_result=WorkExecResult(ok=b'auth'), gas_used=3) + output = SimpleNamespace(work_exec_result=WorkExecResult(ok=b'result'), gas_used=7, + export_segments=[b'x' * 4104, b'y' * 4104]) + monkeypatch.setattr(refinement_module, 'pvm_invoke_is_authorized', lambda *_: auth) + + def refine(**kwargs): + calls.append(kwargs) + return output + + monkeypatch.setattr(refinement_module, 'pvm_invoke_refine', refine) + return SimpleNamespace(calls=calls, auth=auth, output=output) + + +def test_bundle_encoding_and_imports_follow_manifest_order(refine_request): + bundle = refine_request.bundle + data = BundleData.from_parts(bundle.work_package, bundle.extrinsics, bundle.import_segments, bundle.justifications) + extrinsics = [data.extrinsic_data(item) for item in bundle.work_package.items] + segments = [data.import_segments(refine_request.segment_root_lookup, item) for item in bundle.work_package.items] + encoded = data.assemble_bundle(bundle.work_package, refine_request.segment_root_lookup) + assert extrinsics == [[b'abc', b'']] + assert segments == [bundle.import_segments] + assert encoded == (bundle.work_package.to_jam_bytes().to_bytes() + b'abc' + + b''.join(bundle.import_segments) + + b''.join(b'\x02' + b''.join(path) for path in bundle.justifications)) + assert encoded != bundle.to_jam_bytes().to_bytes() + + +@pytest.mark.parametrize('fault', [ + 'extra_extrinsic', 'missing_extrinsic', 'wrong_length', 'wrong_hash', 'extra_import', + 'missing_import', 'missing_proof', 'wrong_segment', 'wrong_proof', 'reversed_proof', + 'missing_mapping', 'extra_mapping', 'wrong_mapping', 'short_segment', +]) +def test_invalid_bundles_are_rejected_before_execution(target, refine_request, vm, fault): + bundle = refine_request.bundle + if fault == 'extra_extrinsic': + bundle.extrinsics.append(b'') + elif fault == 'missing_extrinsic': + bundle.extrinsics.pop() + elif fault == 'wrong_length': + bundle.extrinsics[0] = b'abcd' + elif fault == 'wrong_hash': + bundle.extrinsics[0] = b'xyz' + elif fault == 'extra_import': + bundle.import_segments.append(bytes(4104)) + elif fault == 'missing_import': + bundle.import_segments.pop() + elif fault == 'missing_proof': + bundle.justifications.pop() + elif fault == 'wrong_segment': + bundle.import_segments[0] = bytes(4104) + elif fault == 'wrong_proof': + bundle.justifications[0][0] = bytes(32) + elif fault == 'reversed_proof': + bundle.justifications[0].reverse() + elif fault == 'missing_mapping': + refine_request.segment_root_lookup.clear() + elif fault == 'extra_mapping': + refine_request.segment_root_lookup[b'z' * 32] = bytes(32) + elif fault == 'wrong_mapping': + refine_request.segment_root_lookup[b'r' * 32] = bytes(32) + elif fault == 'short_segment': + bundle.import_segments[0] = bytes(4103) + + async def run(): + await initialize(target) + response = await target._dispatch(FuzzerMessage(refine_bundle=refine_request)) + assert response.error is not None + assert response.work_report is None + assert vm.calls == [] + + asyncio.run(run()) + + +def test_refinement_uses_last_successful_state_without_mutating_it(target, refine_request, vm): + async def run(): + genesis = await initialize(target) + header = Header.default() + header.parent, header.timeslot = genesis.hash, 1 + assert (await target.msg_import_block(FuzzerMessage(import_block=Block(header, Extrinsic.default())))).error is None + selected = target.app.working_state + storage = target.app.state_storage + before = deepcopy(storage.change_sets) + # Reject a sibling based on the older state; refinement must still use the successful head. + rejected = deepcopy(header) + rejected.timeslot, rejected.author_index = 2, 1 + assert (await target.msg_import_block(FuzzerMessage(import_block=Block(rejected, Extrinsic.default())))).error + response = await target._dispatch(FuzzerMessage(refine_bundle=refine_request)) + report = response.work_report + assert report is not None + assert report.segment_root_lookup == refine_request.segment_root_lookup + assert report.package_spec.erasure_shards == refine_request.erasure_shards + assert report.package_spec.erasure_root != bytes(32) + assert report.results[0].result.ok == b'result' + assert vm.calls[0]['work_items_import_segments'] == [refine_request.bundle.import_segments] + assert vm.calls[0]['extrinsics'] == [[b'abc', b'']] + assert vm.calls[0]['services_state'].state_storage.get(MARKER) == (1).to_bytes(4, 'little') + assert storage.change_sets == before + assert storage.block_hash == header.hash + assert target.app.working_state.state_root == selected.state_root + assert storage.transaction == {} + + asyncio.run(run()) + + +@pytest.mark.parametrize('failure', ['out_of_gas', 'panic', 'bad_code', 'code_oversize', 'bad_exports', 'digest_oversize']) +def test_item_failures_are_reports_with_declared_zero_exports(target, refine_request, vm, failure): + if failure == 'bad_exports': + vm.output.export_segments = [] + elif failure == 'digest_oversize': + vm.output.work_exec_result = WorkExecResult(ok=bytes(48 * 1024)) + else: + vm.output.work_exec_result = WorkExecResult(**{failure: True}) + + async def run(): + await initialize(target) + response = await target.msg_refine_bundle(FuzzerMessage(refine_bundle=refine_request)) + assert response.error is None + report = response.work_report + assert report.results[0].result.enum_value()[0] == failure + assert report.package_spec.exports_count == 2 + assert report.package_spec.exports_root == ConstantDepthMerkleTree([bytes(4104)] * 2).root() + assert report.results[0].refine_load.gas_used == 7 + + asyncio.run(run()) + + +def test_authorization_failure_is_recoverable(target, refine_request, vm): + async def run(): + await initialize(target) + vm.auth.work_exec_result = WorkExecResult(panic=True) + assert (await target.msg_refine_bundle(FuzzerMessage(refine_bundle=refine_request))).error + assert vm.calls == [] + vm.auth.work_exec_result = WorkExecResult(ok=b'') + assert (await target.msg_refine_bundle(FuzzerMessage(refine_bundle=refine_request))).work_report + + asyncio.run(run()) + + +@pytest.mark.parametrize('metadata', [b'', bytes(5), b'\xff']) +def test_real_authorizer_and_refine_pvm_on_historical_state(target, refine_request, metadata): + # Return an empty successful result from both real invocations. + raw = bytes((Opcode.load_imm.value, 8, 0, 0, 0, 0, Opcode.jump_ind.value, 0)) + code = PVMCode(0, 0, len(raw), [], raw, [True, False, False, False, False, False, True, False]) + encoded_code = code.to_jam_bytes().to_bytes() + preimage = (Bytes.encode(metadata).to_bytes() + bytes(8) + (4096).to_bytes(3, 'little') + + len(encoded_code).to_bytes(4, 'little') + encoded_code) + code_hash = hash_blob(preimage) + package = refine_request.bundle.work_package + package.auth_code_hash = package.items[0].code_hash = code_hash + package.items[0].export_count = 0 + state = [ + (state_key_constructor_preimage(0, code_hash), preimage), + (state_key_constructor_preimage_availability(0, code_hash, len(preimage)), Vec(U32).encode([0]).to_bytes()), + ] + + async def run(): + await target.msg_initialize(FuzzerMessage(initialize=InitializeMessage(Header.default(), state, []))) + response = await target.msg_refine_bundle(FuzzerMessage(refine_bundle=refine_request)) + assert response.error is None + report = response.work_report + assert report.auth_output == b'' + assert report.auth_gas_used > 0 + assert report.results[0].result.ok == b'' + assert report.results[0].refine_load.gas_used > 0 + assert report.package_spec.exports_count == 0 + assert report.package_spec.erasure_root != bytes(32) + # Reusing the same package at another historical time cannot retain stale code. + package.context.lookup_anchor_slot = 10 + key = state_key_constructor_preimage_availability(0, code_hash, len(preimage)) + target.app.state_db.put(key, Vec(U32).encode([0, 10]).to_bytes()) + assert (await target.msg_refine_bundle(FuzzerMessage(refine_bundle=refine_request))).error == 'Unauthorized' + + asyncio.run(run()) + + +def test_multiple_items_preserve_groups_and_export_offsets(target, refine_request, vm, monkeypatch): + package = refine_request.bundle.work_package + second = deepcopy(package.items[0]) + second.export_count = 1 + second.import_segments.reverse() + package.items.append(second) + bundle = refine_request.bundle + bundle.extrinsics += list(bundle.extrinsics) + bundle.import_segments += list(reversed(bundle.import_segments)) + bundle.justifications += list(reversed(bundle.justifications)) + + def refine(**kwargs): + vm.calls.append(kwargs) + if kwargs['work_item_index'] == 0: + # A failed item must not consume output space available to the next item. + return SimpleNamespace(work_exec_result=WorkExecResult(ok=bytes(48 * 1024 - 4)), + gas_used=2, export_segments=[]) + return SimpleNamespace(work_exec_result=WorkExecResult(ok=b'next'), + gas_used=3, export_segments=[b'z' * 4104]) + + monkeypatch.setattr(refinement_module, 'pvm_invoke_refine', refine) + + async def run(): + await initialize(target) + response = await target.msg_refine_bundle(FuzzerMessage(refine_bundle=refine_request)) + assert response.error is None + report = response.work_report + assert report.results[0].result.bad_exports + assert report.results[1].result.ok == b'next' + assert [c['export_segment_offset'] for c in vm.calls] == [0, 2] + assert vm.calls[1]['work_items_import_segments'] == [bundle.import_segments[:2], bundle.import_segments[2:]] + assert vm.calls[1]['extrinsics'] == [[b'abc', b''], [b'abc', b'']] + assert report.package_spec.exports_root == ConstantDepthMerkleTree([bytes(4104)] * 2 + [b'z' * 4104]).root() + + asyncio.run(run()) + + +@pytest.mark.parametrize(('field', 'value'), [('core_index', 65535), ('erasure_shards', 0), ('erasure_shards', 7)]) +def test_invalid_computation_arguments_are_recoverable(target, refine_request, vm, field, value): + setattr(refine_request, field, value) + + async def run(): + await initialize(target) + assert (await target.msg_refine_bundle(FuzzerMessage(refine_bundle=refine_request))).error + assert vm.calls == [] + + asyncio.run(run()) + + +def test_oversized_authorizer_result_is_rejected(target, refine_request, vm): + vm.auth.work_exec_result = WorkExecResult(ok=bytes(48 * 1024 + 1)) + + async def run(): + await initialize(target) + assert (await target.msg_refine_bundle(FuzzerMessage(refine_bundle=refine_request))).error == 'Oversized auth result' + assert vm.calls == [] + + asyncio.run(run()) + + +@pytest.mark.parametrize('count', [0, 1, 63, 64, 65, 129]) +def test_paged_proof_encoding(count): + exports = [i.to_bytes(4, 'little') * 1026 for i in range(count)] + tree = ConstantDepthMerkleTree(exports) + pages = paged_proof_segments(exports) + assert len(pages) == (count + 63) // 64 + for page_index, page in enumerate(pages): + stream = JamBytes(page) + path_codec, leaves_codec = Vec(H256).new(), Vec(H256).new() + path_codec.decode(stream) + leaves_codec.decode(stream) + path, leaves = path_codec.to_serializable_obj(), leaves_codec.to_serializable_obj() + start = page_index * 64 + assert len(page) == 4104 + assert leaves == [hash_blob(b'leaf' + s) for s in exports[start:start + 64]] + assert path == tree.trace(start)[:max(0, (count - 1).bit_length() - 6)] + assert not any(stream.get_remaining_bytes()) + + +def test_availability_matches_independent_per_segment_construction(refine_request): + # CC0 GP-0.8.0 tiny ec-3 vector from davxy/jam-test-vectors/erasure/tiny. + assert encode_shards(bytes.fromhex('615d17'), 3, 3) == [ + bytes.fromhex(s) for s in ('615d', '1700', '0000', '59d0', '0615', '756d') + ] + bundle = refine_request.bundle + data = BundleData.from_parts(bundle.work_package, bundle.extrinsics, bundle.import_segments, bundle.justifications) + encoded = data.assemble_bundle(bundle.work_package, refine_request.segment_root_lookup) + exports = [bytes([i]) * 4104 for i in range(3)] + originals = erasure_data_shards(6) + shard_lists = [encode_shards(s, originals, 6 - originals) for s in exports + paged_proof_segments(exports)] + + def node(values): + if not values: + return bytes(32) + if len(values) == 1: + return values[0] + split = (len(values) + 1) // 2 + return hash_blob(b'node' + node(values[:split]) + node(values[split:])) + + rows = list(zip(*shard_lists)) + bundle_shards = encode_shards(encoded, originals, 6 - originals) + expected = node([hash_blob(shard) + node(list(row)) for shard, row in zip(bundle_shards, rows)]) + spec = availability_spec(bundle.work_package.hash(), encoded, exports, 6) + assert spec.erasure_root == expected + assert spec.length == len(encoded) + assert spec.hash == bundle.work_package.hash() + + +def test_dynamic_erasure_dimensions(refine_request, monkeypatch): + from pyjamaz import erasure + from pyjamaz.graypaper_constants import valid_validator_count + # Exercise the full-domain counts even when pytest runs with Tiny constants. + monkeypatch.setattr(erasure, 'valid_validator_count', lambda n: valid_validator_count(n, core_count=341)) + for validators, originals in [(6, 3), (9, 4), (1020, 228), (1023, 342)]: + assert erasure_data_shards(validators) == originals + bundle = refine_request.bundle + spec = availability_spec(bundle.work_package.hash(), b'bundle', [bytes(4104)], validators) + assert spec.erasure_shards == validators + assert spec.erasure_root != bytes(32) + + +def test_frame_rejects_trailing_bytes(): + async def run(): + message = FuzzerMessage(state_root=bytes(32)).to_jam_bytes().to_bytes() + b'\0' + reader = asyncio.StreamReader() + reader.feed_data(len(message).to_bytes(4, 'little') + message) + reader.feed_eof() + with pytest.raises(ValueError, match='trailing bytes'): + await FuzzerMessage.fuzzer_decode(reader) + + asyncio.run(run()) + + +def test_socket_negotiation_and_rejection_recovery(target, refine_request, vm): + async def run(): + server = await asyncio.start_unix_server(target._handle_client, path=target.path) + try: + session = FuzzerSession(target.path, app=None) + await session.connect() + try: + assert session.features.refine + init = InitializeMessage(Header.default(), [(MARKER, bytes(4))], []) + await session.send_request(FuzzerMessage(initialize=init)) + invalid = deepcopy(refine_request) + invalid.bundle.extrinsics[0] = b'bad' + assert (await session.send_request(FuzzerMessage(refine_bundle=invalid))).error + expected = (await target.msg_refine_bundle(FuzzerMessage(refine_bundle=deepcopy(refine_request)))).work_report + response = await session.send_request(FuzzerMessage(refine_bundle=refine_request)) + assert response.work_report.to_jam_bytes().to_bytes() == expected.to_jam_bytes().to_bytes() + finally: + await session.close() + finally: + server.close() + await server.wait_closed() + + asyncio.run(run()) + + +def test_unnegotiated_and_uninitialized_refinement_closes_without_error(target, refine_request): + async def run(): + server = await asyncio.start_unix_server(target._handle_client, path=target.path) + try: + for negotiate, initialized in [(False, True), (True, False)]: + session = FuzzerSession(target.path, app=None) + reader, writer = await asyncio.open_unix_connection(target.path) + peer = session.msg_handshake() + peer.peer_info.features.refine = negotiate + writer.write(peer.fuzzer_encode()) + await writer.drain() + assert (await FuzzerMessage.fuzzer_decode(reader)).peer_info + if initialized: + writer.write(FuzzerMessage(initialize=InitializeMessage(Header.default(), [], [])).fuzzer_encode()) + await writer.drain() + assert (await FuzzerMessage.fuzzer_decode(reader)).state_root is not None + writer.write(FuzzerMessage(refine_bundle=refine_request).fuzzer_encode()) + await writer.drain() + assert await asyncio.wait_for(reader.read(), timeout=2) == b'' + writer.close() + await writer.wait_closed() + finally: + server.close() + await server.wait_closed() + + asyncio.run(run()) + + +def test_internal_refinement_error_and_duplicate_initialize_close_session(target, refine_request, monkeypatch): + def fail(*args, **kwargs): + raise RuntimeError('internal failure') + + monkeypatch.setattr(Refinement, 'compute_work_report', fail) + + async def run(): + server = await asyncio.start_unix_server(target._handle_client, path=target.path) + try: + for duplicate_init in (False, True): + session = FuzzerSession(target.path, app=None) + await session.connect() + init = FuzzerMessage(initialize=InitializeMessage(Header.default(), [], [])) + await session.send_request(init) + request_message = init if duplicate_init else FuzzerMessage(refine_bundle=refine_request) + session.writer.write(request_message.fuzzer_encode()) + await session.writer.drain() + assert await asyncio.wait_for(session.reader.read(), timeout=2) == b'' + await session.close() + finally: + server.close() + await server.wait_closed() + + asyncio.run(run()) + + +def test_trace_runner_reconnects_for_separate_chains(target, tmp_path, monkeypatch): + from pyjamaz import cli + + monkeypatch.setattr(cli, 'setup_logging', lambda *_: None) + initial = [(MARKER, bytes(4))] + posterior = [(MARKER, (1).to_bytes(4, 'little'))] + pre_state = StateDump(PatriciaMerkleTrie(initial).root(), initial) + post_state = StateDump(PatriciaMerkleTrie(posterior).root(), posterior) + genesis = Header.default() + header = Header.default() + header.parent, header.timeslot = genesis.hash, 1 + for name in ('first', 'second'): + directory = tmp_path / name + directory.mkdir() + (directory / 'genesis.bin').write_bytes(TraceGenesis(genesis, pre_state).to_jam_bytes().to_bytes()) + trace = Trace(pre_state, Block(header, Extrinsic.default()), post_state) + (directory / '00000001.bin').write_bytes(trace.to_jam_bytes().to_bytes()) + + async def run(): + server = await asyncio.start_unix_server(target._handle_client, path=target.path) + try: + result = await CliRunner().invoke(cli.main, [ + 'fuzzer', 'traces', str(tmp_path), '--socket-path', target.path, + ]) + assert result.exit_code == 0, result.output + assert target.app.state_storage.get(MARKER) == posterior[0][1] + finally: + server.close() + await server.wait_closed() + + asyncio.run(run()) diff --git a/test/test_runtime_refinement.py b/test/test_runtime_refinement.py new file mode 100644 index 00000000..d21dc110 --- /dev/null +++ b/test/test_runtime_refinement.py @@ -0,0 +1,250 @@ +import asyncio +from copy import deepcopy +from types import SimpleNamespace + +import pytest + +from pyjamaz import refinement as refinement_module +from pyjamaz.availability import BundleData, DataUnavailable, availability_spec, zero_pad +from pyjamaz.d3l import DataAvailabilityStore +from pyjamaz.erasure import encode_chunks, erasure_data_shards +from pyjamaz.models.app import D3LEntry +from pyjamaz.models.common import ( + Credential, ImportSegment, RefinementContext, WorkExecResult, WorkItem, WorkPackage, +) +from pyjamaz.models.state import PendingChanges, ServicesState +from pyjamaz.refinement import Refinement +from pyjamaz.runtime.pipelines import refine +from pyjamaz.state.storage import StateStorage +from pyjamaz.storage import InMemoryStorageEngine + + +def package(payload=b'package', imports=None): + return WorkPackage(0, bytes(32), RefinementContext( + bytes(32), bytes(32), bytes(32), bytes(32), 0, [], + ), b'', b'', [WorkItem(0, bytes(32), 1000, 1000, 2, payload, imports or [], [])]) + + +@pytest.fixture +def runtime(monkeypatch): + engine = InMemoryStorageEngine() + services = ServicesState() + services.set_state_storage(StateStorage(engine)) + services.pending_changes = PendingChanges() + app = SimpleNamespace( + working_state=SimpleNamespace( + services=services, state_root=bytes(32), timeslot=SimpleNamespace(number=3), + validator_pool=SimpleNamespace(validators=[SimpleNamespace(ed25519=bytes([i]) * 32) for i in range(6)]), + recent_history=SimpleNamespace(recent_blocks=[]), + ), + d3l_store=DataAvailabilityStore(InMemoryStorageEngine()), pubsub=None, + get_core_assigment=lambda: 0, retrieve_block_hash=lambda _: b'h' * 32, + block_context=SimpleNamespace(guarantor_assignments=[]), + config=SimpleNamespace(keys=SimpleNamespace(ed25519=SimpleNamespace(public_key=bytes(32)))), + ) + signed, invocations = [], [] + + async def sign(report): + signed.append(report) + return Credential(0, bytes(64)) + + app.create_guarantee_signature = sign + monkeypatch.setattr(WorkPackage, 'set_authorization_code', lambda *_: None) + monkeypatch.setattr(refinement_module, 'pvm_invoke_is_authorized', lambda *_: SimpleNamespace( + work_exec_result=WorkExecResult(ok=b'auth'), gas_used=1, + )) + + def invoke(**kwargs): + invocations.append(kwargs) + return SimpleNamespace(work_exec_result=WorkExecResult(ok=b'result'), gas_used=2, + export_segments=[b'a' * 4104, b'b' * 4104]) + + monkeypatch.setattr(refinement_module, 'pvm_invoke_refine', invoke) + pipeline = refine.RefinePipeline(app) + return SimpleNamespace(app=app, pipeline=pipeline, signed=signed, invocations=invocations, engine=engine) + + +def submit(runtime, value): + runtime.pipeline.add_work_package(value, []) + return runtime.pipeline._pending_work_packages[value.hash()] + + +def test_gp_interfaces_return_the_equations_results(runtime): + value = package() + data = BundleData() + # B(p,l) returns exactly the bytes, while Xi(p,c,l,v) returns exactly the report. + bundle = data.assemble_bundle(value, {}) + assert bundle == value.to_jam_bytes().to_bytes() + execution = Refinement(runtime.app.working_state.services.snapshot(), data) + report = execution.compute_work_report(value, 0, {}, 6) + # A takes the hash, rather than the work-package or a caller-supplied exports root. + assert report.package_spec == availability_spec(value.hash(), bundle, execution.exported_segments, 6) + assert report.package_spec.length == len(bundle) + assert execution.bundle == bundle + assert runtime.signed == [] + assert runtime.app.d3l_store.storage_engine.as_list() == [] + + +def test_runtime_resolves_tagged_and_duplicate_imports_from_retained_exports(runtime): + async def run(): + first = package(b'first') + first_item = submit(runtime, first) + await runtime.pipeline._refine_work_package(first_item) + root = first_item.work_report.package_spec.exports_root + assert runtime.app.d3l_store.segment_root_for_package(first.hash()) == root + assert runtime.app.d3l_store.storage_engine.get(b'bundle:' + first.hash()) + # Clear the queue cache to exercise the persisted mapping, not the in-memory report. + runtime.pipeline._pending_work_packages.clear() + second = package(b'second', [ + ImportSegment.from_work_package_hash(first.hash(), 1), + ImportSegment.from_segment_root(root, 0), + ImportSegment.from_work_package_hash(first.hash(), 1), + ]) + second_item = submit(runtime, second) + await runtime.pipeline._refine_work_package(second_item) + assert second_item.work_report.segment_root_lookup == {first.hash(): root} + assert runtime.invocations[-1]['work_items_import_segments'] == [[b'b' * 4104, b'a' * 4104, b'b' * 4104]] + assert len(runtime.signed) == 2 + assert second_item.status.Reported.reported_in.slot == 3 + + asyncio.run(run()) + + +def test_runtime_uses_recent_history_mapping_for_external_package(runtime): + source_hash = b's' * 32 + from pyjamaz.merkle import ConstantDepthMerkleTree + + exports = [b'x' * 4104, b'y' * 4104] + root = ConstantDepthMerkleTree(exports).root() + runtime.app.d3l_store.store_segments(D3LEntry(root, exports)) + runtime.app.working_state.recent_history.recent_blocks = [SimpleNamespace( + reported=[SimpleNamespace(hash=source_hash, exports_root=root)], + )] + value = package(imports=[ImportSegment.from_work_package_hash(source_hash, 0)]) + item = submit(runtime, value) + asyncio.run(runtime.pipeline._refine_work_package(item)) + assert item.work_report.segment_root_lookup == {source_hash: root} + + +@pytest.mark.parametrize('in_progress', ['transaction', 'candidate']) +def test_runtime_waits_for_block_transition_before_taking_snapshot(runtime, in_progress): + item = submit(runtime, package()) + storage = runtime.app.working_state.services.state_storage + if in_progress == 'transaction': + storage.transaction[b'k' * 31] = b'in progress' + else: + storage.block_hash = b'c' * 32 + asyncio.run(runtime.pipeline._refine_work_package(item)) + assert item.status.Reportable + assert runtime.invocations == runtime.signed == [] + + +def test_runtime_signs_only_its_own_credential_outside_solo_mode(runtime, monkeypatch): + monkeypatch.setattr(refine, 'SOLO_MODE', False) + runtime.app.block_context.guarantor_assignments = [SimpleNamespace( + validator_ed25519=b'z' * 32, core_index=0, + )] + + async def other_signature(*_): + pytest.fail("runtime attempted to sign for another validator") + + runtime.app.create_guarantee_signature_for_validator = other_signature + asyncio.run(runtime.pipeline._refine_work_package(submit(runtime, package()))) + assert len(runtime.signed) == 1 + + +@pytest.mark.parametrize('change', ['branch', 'validators', 'core']) +def test_changed_runtime_view_discards_result_and_keeps_inputs_for_retry(runtime, monkeypatch, change): + value = package() + item = submit(runtime, value) + runtime.engine.put(b'k' * 31, b'captured') + original_state = runtime.app.working_state + reports = [] + + async def dispatch(compute, *args): + runtime.engine.put(b'k' * 31, b'changed') + assert compute.__self__.services.state_storage.get(b'k' * 31) == b'captured' + reports.append(compute(*args)) + if change == 'branch': + runtime.app.working_state = deepcopy(original_state) + runtime.app.working_state.state_root = b'z' * 32 + elif change == 'validators': + original_state.validator_pool.validators[0].ed25519 = b'z' * 32 + else: + runtime.app.get_core_assigment = lambda: 1 + return reports[-1] + + monkeypatch.setattr(refine.anyio.to_thread, 'run_sync', dispatch) + asyncio.run(runtime.pipeline._refine_work_package(item)) + assert reports[0].package_spec.erasure_shards == 6 + assert item.status.Reportable is not None + assert item.work_report is None and item.signatures == [] + assert runtime.signed == [] + assert runtime.app.d3l_store.storage_engine.as_list() == [] + assert value.hash() in runtime.pipeline._work_package_extrinsics.extrinsic_data + + +def test_worker_rejects_one_package_then_processes_the_next(runtime): + invalid = package(b'invalid') + invalid.items = [] + bad_item, good_item = submit(runtime, invalid), submit(runtime, package(b'valid')) + + async def run(): + worker = asyncio.create_task(runtime.pipeline._refine_worker()) + try: + await runtime.pipeline._refine_queue.put(bad_item) + await runtime.pipeline._refine_queue.put(good_item) + await asyncio.wait_for(runtime.pipeline._refine_queue.join(), timeout=5) + finally: + worker.cancel() + with pytest.raises(asyncio.CancelledError): + await worker + + asyncio.run(run()) + assert bad_item.status.Failed + assert good_item.status.Reported + assert len(runtime.signed) == 1 + + +def test_unavailable_dependency_retries_without_killing_worker(runtime): + item = submit(runtime, package(imports=[ImportSegment.from_work_package_hash(b'x' * 32, 0)])) + + async def run(): + worker = asyncio.create_task(runtime.pipeline._refine_worker()) + try: + await runtime.pipeline._refine_queue.put(item) + await asyncio.wait_for(runtime.pipeline._refine_queue.join(), timeout=2) + finally: + worker.cancel() + with pytest.raises(asyncio.CancelledError): + await worker + + asyncio.run(run()) + assert item.status.Reportable + assert runtime.invocations == runtime.signed == [] + + +def test_services_snapshot_detaches_storage_and_pending_preimages(runtime): + services = runtime.app.working_state.services + key = (0, b'p' * 32) + runtime.engine.put(b'k' * 31, b'old') + services.pending_changes.preimages[key] = b'pending' + snapshot = services.snapshot() + runtime.engine.put(b'k' * 31, b'new') + services.pending_changes.preimages[key] = b'changed' + assert snapshot.state_storage.get(b'k' * 31) == b'old' + assert snapshot.pending_changes.preimages[key] == b'pending' + + +def test_erasure_signature_dimensions_and_padding(): + assert erasure_data_shards(6) == 3 + data = b'abc' + padded = zero_pad(6, data) + assert padded == b'abc' + bytes(3) + assert zero_pad(6, padded) == padded + assert len(encode_chunks(6, 1, padded)) == 6 + assert encode_chunks(6, 0, b'') == [b''] * 6 + with pytest.raises(ValueError, match='dimensions'): + encode_chunks(6, 1, data) + with pytest.raises(ValueError, match='piece count'): + encode_chunks(6, -1, b'') diff --git a/test/test_work_report_resize.py b/test/test_work_report_resize.py new file mode 100644 index 00000000..5bf582eb --- /dev/null +++ b/test/test_work_report_resize.py @@ -0,0 +1,90 @@ +import asyncio +from types import SimpleNamespace + +import pytest + +from pyjamaz import refinement as refinement_module, graypaper_constants as gp_const +from pyjamaz.availability import availability_spec +from pyjamaz.models.state import PendingChanges, ServicesState +from pyjamaz.state.storage import StateStorage +from pyjamaz.storage import InMemoryStorageEngine +from pyjamaz.models.common import ( + RefinementContext, WorkExecResult, WorkItem, WorkPackage, WorkReport, +) +from pyjamaz.runtime.pipelines import refine + + +@pytest.fixture +def work_package(): + return WorkPackage( + auth_code_host=0, + auth_code_hash=bytes(32), + context=RefinementContext(bytes(32), bytes(32), bytes(32), bytes(32), 0, []), + authorization=b"", + authorizer_config=b"", + items=[WorkItem(0, bytes(32), 1000, 1000, 0, b"", [], [])], + ) + + +@pytest.mark.parametrize("assurer_count", [6, 9]) +def test_refine_dispatch_preserves_assurer_count_across_live_set_change( + monkeypatch, work_package, assurer_count, +): + if assurer_count > gp_const.VALIDATOR_COUNT: + pytest.skip("Nine validators exceed the Tiny validator maximum") + + services = ServicesState() + services.set_state_storage(StateStorage(InMemoryStorageEngine())) + services.pending_changes = PendingChanges() + app = SimpleNamespace( + working_state=SimpleNamespace( + services=services, + state_root=bytes(32), + timeslot=SimpleNamespace(number=0), + recent_history=SimpleNamespace(recent_blocks=[]), + validator_pool=SimpleNamespace(validators=[SimpleNamespace(ed25519=bytes([i]) * 32) for i in range(assurer_count)]), + safrole=SimpleNamespace(validators=[None] * gp_const.VALIDATOR_COUNT), + ), + get_core_assigment=lambda: 0, + retrieve_block_hash=lambda _: bytes(32), + d3l_store=None, + ) + monkeypatch.setattr(WorkPackage, "set_authorization_code", lambda *_: None) + monkeypatch.setattr(refinement_module, "pvm_invoke_is_authorized", lambda *_: SimpleNamespace( + work_exec_result=WorkExecResult(ok=b"authorized"), gas_used=1, + )) + monkeypatch.setattr(refinement_module, "pvm_invoke_refine", lambda **_: SimpleNamespace( + work_exec_result=WorkExecResult(ok=b"result"), gas_used=2, export_segments=[], + )) + reports = [] + + async def dispatch(compute, *args): + # Model a live epoch change after worker arguments have been captured. + app.working_state.validator_pool.validators = [SimpleNamespace(ed25519=bytes([i % 256]) * 32) for i in range(gp_const.VALIDATOR_COUNT)] + reports.append(compute(*args)) + # Stop this worker after inspecting the real report-construction path. + raise asyncio.CancelledError + + monkeypatch.setattr(refine.anyio.to_thread, "run_sync", dispatch) + + async def run_worker(): + pipeline = refine.RefinePipeline(app) + pipeline.add_work_package(work_package, []) + await pipeline._refine_queue.put(pipeline._pending_work_packages[work_package.hash()]) + with pytest.raises(asyncio.CancelledError): + await pipeline._refine_worker() + + asyncio.run(run_worker()) + + assert len(reports) == 1 + report = reports[0] + assert report.package_spec.erasure_shards == assurer_count + decoded = WorkReport.from_jam_bytes(report.to_jam_bytes()) + assert decoded.package_spec.erasure_shards == assurer_count + assert decoded.results[0].result.ok == b"result" + + +@pytest.mark.parametrize("count", [0, 5, 7, 3 * gp_const.CORE_COUNT + 3, True]) +def test_package_spec_rejects_invalid_assurer_count(work_package, count): + with pytest.raises(ValueError, match="invalid assurer count"): + availability_spec(work_package.hash(), b'bundle', [], count) From 87eb013b4ee8f73614b59a8676c90a2a24160ffa Mon Sep 17 00:00:00 2001 From: Arjan Zijderveld <5286904+arjanz@users.noreply.github.com> Date: Thu, 29 Oct 2026 14:08:06 +0100 Subject: [PATCH 2/4] Equation updates --- pyjamaz/models/common.py | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/pyjamaz/models/common.py b/pyjamaz/models/common.py index 0e0f4245..5f42bce8 100644 --- a/pyjamaz/models/common.py +++ b/pyjamaz/models/common.py @@ -174,7 +174,7 @@ class ImportSegment(Serializable): Attributes ---------- tree_root: H256 - GP-0.8.0-eq:14.3 (blackboard_H) | Root of the segment tree. # TODO what about H^[+] ? + GP-0.8.0-eq:14.3 (blackboard_H) | Root of the segment tree. index: U16 GP-0.8.0-eq:14.3 (blackboard_N type derived from encoding appendix) | Index into the segment tree. """ @@ -191,7 +191,7 @@ def is_work_package_hash(self) -> bool: @property def segment_index(self) -> int: - """The logical export index after removing C.35's discriminator.""" + """The logical export index after removing C.37's discriminator.""" return self.index & self.SEGMENT_INDEX_MASK @classmethod @@ -428,13 +428,13 @@ class WorkExecResult(Serializable): GP-0.8.0-eq:11.7 (BIG) | Code oversize error. """ # TODO: JSON labels for out_of_gas (out-of-gas), bad_code (bad-code) and code_oversize (code-oversize) don't match - ok: bytes = field(default=None, metadata={'codec': Bytes}) - out_of_gas: bool = field(default=None, metadata={'codec': Null}) - panic: bool = field(default=None, metadata={'codec': Null}) - bad_exports: bool = field(default=None, metadata={'codec': Null}) - digest_oversize: bool = field(default=None, metadata={'codec': Null}) - bad_code: bool = field(default=None, metadata={'codec': Null}) - code_oversize: bool = field(default=None, metadata={'codec': Null}) + ok: typing.Optional[bytes] = field(default=None, metadata={'codec': Bytes}) + out_of_gas: typing.Optional[bool] = field(default=None, metadata={'codec': Null}) + panic: typing.Optional[bool] = field(default=None, metadata={'codec': Null}) + bad_exports: typing.Optional[bool] = field(default=None, metadata={'codec': Null}) + digest_oversize: typing.Optional[bool] = field(default=None, metadata={'codec': Null}) + bad_code: typing.Optional[bool] = field(default=None, metadata={'codec': Null}) + code_oversize: typing.Optional[bool] = field(default=None, metadata={'codec': Null}) _codec_enum = True From 764c11b1f83d762bbb44b4b35b86d329bb6cd8de Mon Sep 17 00:00:00 2001 From: Arjan Zijderveld <5286904+arjanz@users.noreply.github.com> Date: Fri, 30 Oct 2026 13:39:37 +0100 Subject: [PATCH 3/4] Modified piece_count to reflect GP equation 14.18 --- pyjamaz/availability.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/pyjamaz/availability.py b/pyjamaz/availability.py index ccec75eb..a575bb33 100644 --- a/pyjamaz/availability.py +++ b/pyjamaz/availability.py @@ -189,8 +189,10 @@ def availability_spec(package_hash: bytes, bundle: bytes, exports: list[bytes], raise ProcessWorkpackageError("invalid exported segments") originals = erasure_data_shards(validator_count) alignment = 2 * originals + # GP-0.8.0-eq:14.18: k = ceil(|b| / z), where z = 2 D(v). + piece_count = (len(bundle) + alignment - 1) // alignment padded_bundle = zero_pad(alignment, bundle) - bundle_shards = encode_chunks(validator_count, len(padded_bundle) // alignment, padded_bundle) + bundle_shards = encode_chunks(validator_count, piece_count, padded_bundle) segments = exports + paged_proof_segments(exports) segment_shard_size = EC_SEGMENT_SIZE // originals # H.4-H.5: transpose segments so every validator row contains one shard of each segment. From 055e31a08cfe349d9c5fffda4aeb5e416585fa35 Mon Sep 17 00:00:00 2001 From: Arjan Zijderveld <5286904+arjanz@users.noreply.github.com> Date: Fri, 30 Oct 2026 15:48:45 +0100 Subject: [PATCH 4/4] Implemented refine feature for trace replay fuzzer --- pyjamaz/cli.py | 108 ++++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 106 insertions(+), 2 deletions(-) diff --git a/pyjamaz/cli.py b/pyjamaz/cli.py index dc596008..d556db2b 100644 --- a/pyjamaz/cli.py +++ b/pyjamaz/cli.py @@ -789,7 +789,92 @@ async def replay_traces( f"{mismatch_count} trace state-root mismatch{'es' if mismatch_count != 1 else ''}" ) -@fuzzer.command('traces', help='Start Fuzzer target over UNIX socket.') +def _report_differences(expected, actual, field_path=''): + """Describe report fields without hiding gas, output or commitment differences.""" + if is_dataclass(expected) and is_dataclass(actual): + for item in fields(expected): + name = f'{field_path}.{item.name}' if field_path else item.name + yield from _report_differences(getattr(expected, item.name), getattr(actual, item.name), name) + elif isinstance(expected, (list, tuple)) and isinstance(actual, (list, tuple)): + if len(expected) != len(actual): + yield f'{field_path}.length: expected={len(expected)} actual={len(actual)}' + for index, (left, right) in enumerate(zip(expected, actual)): + yield from _report_differences(left, right, f'{field_path}[{index}]') + elif isinstance(expected, dict) and isinstance(actual, dict): + for key in sorted(expected.keys() | actual.keys()): + name = f'{field_path}[{key.hex() if isinstance(key, bytes) else key!s}]' + if key not in actual: + yield f'{name}: missing from actual report' + elif key not in expected: + yield f'{name}: unexpected in actual report' + else: + yield from _report_differences(expected[key], actual[key], name) + elif expected != actual: + def display(value): + if isinstance(value, bytes): + return '0x' + value.hex() + return repr(value) + yield f'{field_path}: expected={display(expected)} actual={display(actual)}' + + +def _read_refinement_fixture(file, codec): + if not file.is_file(): + raise click.ClickException(f'Incomplete refinement fixture: missing {file}') + try: + raw = file.read_bytes() + stream = JamBytes(raw) + value = codec.from_jam_bytes(stream) + if stream.get_remaining_length() or value.to_jam_bytes().to_bytes() != raw: + raise ValueError('noncanonical encoding or trailing bytes') + return value + except Exception as exc: + raise click.ClickException(f'Invalid refinement fixture {file}: {exc}') from exc + + +async def _replay_refinement_trace(session, fixture_file): + from pyjamaz.fuzzer.v1.types import RefineBundle + from pyjamaz.models.common import WorkReport + + folder = fixture_file.parent + genesis = _read_refinement_fixture(folder / 'genesis.bin', TraceGenesis) + request = _read_refinement_fixture(fixture_file, RefineBundle) + expected = _read_refinement_fixture(folder / 'report.bin', WorkReport) + if not session.features.refine: + raise click.ClickException(f'Refinement was not negotiated; target cannot replay {fixture_file}') + + response = await session.send_request(FuzzerMessage(initialize=InitializeMessage( + header=genesis.header, state=genesis.state.keyvals, ancestry=[], + ))) + if response.state_root != genesis.state.state_root: + raise click.ClickException(f'Fuzzer initial state root mismatch in refinement fixture {folder}') + + response = await session.send_request(FuzzerMessage(refine_bundle=request)) + failures = [] + if response.error is not None: + failures.append(f'Fuzzer RefineBundle failed: {response.error}') + elif response.work_report is None: + failures.append('Unexpected fuzzer response to RefineBundle: expected work_report') + elif response.work_report.to_jam_bytes().to_bytes() != expected.to_jam_bytes().to_bytes(): + failures.append('Work report mismatch:\n' + '\n'.join( + _report_differences(expected, response.work_report), + )) + + state_response = await session.send_request(FuzzerMessage(get_state=genesis.header.hash)) + if state_response.state is None: + failures.append(f'Fuzzer GetState failed after refinement: {state_response.error or "expected state"}') + else: + state_root = PatriciaMerkleTrie(state_response.state).root() + if state_root != genesis.state.state_root: + failures.append( + f'Refinement changed initialized state: expected={format_hash(genesis.state.state_root)} ' + f'actual={format_hash(state_root)}' + ) + if failures: + raise click.ClickException(f'Refinement trace {fixture_file}:\n' + '\n'.join(failures)) + logging.info(f'βœ… Refinement report matches {folder / "report.bin"}; initialized state unchanged') + + +@fuzzer.command('traces', help='Replay block and refinement traces against a fuzzer target over UNIX socket.') @click.argument('traces_dir', type=click.Path(exists=True)) @click.option('--socket-path', 'socket_path', type=str, default="/tmp/jam_target.sock", show_default=True) @click.option('--verbose', is_flag=True, help="Enable verbose output") @@ -805,6 +890,7 @@ async def fuzzer_traces(traces_dir: str, socket_path: str, verbose: bool, prompt connected = False mismatch_count = 0 last_parent = None + last_was_refinement = False def require_response(response, field, operation): if response.error is not None: @@ -823,12 +909,29 @@ def require_response(response, field, operation): traces_files = await anyio.to_thread.run_sync( lambda: sorted(f for f in traces_folder.rglob("*.bin") if f.name not in ['genesis.bin', 'report.bin']), ) + for report_file in traces_folder.rglob('report.bin'): + if not any(candidate.parent == report_file.parent for candidate in traces_files): + raise click.ClickException( + f'Incomplete refinement fixture: missing {report_file.parent / "refine.bin"}' + ) for nr, block_file in enumerate(traces_files, start=1): logging.info(f'πŸ“‚ Processing trace file {block_file}') + if block_file.name == 'refine.bin': + # Refinement fixtures always start from their own genesis state, + # even when the directory also contains block traces. + if last_parent is not None: + await fuzzer_session.close() + connected = False + await fuzzer_session.connect() + connected = True + await _replay_refinement_trace(fuzzer_session, block_file) + last_parent = block_file.parent + last_was_refinement = True + continue trace = Trace.from_jam_bytes(JamBytes(block_file.read_bytes())) - if block_file.parent != last_parent: + if block_file.parent != last_parent or last_was_refinement: if last_parent is not None: # Each trace chain has its own handshake and Initialize. await fuzzer_session.close() @@ -876,6 +979,7 @@ def require_response(response, field, operation): ) last_parent = block_file.parent + last_was_refinement = False if checkpoint: logging.info( f"πŸ“Œ Using {block_file.name} as initialization checkpoint "