Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 3 additions & 1 deletion pyjamaz/settings.py
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@

TEST_SUITE = 'tiny' # tiny, full or fuzz
GP_VERSION = '0.8.0'
APP_VERSION = '0.3.9'
APP_VERSION = '0.3.10'


def _apply_jam_fuzz_spec_override():
Expand Down Expand Up @@ -43,6 +43,8 @@ def _apply_jam_fuzz_spec_override():
DEBUG_PROGRAM_OVERRIDE = {}

USE_THREAD_POOL_SAFROLE = True
# Native VRF verification also uses threads; bound concurrent ticket jobs.
SAFROLE_THREAD_POOL_MAX_WORKERS = min(4, os.cpu_count() or 1)
USE_THREAD_POOL_ACCUMULATE = False
THREAD_POOL_MAX_WORKERS = os.cpu_count()

Expand Down
91 changes: 48 additions & 43 deletions pyjamaz/state/components.py
Original file line number Diff line number Diff line change
@@ -1,8 +1,9 @@
import bisect
import logging
from concurrent.futures import as_completed
from concurrent.futures import as_completed, wait
from concurrent.futures.thread import ThreadPoolExecutor
from copy import deepcopy, copy
from threading import Lock
from typing import List, Union, Optional, Set, Dict

from bandersnatch_vrfs import RingContext, ietf_vrf_verify
Expand All @@ -19,7 +20,7 @@
from pyjamaz.hostcalls.models import PvmAccumulateOutput
from pyjamaz.merkle import MerkleMountainRange
from pyjamaz.settings import SOLO_MODE, THREAD_POOL_MAX_WORKERS, USE_THREAD_POOL_SAFROLE, DEBUG, \
USE_THREAD_POOL_ACCUMULATE
USE_THREAD_POOL_ACCUMULATE, SAFROLE_THREAD_POOL_MAX_WORKERS
from pyjamaz.signing import Ed25519Keypair, jam_ring_context
from pyjamaz.storage import Transaction
from pyjamaz.models.common import ValidatorData, WorkReport, TicketBody, DeferredTransfer, AccumulationInput, \
Expand Down Expand Up @@ -49,6 +50,23 @@
from pyjamaz.utils import reorder_list_outside_in, list_has_duplicates, format_hash, log_execution_time, sum_dict_values


_safrole_ticket_executor = None
_safrole_ticket_executor_lock = Lock()


def _get_safrole_ticket_executor():
"""Share a lazy verifier pool across blocks and app instances."""
global _safrole_ticket_executor
with _safrole_ticket_executor_lock:
if _safrole_ticket_executor is None:
# ThreadPoolExecutor joins its workers when the process exits.
_safrole_ticket_executor = ThreadPoolExecutor(
max_workers=SAFROLE_THREAD_POOL_MAX_WORKERS,
thread_name_prefix='safrole',
)
return _safrole_ticket_executor


class Timeslot(StateComponent):
component_id = 11

Expand Down Expand Up @@ -425,7 +443,7 @@ def state_transition(
# Don't accept tickets after TICKET_SUBMISSION_END_SLOT:
raise StateTransitionError(SafroleErrorCode.unexpected_ticket)

input_tickets = [None] * len(extrinsic_tickets)
input_tickets: List[Optional[TicketBody]] = [None] * len(extrinsic_tickets)

if len(extrinsic_tickets) > 0:

Expand All @@ -447,49 +465,36 @@ def state_transition(

ring_context = jam_ring_context(self.ring_data, ring_public_keys)

if USE_THREAD_POOL_SAFROLE:

DEBUG and logging.debug(f'Using ThreadPool max_workers={THREAD_POOL_MAX_WORKERS}')

with ThreadPoolExecutor(max_workers=THREAD_POOL_MAX_WORKERS) as tp:
futs = {
tp.submit(
self.create_ticket_body,
ticket_data,
ring_context,
post_state_entropy.entropy[2]
): idx
for idx, ticket_data in enumerate(extrinsic_tickets)
}

for fut in as_completed(futs):
ticket = fut.result()
idx = futs[fut]

# Check if ticket already exists
if any(
existing.id == ticket.id
for existing in self.post_state_safrole.ticket_accumulator
):
# GP-0.8.0-eq:6.34
raise StateTransitionError(SafroleErrorCode.duplicate_ticket)
else:
input_tickets[idx] = ticket
else:
# Validate extrinsic
for idx, ticket_data in enumerate(extrinsic_tickets):

ticket = self.create_ticket_body(ticket_data, ring_context, post_state_entropy.entropy[2])
futures = []
try:
if USE_THREAD_POOL_SAFROLE and len(extrinsic_tickets) > 1:
executor = _get_safrole_ticket_executor()
for ticket_data in extrinsic_tickets:
futures.append(executor.submit(
self.create_ticket_body, ticket_data, ring_context,
post_state_entropy.entropy[2],
))
# Consume in extrinsic order, preserving sequential error
# precedence even when later proofs finish first.
tickets = (future.result() for future in futures)
else:
tickets = (
self.create_ticket_body(ticket_data, ring_context, post_state_entropy.entropy[2])
for ticket_data in extrinsic_tickets
)

# Check if ticket already exists
if any(
existing.id == ticket.id
for existing in self.post_state_safrole.ticket_accumulator
):
for idx, ticket in enumerate(tickets):
if any(existing.id == ticket.id for existing in self.post_state_safrole.ticket_accumulator):
# GP-0.8.0-eq:6.34
raise StateTransitionError(SafroleErrorCode.duplicate_ticket)
else:
input_tickets[idx] = ticket
input_tickets[idx] = ticket
finally:
if futures:
for future in futures:
future.cancel()
# Workers read this component's post-state. Finish running
# jobs before a rejected block can be followed by another.
wait(futures)

# Check if tickets are in order: GP-0.8.0-eq:6.33
if not self.tickets_in_order(input_tickets):
Expand Down
2 changes: 1 addition & 1 deletion requirements.txt
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,7 @@ py-bip39-bindings~=0.3
rocksdict~=0.3

bandersnatch-vrfs==0.6.3
jamcodec~=0.1.8
jamcodec~=0.1.9
py-jam-erasure-bindings==0.1.0

numpy==2.3.5
Expand Down
Loading