From cdcc297b06d6f23d4083314b0aa011c08601b0e9 Mon Sep 17 00:00:00 2001 From: HusseinAdeiza Date: Sun, 27 Sep 2026 23:26:19 +0200 Subject: [PATCH] test(cross-major): extract the shared manager harness into gvm_extra `cross-major-observability` drove the manager by loading `cross-major/test.py` through `importlib.util.spec_from_file_location` and then reaching across that module boundary for ten names, eight of them private. The path coupling breaks silently if the sibling directory is renamed, and the duplication was already real: the observability case carried its own `_execute` and `_lvs_extended` because the originals could not be parameterized. `tests/runner/gvm_extra/` is already on the test path, so the shared mechanism moves there as `gvm_extra/cross_major.py`: the case and step bases, the call encoding helpers, and the host plumbing. Each case keeps its own fixtures, addresses and assertions. Three deliberate changes let one definition serve both cases: - `_execute` grows a `permissions` argument, so the observability override disappears. It is only sent when a case asks for it, which keeps cross-major's request payload unchanged. - `_execute`'s `host_data` tx_id becomes `host_data_tx_id`, set by the new `_setup` helper, so each case labels its own runs. - `_lvs` takes `**kwargs` instead of naming four parameters, which is what `_lvs_extended` was for. Every other moved body is byte-identical apart from the `_TestContext` to `TestContext` rename. Verified by collecting both cases before and after and diffing name, method, fixtures, arguments and tags: 20 cross-major and 23 observability cases, identical in both runs. `ruff check` is at parity with the pre-change baseline (the one remaining BLE001 predates this change). The runner-artifact tier is unrun: runner artifacts build on Linux only and this was not built locally. (cherry picked from commit 309d45ce7b233b231684068c85b597dc1724bd19) --- tests/runner/gvm_extra/cross_major.py | 382 ++++++++++++++++++ .../system/cross-major-observability/test.py | 159 ++------ tests/system/cross-major/test.py | 353 +--------------- 3 files changed, 429 insertions(+), 465 deletions(-) create mode 100644 tests/runner/gvm_extra/cross_major.py diff --git a/tests/runner/gvm_extra/cross_major.py b/tests/runner/gvm_extra/cross_major.py new file mode 100644 index 0000000..c7c603c --- /dev/null +++ b/tests/runner/gvm_extra/cross_major.py @@ -0,0 +1,382 @@ +""" +The harness shared by the cross-major system test cases. + +`tests/system/cross-major/` and `tests/system/cross-major-observability/` +both drive a real manager over the socket with a mock host, so the pieces +they have in common live here: the case/step bases, the call encoding +helpers, and the host plumbing (`_new_host`, `_execute`, `_deploy`, +`_route`, `_lvs`). + +Each case owns its own fixtures, addresses and assertions; only the +mechanism is shared. +""" + +import asyncio +import json +import os +import pickle +import typing +from dataclasses import dataclass +from pathlib import Path + +import genvm_tool.io as gvm_io +import genvm_tool.tests +import genvm_tool.tests.exec.step +import genvm_tool.tests.stage.collection +import genvm_tool.tests.test +import origin.calldata as gvm_calldata +from genvm_tool_plugins import runners +from gvm_extra.mock_host import MockHost, MockStorage +from origin import base_host, host_fns, public_abi +from origin.calldata import Address +from origin.leader_public_data import LeaderPublicData + +SENDER = Address('0x' + '01' * 20) +TIMESTAMP = '2024-11-26T06:42:42.424242Z' + + +def leader_public_data(output: bytes) -> bytes: + return LeaderPublicData([output]).encode() + + +def resolve_runners(code: bytes | None, root_dir: Path) -> bytes | None: + """ + Turns the `@RUNNER_LATEST_...@` tokens of a contract source into uids. + + Every comparison against a deployed contract's bytes has to go through this + too, because storage holds what the executor was given. + """ + return None if code is None else runners.substitute(code, root_dir) + + +def contract_source(line: int, name: str, assets_dir: Path) -> bytes: + """ + Reads a contract asset written against `line`'s sdk. + + `assets_dir` is required rather than defaulted: the two cases keep their + assets in their own directories, so the caller says which. + """ + return (assets_dir / f'v0.{line}' / f'{name}.py').read_bytes() + + +class TestContext(base_host.Context): + def __init__(self, logger: base_host.Logger): + self.logger = logger + self.stats: dict[str, typing.Any] = {} + + def on_genvm_success(self): ... + def on_genvm_failure(self): ... + + def add_stat(self, key: str, value: typing.Any, /): + self.stats[key] = value + + +def message(address: Address, *, is_init: bool) -> base_host.Message: + return { + 'contract_address': address, + 'sender_address': SENDER, + 'origin_address': SENDER, + 'signer_address': SENDER, + 'chain_id': 61999, + 'value': 0, + 'is_init': is_init, + 'transaction_timestamp': TIMESTAMP, + } + + +def calldata(method: str, *args: typing.Any) -> bytes: + return gvm_calldata.encode({'': method, 'args': list(args)}) + + +def apply_storage_deltas( + storage: MockStorage, + address: Address, + changes: list[tuple[bytes, bytes]], +) -> None: + for key, value in changes: + assert len(key) == 36 + storage.write( + address, + key[:32], + int.from_bytes(key[32:], byteorder='big') * 32, + value, + ) + + +def assert_hashes_agree(label: str, runs: list[tuple[str, typing.Any]]) -> None: + """ + A disagreement is only actionable once the failure says who disagreed. + """ + hashes = [(name, run.execution_hash.hex()) for name, run in runs] + assert len({digest for _name, digest in hashes}) == 1, '\n'.join( + [f'{label}: execution hashes disagree'] + + [f' {name}: {digest}' for name, digest in hashes] + ) + + +@dataclass +class CrossMajorCase(genvm_tool.tests.test.Case): + """ + A case that deploys its fixtures against a manager service and then runs + one named assertion method. + + Subclasses set the step: `into_steps` is what distinguishes a case that + drives `CrossMajorStep` from one that drives its own. + """ + + description: genvm_tool.tests.test.Description + shared: genvm_tool.tests.SharedContext + manager_service: genvm_tool.tests.stage.collection.Service + method: str + fixtures: tuple[str, ...] + arguments: tuple[typing.Any, ...] = () + + async def into_steps(self) -> list[genvm_tool.tests.exec.step.Step]: + raise NotImplementedError + + +class CrossMajorStep(genvm_tool.tests.exec.step.Python): + """ + Drives a real manager over the socket with a mock host. + + Subclasses implement `_run_all`: it resolves the build, prepares the + storage file, deploys the case's fixtures and dispatches `self.case.method`. + Everything below is mechanism, and is shared by both cross-major cases. + """ + + def __init__(self, case: CrossMajorCase): + self.case = case + self.phase = 'start' + self.notes: list[typing.Any] = [] + + def to_str(self) -> str: + return '' + + async def run( + self, previous_results: list[typing.Any] + ) -> genvm_tool.tests.test.Result: + try: + await self._run_all() + return genvm_tool.tests.test.Result( + passed=True, context={'notes': self.notes}, elapsed_seconds=0 + ) + except BaseException as exc: + return genvm_tool.tests.test.Result( + passed=False, + context={'phase': self.phase, 'error': exc, 'notes': self.notes}, + elapsed_seconds=0, + ) + + async def _run_all(self): + raise NotImplementedError + + async def _setup(self, *, host_data_tx_id: str) -> None: + """ + Resolves the build, prepares the storage file and grabs the manager + handle. Every case does this before deploying its fixtures. + """ + root = self.case.shared.root_dir + build_info = json.loads((root / 'build' / 'info.json').read_text()) + self.build_dir = Path(build_info['build_dir']) + self.versions = { + 2: build_info['executor_versions']['v0.2'], + 3: build_info['executor_versions']['v0.3'], + } + self.work_dir = self.case.shared.case_dir_for(self.case.description.name) + self.storage_path = self.work_dir / 'storage.pickle' + self.work_dir.mkdir(parents=True, exist_ok=True) + await gvm_io.write_file_bytes(self.storage_path, pickle.dumps(MockStorage())) + self.host_data_tx_id = host_data_tx_id + + manager = self.case.manager_service.handle + assert manager is not None + self.manager = manager + + async def _new_host( + self, + name: str, + running_address: Address, + resolve_hook=None, + read_log: list[tuple[Address, public_abi.StorageView]] | None = None, + host_fuel: int | None = None, + ) -> MockHost: + ctx = TestContext(self.case.shared.logger) + host_path = ( + Path('/tmp') + / f'gvm-cross-major-{os.getpid()}' + / f'{name[:20]}-{os.urandom(4).hex()}.sock' + ) + host_path.parent.mkdir(parents=True, exist_ok=True) + host = MockHost( + ctx=ctx, + path=str(host_path), + storage_path_pre=self.storage_path, + storage_path_post=self.storage_path, + balances={}, + running_address=running_address, + resolve_call_contract_executor_hook=resolve_hook, + ) + if read_log is not None: + original = host.storage_read + + async def logged(mode, address, slot, offset, le, /): + read_log.append((Address(address), mode)) + return await original(mode, address, slot, offset, le) + + host.storage_read = logged # type: ignore[method-assign] + if host_fuel is not None: + + async def get_remaining_time_fee_gen_wei() -> int: + return host_fuel + + host.get_remaining_time_fee_gen_wei = ( # type: ignore[method-assign] + get_remaining_time_fee_gen_wei + ) + return host + + async def _execute( + self, + *, + name: str, + line: int, + address: Address, + calldata: bytes, + code: bytes | None, + is_init: bool, + resolve_hook=None, + timeout: float = 30, + debug_initial_recursion: int | None = None, + is_sync: bool = True, + leader_public_data: bytes | None = None, + apply_changes: bool = True, + read_log: list[tuple[Address, public_abi.StorageView]] | None = None, + host_fuel: int | None = None, + hook_cross_contract_calls: bool = True, + permissions: str | None = None, + ): + host = await self._new_host( + name, address, resolve_hook, read_log=read_log, host_fuel=host_fuel + ) + ctx = host.ctx + with host as mock_host: + try: + async with base_host.ManagerClient(self.manager.uri) as manager_client: + result = await base_host.run_genvm( + mock_host, + manager_uri=self.manager.uri, + manager_client=manager_client, + ctx=ctx, + is_sync=is_sync, + leader_public_data=leader_public_data, + message=message(address, is_init=is_init), + host_data='{"node_address":"test","tx_id":"' + self.host_data_tx_id + '"}', + host='unix://' + mock_host.path, + code=resolve_runners(code, self.case.shared.root_dir), + calldata=calldata, + timeout=timeout, + debug_mode='unsafe', + unsafe_overrides=base_host.UnsafeOverrides( + reroute_to=self.versions[line], + initial_recursion=debug_initial_recursion, + ), + request_extra=self._request_extra( + hook_cross_contract_calls=hook_cross_contract_calls, + permissions=permissions, + ), + bucket_totals=base_host.default_bucket_totals(line), + ) + if apply_changes and result.result_kind == host_fns.ResultCode.RETURN: + assert mock_host.storage is not None + apply_storage_deltas( + mock_host.storage, + address, + result.result_storage_deltas, + ) + return result + finally: + await host.stop_connections() + + def _request_extra( + self, + *, + hook_cross_contract_calls: bool, + permissions: str | None, + ) -> dict[str, typing.Any]: + """ + `permissions` is only sent when the case asks for it: the observability + case varies it per call, while cross-major leaves it to the executor + default. + """ + request_extra: dict[str, typing.Any] = { + 'no_modules': True, + 'hook_cross_contract_calls': hook_cross_contract_calls, + } + if permissions is not None: + request_extra['permissions'] = permissions + return request_extra + + async def _deploy(self, line: int, address: Address, code: bytes): + result = await self._execute( + name=f'deploy-{line}-{address.as_bytes.hex()}', + line=line, + address=address, + calldata=gvm_calldata.encode({}), + code=code, + is_init=True, + timeout=10 * 60, + ) + assert result.result_kind == host_fns.ResultCode.RETURN, result + assert len(result.execution_hash) == 32, result.execution_hash + + async def _compile_wat(self, wat: str, path: Path) -> bytes: + await gvm_io.make_dir(path) + wat_path = path / 'contract.wat' + wasm_path = path / 'contract.wasm' + await gvm_io.write_file_text(wat_path, wat) + process = await asyncio.create_subprocess_exec( + 'wat2wasm', + '--enable-annotations', + '-o', + str(wasm_path), + str(wat_path), + stdout=asyncio.subprocess.PIPE, + stderr=asyncio.subprocess.PIPE, + ) + stdout, stderr = await process.communicate() + assert process.returncode == 0, (stdout, stderr) + return await gvm_io.read_file_bytes(wasm_path) + + def _route(self, line: int) -> bytes: + """ + Routing payload placing a callee on `line`'s executor. + + It names the directory rather than the major, because a major cannot + pick between the live lines: every manifest key is semver major 0, so + `{'kind': 'major'}` resolves to the newest line whatever it asks for. + """ + return gvm_calldata.encode({'kind': 'version', 'version': self.versions[line]}) + + async def _lvs(self, *, name: str, **kwargs): + """ + Run the same step as leader, validator and sync run. + + The three must agree on the execution hash; anything else is a + determinism violation an honest validator would see as a disagreement. + """ + + async def one(suffix: str, **mode): + return await self._execute( + name=f'{name}-{suffix}', + code=None, + is_init=False, + apply_changes=False, + **kwargs, + **mode, + ) + + leader = await one('leader', is_sync=False) + validator = await one( + 'validator', is_sync=False, leader_public_data=leader.result_leader_public_data + ) + sync = await one('sync', is_sync=True) + return leader, validator, sync diff --git a/tests/system/cross-major-observability/test.py b/tests/system/cross-major-observability/test.py index ab279f5..8797f88 100644 --- a/tests/system/cross-major-observability/test.py +++ b/tests/system/cross-major-observability/test.py @@ -1,39 +1,23 @@ """ Cross-major CallContract observability and determinism integration tests. -This case reuses the real-manager harness from ``tests/system/cross-major`` -without adding more concerns to that already-large case. +It drives the manager with the shared harness in ``gvm_extra.cross_major``. """ import functools -import importlib.util -import json -import pickle -import sys -import typing from dataclasses import dataclass from pathlib import Path -import genvm_tool.io as gvm_io import genvm_tool.tests import genvm_tool.tests.stage.collection -from gvm_extra.mock_host import MockStorage +import origin.calldata as gvm_calldata +from gvm_extra import cross_major as shared +from gvm_extra.cross_major import SENDER, contract_source +from gvm_extra.cross_major import calldata as _calldata +from gvm_extra.cross_major import leader_public_data as _leader_public_data from origin import host_fns, public_abi from origin.calldata import Address - -def _load_cross_major_harness(): - path = Path(__file__).parents[1] / 'cross-major' / 'test.py' - spec = importlib.util.spec_from_file_location('cross_major_system_harness', path) - assert spec is not None and spec.loader is not None - module = importlib.util.module_from_spec(spec) - sys.modules[spec.name] = module - spec.loader.exec_module(module) - return module - - -base = _load_cross_major_harness() - ADDR_CALLER_V02 = Address('0x' + '41' * 20) ADDR_OBSERVER_V03 = Address('0x' + '42' * 20) ADDR_HELPER_V03 = Address('0x' + '43' * 20) @@ -50,7 +34,7 @@ def _load_cross_major_harness(): _contract_source = functools.partial( - base._contract_source, + contract_source, assets_dir=Path(__file__).resolve().parent / 'assets', ) @@ -76,35 +60,16 @@ def _load_cross_major_harness(): @dataclass -class ObservabilityCase(genvm_tool.tests.test.Case): - description: genvm_tool.tests.test.Description - shared: genvm_tool.tests.SharedContext - manager_service: genvm_tool.tests.stage.collection.Service - method: str - fixtures: tuple[str, ...] - arguments: tuple[typing.Any, ...] = () - +class ObservabilityCase(shared.CrossMajorCase): async def into_steps(self) -> list[genvm_tool.tests.exec.step.Step]: return [ObservabilityStep(self)] -class ObservabilityStep(base.CrossMajorStep): +# This case owns its fixtures, addresses and assertions; the host plumbing and +# the call encoding come from `gvm_extra.cross_major`. +class ObservabilityStep(shared.CrossMajorStep): async def _run_all(self): - root = self.case.shared.root_dir - build_info = json.loads((root / 'build' / 'info.json').read_text()) - self.build_dir = Path(build_info['build_dir']) - self.versions = { - 2: build_info['executor_versions']['v0.2'], - 3: build_info['executor_versions']['v0.3'], - } - work_dir = self.case.shared.case_dir_for(self.case.description.name) - self.storage_path = work_dir / 'storage.pickle' - work_dir.mkdir(parents=True, exist_ok=True) - await gvm_io.write_file_bytes(self.storage_path, pickle.dumps(MockStorage())) - - manager = self.case.manager_service.handle - assert manager is not None - self.manager = manager + await self._setup(host_data_tx_id='cross-major-observe') self.phase = 'deploy observability fixtures' for line, address, code in (FIXTURES[name] for name in self.case.fixtures): @@ -113,76 +78,6 @@ async def _run_all(self): self.phase = self.case.method await getattr(self, self.case.method)(*self.case.arguments) - async def _execute( - self, - *, - permissions: str = 'wscn', - **kwargs, - ): - name = typing.cast(str, kwargs['name']) - address = typing.cast(Address, kwargs['address']) - resolve_hook = kwargs.get('resolve_hook') - read_log = kwargs.get('read_log') - host_fuel = kwargs.get('host_fuel') - host = await self._new_host( - name, - address, - resolve_hook, - read_log=read_log, - host_fuel=host_fuel, - ) - ctx = host.ctx - with host as mock_host: - try: - async with base.base_host.ManagerClient(self.manager.uri) as manager_client: - result = await base.base_host.run_genvm( - mock_host, - manager_uri=self.manager.uri, - manager_client=manager_client, - ctx=ctx, - is_sync=kwargs.get('is_sync', True), - leader_public_data=kwargs.get('leader_public_data'), - message=base._message( - address, is_init=typing.cast(bool, kwargs['is_init']) - ), - host_data='{"node_address":"test","tx_id":"cross-major-observe"}', - host='unix://' + mock_host.path, - code=base.resolve_runners( - typing.cast('bytes | None', kwargs.get('code')), - self.case.shared.root_dir, - ), - calldata=typing.cast(bytes, kwargs['calldata']), - timeout=kwargs.get('timeout', 30), - debug_mode='unsafe', - unsafe_overrides=base.base_host.UnsafeOverrides( - reroute_to=self.versions[typing.cast(int, kwargs['line'])], - initial_recursion=kwargs.get('debug_initial_recursion'), - ), - request_extra={ - 'permissions': permissions, - 'no_modules': True, - 'hook_cross_contract_calls': kwargs.get( - 'hook_cross_contract_calls', True - ), - }, - bucket_totals=base.base_host.default_bucket_totals( - typing.cast(int, kwargs['line']) - ), - ) - if ( - kwargs.get('apply_changes', True) - and result.result_kind == host_fns.ResultCode.RETURN - ): - assert mock_host.storage is not None - base._apply_storage_deltas( - mock_host.storage, - address, - result.result_storage_deltas, - ) - return result - finally: - await host.stop_connections() - async def _lvs_extended(self, *, name: str, **kwargs): async def one(suffix: str, **mode): return await self._execute( @@ -248,13 +143,13 @@ async def _assert_validator_mode(self): name='validator-mode-is-real', line=3, address=ADDR_CHAIN_V03, - calldata=base._calldata('hop', 1, ADDR_CHAIN_V02), + calldata=_calldata('hop', 1, ADDR_CHAIN_V02), code=None, is_init=False, resolve_hook=self._route_chain, apply_changes=False, is_sync=False, - leader_public_data=base._leader_public_data(base.gvm_calldata.encode({})), + leader_public_data=_leader_public_data(gvm_calldata.encode({})), ) assert bogus.result_kind == host_fns.ResultCode.VM_ERROR, bogus @@ -269,14 +164,14 @@ def resolve(address, state, major): name='context', line=2, address=ADDR_CALLER_V02, - calldata=base._calldata('context', ADDR_OBSERVER_V03), + calldata=_calldata('context', ADDR_OBSERVER_V03), resolve_hook=resolve, ) expected = { 'contract': ADDR_OBSERVER_V03.as_hex, - 'sender': base.SENDER.as_hex, - 'origin': base.SENDER.as_hex, - 'signer': base.SENDER.as_hex, + 'sender': SENDER.as_hex, + 'origin': SENDER.as_hex, + 'signer': SENDER.as_hex, 'stack': [ADDR_CALLER_V02.as_hex], 'value': 0, 'is_init': False, @@ -292,7 +187,7 @@ async def _assert_permissions(self): name='permissions', line=2, address=ADDR_CALLER_V02, - calldata=base._calldata('permissions', ADDR_OBSERVER_V03, ADDR_HELPER_V03), + calldata=_calldata('permissions', ADDR_OBSERVER_V03, ADDR_HELPER_V03), resolve_hook=self._route_v03, permissions='wscnu', ) @@ -311,7 +206,7 @@ async def _assert_debug_alias(self): name='debug-alias-top-level', line=3, address=ADDR_OBSERVER_V03, - calldata=base._calldata('debug_alias'), + calldata=_calldata('debug_alias'), code=None, is_init=False, resolve_hook=self._route_v03, @@ -331,7 +226,7 @@ async def _assert_debug_alias(self): name='debug-alias-nested', line=2, address=ADDR_CALLER_V02, - calldata=base._calldata('debug_alias', ADDR_OBSERVER_V03), + calldata=_calldata('debug_alias', ADDR_OBSERVER_V03), resolve_hook=self._route_v03, ) for result in (leader, validator, sync): @@ -352,7 +247,7 @@ async def _assert_read_only_storage(self, permissions: str): name=f'storage-{permissions}', line=2, address=ADDR_CALLER_V02, - calldata=base._calldata('read', ADDR_OBSERVER_V03), + calldata=_calldata('read', ADDR_OBSERVER_V03), resolve_hook=self._route_v03, permissions=permissions, read_log=reads, @@ -369,7 +264,7 @@ async def _assert_self_recursion(self, budget: int): name=f'self-recursion-{budget}', line=3, address=ADDR_SELF_V03, - calldata=base._calldata('observe_recursion', 4), + calldata=_calldata('observe_recursion', 4), resolve_hook=self._route_v03, debug_initial_recursion=budget, ) @@ -403,7 +298,7 @@ async def _assert_deep_nesting(self, depth: int): name=f'deep-{depth}', line=3, address=ADDR_CHAIN_V03, - calldata=base._calldata('hop', depth, ADDR_CHAIN_V02), + calldata=_calldata('hop', depth, ADDR_CHAIN_V02), resolve_hook=self._route_chain, ) if depth > CROSS_MAJOR_RECURSION: @@ -426,7 +321,7 @@ async def _assert_error(self, label: str, target: Address): name=f'error-{label}', line=2, address=ADDR_CALLER_V02, - calldata=base._calldata('answer', target), + calldata=_calldata('answer', target), resolve_hook=lambda address, state, major: ( self._route(3) if address == target else self._route_v03(address, state, major) ), @@ -447,7 +342,7 @@ async def _assert_undeployed_error(self): await self._assert_error('undeployed', ABSENT) async def _assert_non_contract_error(self): - await self._assert_error('non-contract', base.SENDER) + await self._assert_error('non-contract', SENDER) async def _assert_trap_error(self): await self._assert_error('trap', ADDR_TRAP_V03) @@ -460,7 +355,7 @@ async def _assert_fuel(self, fuel: int): name=f'fuel-{fuel}', line=2, address=ADDR_CALLER_V02, - calldata=base._calldata('read', ADDR_OBSERVER_V03), + calldata=_calldata('read', ADDR_OBSERVER_V03), resolve_hook=self._route_v03, host_fuel=fuel, ) diff --git a/tests/system/cross-major/test.py b/tests/system/cross-major/test.py index 9e62958..e307323 100644 --- a/tests/system/cross-major/test.py +++ b/tests/system/cross-major/test.py @@ -2,23 +2,22 @@ import asyncio import contextlib -import json -import os -import pickle +import functools import typing from dataclasses import dataclass from pathlib import Path -import genvm_tool.io as gvm_io import genvm_tool.tests import genvm_tool.tests.stage.collection -import genvm_tool_plugins.runners as runners -import origin.base_host as base_host import origin.calldata as gvm_calldata -from gvm_extra.mock_host import MockHost, MockStorage -from origin import host_fns, public_abi +from gvm_extra import cross_major as shared +from gvm_extra.cross_major import TIMESTAMP +from gvm_extra.cross_major import assert_hashes_agree as _assert_hashes_agree +from gvm_extra.cross_major import calldata as _calldata +from gvm_extra.cross_major import leader_public_data as _leader_public_data +from gvm_extra.cross_major import message as _message +from origin import base_host, host_fns, public_abi from origin.calldata import Address -from origin.leader_public_data import LeaderPublicData from origin.process_tree import descendants # One directory per line, because a contract is written against its own sdk. @@ -40,33 +39,15 @@ ADDR_CHAIN_V02 = Address('0x' + '28' * 20) ABSENT_CALLEE = Address('0x' + '99' * 20) -SENDER = Address('0x' + '01' * 20) # Larger than the manager cap the test configures below, so the nested envelope # cannot fit through the socketpair. MANAGER_MAX_MESSAGE_BYTES = 1 * 1024 * 1024 BLOB_BYTES = 2 * 1024 * 1024 -TIMESTAMP = '2024-11-26T06:42:42.424242Z' - -def _leader_public_data(output: bytes) -> bytes: - return LeaderPublicData([output]).encode() - - -def resolve_runners(code: bytes | None, root_dir: Path) -> bytes | None: - """ - Turns the `@RUNNER_LATEST_...@` tokens of a contract source into uids. - - Every comparison against a deployed contract's bytes has to go through this - too, because storage holds what the executor was given. - """ - return None if code is None else runners.substitute(code, root_dir) - - -def _contract_source(line: int, name: str, assets_dir: Path = ASSETS_DIR) -> bytes: - """ - Reads a contract asset written against `line`'s sdk. - """ - return (assets_dir / f'v0.{line}' / f'{name}.py').read_bytes() +_contract_source = functools.partial( + shared.contract_source, + assets_dir=ASSETS_DIR, +) CALLER_V03 = _contract_source(3, 'caller') @@ -165,61 +146,6 @@ def _loop_wat(target: Address, *, caller_line: typing.Literal[2, 3]) -> str: """ -class _TestContext(base_host.Context): - def __init__(self, logger: base_host.Logger): - self.logger = logger - self.stats: dict[str, typing.Any] = {} - - def on_genvm_success(self): ... - def on_genvm_failure(self): ... - - def add_stat(self, key: str, value: typing.Any, /): - self.stats[key] = value - - -def _message(address: Address, *, is_init: bool) -> base_host.Message: - return { - 'contract_address': address, - 'sender_address': SENDER, - 'origin_address': SENDER, - 'signer_address': SENDER, - 'chain_id': 61999, - 'value': 0, - 'is_init': is_init, - 'transaction_timestamp': TIMESTAMP, - } - - -def _calldata(method: str, *args: typing.Any) -> bytes: - return gvm_calldata.encode({'': method, 'args': list(args)}) - - -def _assert_hashes_agree(label: str, runs: list[tuple[str, typing.Any]]) -> None: - """ - A disagreement is only actionable once the failure says who disagreed. - """ - hashes = [(name, run.execution_hash.hex()) for name, run in runs] - assert len({digest for _name, digest in hashes}) == 1, '\n'.join( - [f'{label}: execution hashes disagree'] - + [f' {name}: {digest}' for name, digest in hashes] - ) - - -def _apply_storage_deltas( - storage: MockStorage, - address: Address, - changes: list[tuple[bytes, bytes]], -) -> None: - for key, value in changes: - assert len(key) == 36 - storage.write( - address, - key[:32], - int.from_bytes(key[32:], byteorder='big') * 32, - value, - ) - - async def _wait_for_descendants( pid: int, expected: int, timeout: float = 10 ) -> set[int]: @@ -233,73 +159,31 @@ async def _wait_for_descendants( async def _wait_for_no_descendants(pid: int, timeout: float = 10) -> None: async with asyncio.timeout(timeout): - while descendants(pid): # noqa: ASYNC110 — polls the OS process table, no event to await + while descendants(pid): await asyncio.sleep(0.05) @dataclass -class CrossMajorCase(genvm_tool.tests.test.Case): - description: genvm_tool.tests.test.Description - shared: genvm_tool.tests.SharedContext - manager_service: genvm_tool.tests.stage.collection.Service - method: str - fixtures: tuple[str, ...] - arguments: tuple[typing.Any, ...] = () - +class CrossMajorCase(shared.CrossMajorCase): async def into_steps(self) -> list[genvm_tool.tests.exec.step.Step]: return [CrossMajorStep(self)] -class CrossMajorStep(genvm_tool.tests.exec.step.Python): - def __init__(self, case: CrossMajorCase): - self.case = case - self.phase = 'start' - self.notes: list[typing.Any] = [] - - def to_str(self) -> str: - return '' - - async def run( - self, previous_results: list[typing.Any] - ) -> genvm_tool.tests.test.Result: - try: - await self._run_all() - return genvm_tool.tests.test.Result( - passed=True, context={'notes': self.notes}, elapsed_seconds=0 - ) - except BaseException as exc: - return genvm_tool.tests.test.Result( - passed=False, - context={'phase': self.phase, 'error': exc, 'notes': self.notes}, - elapsed_seconds=0, - ) - +# This case owns its fixtures, addresses and assertions; the host plumbing and +# the call encoding come from `gvm_extra.cross_major`. +class CrossMajorStep(shared.CrossMajorStep): async def _run_all(self): - root = self.case.shared.root_dir - build_info = json.loads((root / 'build' / 'info.json').read_text()) - self.build_dir = Path(build_info['build_dir']) - self.versions = { - 2: build_info['executor_versions']['v0.2'], - 3: build_info['executor_versions']['v0.3'], - } - work_dir = self.case.shared.case_dir_for(self.case.description.name) - self.storage_path = work_dir / 'storage.pickle' - work_dir.mkdir(parents=True, exist_ok=True) - await gvm_io.write_file_bytes(self.storage_path, pickle.dumps(MockStorage())) - - manager = self.case.manager_service.handle - assert manager is not None - self.manager = manager + await self._setup(host_data_tx_id='cross-major') self.phase = 'deploy fixtures' for line, address, code in (FIXTURES[name] for name in self.case.fixtures): await self._deploy(line, address, code) if 'loop-v03' in self.case.fixtures: loop_v03 = await self._compile_wat( - _loop_wat(ADDR_LOOP_V02, caller_line=3), work_dir / 'loop-v03' + _loop_wat(ADDR_LOOP_V02, caller_line=3), self.work_dir / 'loop-v03' ) loop_v02 = await self._compile_wat( - _loop_wat(ADDR_LOOP_V03, caller_line=2), work_dir / 'loop-v02' + _loop_wat(ADDR_LOOP_V03, caller_line=2), self.work_dir / 'loop-v02' ) self._replace_code(ADDR_LOOP_V03, LOOP_V03, loop_v03) self._replace_code(ADDR_LOOP_V02, LOOP_V02, loop_v02) @@ -313,173 +197,6 @@ async def _assert_cancel_stops_chain(self): async def _assert_deadline_stops_chain(self): await self._assert_chain_stopped('deadline') - async def _new_host( - self, - name: str, - running_address: Address, - resolve_hook=None, - read_log: list[tuple[Address, public_abi.StorageView]] | None = None, - host_fuel: int | None = None, - ) -> MockHost: - ctx = _TestContext(self.case.shared.logger) - host_path = ( - Path('/tmp') - / f'gvm-cross-major-{os.getpid()}' - / f'{name[:20]}-{os.urandom(4).hex()}.sock' - ) - host_path.parent.mkdir(parents=True, exist_ok=True) - host = MockHost( - ctx=ctx, - path=str(host_path), - storage_path_pre=self.storage_path, - storage_path_post=self.storage_path, - balances={}, - running_address=running_address, - resolve_call_contract_executor_hook=resolve_hook, - ) - if read_log is not None: - original = host.storage_read - - async def logged(mode, address, slot, offset, le, /): - read_log.append((Address(address), mode)) - return await original(mode, address, slot, offset, le) - - host.storage_read = logged # type: ignore[method-assign] - if host_fuel is not None: - - async def get_remaining_time_fee_gen_wei() -> int: - return host_fuel - - host.get_remaining_time_fee_gen_wei = ( # type: ignore[method-assign] - get_remaining_time_fee_gen_wei - ) - return host - - async def _execute( - self, - *, - name: str, - line: int, - address: Address, - calldata: bytes, - code: bytes | None, - is_init: bool, - resolve_hook=None, - timeout: float = 30, - debug_initial_recursion: int | None = None, - is_sync: bool = True, - leader_public_data: bytes | None = None, - apply_changes: bool = True, - read_log: list[tuple[Address, public_abi.StorageView]] | None = None, - host_fuel: int | None = None, - hook_cross_contract_calls: bool = True, - ): - host = await self._new_host( - name, address, resolve_hook, read_log=read_log, host_fuel=host_fuel - ) - ctx = host.ctx - with host as mock_host: - try: - async with base_host.ManagerClient(self.manager.uri) as manager_client: - result = await base_host.run_genvm( - mock_host, - manager_uri=self.manager.uri, - manager_client=manager_client, - ctx=ctx, - is_sync=is_sync, - leader_public_data=leader_public_data, - message=_message(address, is_init=is_init), - host_data='{"node_address":"test","tx_id":"cross-major"}', - host='unix://' + mock_host.path, - code=resolve_runners(code, self.case.shared.root_dir), - calldata=calldata, - timeout=timeout, - debug_mode='unsafe', - unsafe_overrides=base_host.UnsafeOverrides( - reroute_to=self.versions[line], - initial_recursion=debug_initial_recursion, - ), - request_extra={ - 'no_modules': True, - 'hook_cross_contract_calls': hook_cross_contract_calls, - }, - bucket_totals=base_host.default_bucket_totals(line), - ) - if apply_changes and result.result_kind == host_fns.ResultCode.RETURN: - assert mock_host.storage is not None - _apply_storage_deltas( - mock_host.storage, - address, - result.result_storage_deltas, - ) - return result - finally: - await host.stop_connections() - - async def _deploy(self, line: int, address: Address, code: bytes): - result = await self._execute( - name=f'deploy-{line}-{address.as_bytes.hex()}', - line=line, - address=address, - calldata=gvm_calldata.encode({}), - code=code, - is_init=True, - timeout=10 * 60, - ) - assert result.result_kind == host_fns.ResultCode.RETURN, result - assert len(result.execution_hash) == 32, result.execution_hash - - async def _compile_wat(self, wat: str, path: Path) -> bytes: - await gvm_io.make_dir(path) - wat_path = path / 'contract.wat' - wasm_path = path / 'contract.wasm' - await gvm_io.write_file_text(wat_path, wat) - process = await asyncio.create_subprocess_exec( - 'wat2wasm', - '--enable-annotations', - '-o', - str(wasm_path), - str(wat_path), - stdout=asyncio.subprocess.PIPE, - stderr=asyncio.subprocess.PIPE, - ) - stdout, stderr = await process.communicate() - assert process.returncode == 0, (stdout, stderr) - return await gvm_io.read_file_bytes(wasm_path) - - def _replace_code(self, address: Address, old_code: bytes, new_code: bytes) -> None: - root_dir = self.case.shared.root_dir - old_code = typing.cast(bytes, resolve_runners(old_code, root_dir)) - new_code = typing.cast(bytes, resolve_runners(new_code, root_dir)) - with open(self.storage_path, 'rb') as storage_file: - storage: MockStorage = pickle.load(storage_file) - code_slots = [ - slot - for slot in storage.slots(address) - if int.from_bytes(storage.read(address, slot, 0, 4), byteorder='little') - == len(old_code) - and storage.read(address, slot, 4, len(old_code)) == old_code - ] - assert len(code_slots) == 1, code_slots - storage.write( - address, - code_slots[0], - 0, - len(new_code).to_bytes(4, byteorder='little') + new_code, - ) - with open(self.storage_path, 'wb') as storage_file: - pickle.dump(storage, storage_file) - - def _route(self, line: int) -> bytes: - """ - Routing payload placing a callee on `line`'s executor. - - It names the directory rather than the major, because a major cannot - pick between the live lines: every manifest key is semver major 0, so - `{'kind': 'major'}` resolves to the newest line whatever it asks for. - """ - return gvm_calldata.encode({'kind': 'version', 'version': self.versions[line]}) - def _raw_request( self, *, @@ -599,36 +316,6 @@ async def _assert_nested_deadline_is_timeout(self): assert result.result_kind == host_fns.ResultCode.VM_ERROR, result assert result.result_data == str(public_abi.VmError.timeout()), result - async def _lvs( - self, *, name: str, line: int, address: Address, calldata: bytes, resolve_hook - ): - """ - Run the same step as leader, validator and sync run. - - The three must agree on the execution hash; anything else is a - determinism violation an honest validator would see as a disagreement. - """ - - async def one(suffix: str, **kwargs): - return await self._execute( - name=f'{name}-{suffix}', - line=line, - address=address, - calldata=calldata, - code=None, - is_init=False, - resolve_hook=resolve_hook, - apply_changes=False, - **kwargs, - ) - - leader = await one('leader', is_sync=False) - validator = await one( - 'validator', is_sync=False, leader_public_data=leader.result_leader_public_data - ) - sync = await one('sync', is_sync=True) - return leader, validator, sync - async def _assert_lvs_validator_mode(self): # A leader/validator run that silently degraded to a sync one would make # the agreement below vacuous, so first pin that the validator mode is