From c24ee9c30f14f36a70ee1fa988701d2a97680d04 Mon Sep 17 00:00:00 2001 From: Sjors Provoost Date: Mon, 24 Aug 2026 17:32:20 +0200 Subject: [PATCH] test: allow selecting device tests --- test/README.md | 7 ++++ test/run_tests.py | 26 ++++++++++++++- test/test_device.py | 6 ++-- test/test_keepkey.py | 5 ++- test/test_selector.py | 75 +++++++++++++++++++++++++++++++++++++++++++ test/test_trezor.py | 5 ++- 6 files changed, 114 insertions(+), 10 deletions(-) create mode 100644 test/test_selector.py diff --git a/test/README.md b/test/README.md index 099f9d027..6e9053afc 100644 --- a/test/README.md +++ b/test/README.md @@ -34,6 +34,13 @@ If an environment variable is not present or not set, then the simulator/emulato Otherwise the paths to those will need to be specified on the command line. `test_trezor.py`, `test_coldcard.py`, `test_keepkey.py`, `test_jade.py`, `test_bitbox02.py` and `test/test_digitalbitbox.py` can be disabled. +To run specific device tests, repeat `--test` with a method name. Using `--test` implies `--device-only`: + +``` +./run_tests.py --ledger --test test_signtx +./run_tests.py --ledger --test test_signtx --test test_big_tx +``` + If you are building the Trezor emulator, the Coldcard simulator, the Keepkey emulator, the Jade emulator, the Digital Bitbox simulator, and `bitcoind` without `setup_environment.sh`, then you will need to make `work/` inside of `test/`. ``` diff --git a/test/run_tests.py b/test/run_tests.py index 28d8f7a95..a2b3a775e 100755 --- a/test/run_tests.py +++ b/test/run_tests.py @@ -11,6 +11,11 @@ from test_descriptor import TestDescriptor from test_device import Bitcoind from test_psbt import TestPSBT +from test_selector import ( + TestTestSelector, + get_unmatched_test_selectors, + set_test_selectors, +) from test_trezor import trezor_test_suite from test_ledger import ledger_test_suite from test_digitalbitbox import digitalbitbox_test_suite @@ -73,12 +78,20 @@ parser.add_argument('--all', help='Run tests on all existing simulators', default=False, action='store_true') parser.add_argument('--bitcoind', help='Path to bitcoind', default='work/bitcoin/build/bin/bitcoind') parser.add_argument('--interface', help='Which interface to send commands over', choices=['library', 'cli', 'bindist', 'stdin'], default='library') +parser.add_argument( + '--test', + action='append', + metavar='METHOD', + help='Run only this device test method; may be specified more than once', +) parser.add_argument("--device-only", help="Only run device tests", action="store_true") parser.set_defaults(trezor_1=None, trezor_t=None, coldcard=None, coldcard_edge=None, keepkey=None, bitbox01=None, ledger=None, ledger_legacy=None, jade=None, bitbox02=None) args = parser.parse_args() +if args.test: + args.device_only = True # Run tests success = True @@ -90,6 +103,7 @@ suite.addTests(unittest.defaultTestLoader.loadTestsFromTestCase(TestBase58)) suite.addTests(unittest.defaultTestLoader.loadTestsFromTestCase(TestBIP32)) suite.addTests(unittest.defaultTestLoader.loadTestsFromTestCase(TestColdcardFirmware)) + suite.addTests(unittest.defaultTestLoader.loadTestsFromTestCase(TestTestSelector)) if sys.platform.startswith("linux"): suite.addTests(unittest.defaultTestLoader.loadTestsFromTestCase(TestUdevRulesInstaller)) success = unittest.TextTestRunner(stream=sys.stdout, verbosity=2).run(suite).wasSuccessful() @@ -119,7 +133,12 @@ args.jade = False if args.jade is None else args.jade args.bitbox02 = False if args.bitbox02 is None else args.bitbox02 -if args.trezor_1 or args.trezor_t or args.coldcard or args.coldcard_edge or args.ledger or args.ledger_legacy or args.keepkey or args.bitbox01 or args.jade or args.bitbox02: +run_device_tests = args.trezor_1 or args.trezor_t or args.coldcard or args.coldcard_edge or args.ledger or args.ledger_legacy or args.keepkey or args.bitbox01 or args.jade or args.bitbox02 +if args.test and not run_device_tests: + parser.error("--test requires at least one device") + +set_test_selectors(args.test) +if run_device_tests: # Start bitcoind bitcoind = Bitcoind.create(args.bitcoind) @@ -144,4 +163,9 @@ if success and args.bitbox02: success &= bitbox02_test_suite(args.bitbox02_path, bitcoind, args.interface) + unmatched_tests = get_unmatched_test_selectors() + if unmatched_tests: + print("No device tests matched: {}".format(", ".join(unmatched_tests)), file=sys.stderr) + success = False + sys.exit(not success) diff --git a/test/test_device.py b/test/test_device.py index 421054dbd..cde554ae2 100644 --- a/test/test_device.py +++ b/test/test_device.py @@ -13,6 +13,8 @@ from typing import Dict +from test_selector import get_test_case_names + from authproxy import AuthServiceProxy, JSONRPCException from hwilib import _bech32 as bech32 from hwilib._base58 import xpub_to_pub_hex, to_address, decode @@ -153,10 +155,8 @@ def __init__(self, bitcoind, emulator: DeviceEmulator = None, interface='library @staticmethod def parameterize(testclass, bitcoind, emulator, interface='library', *args, **kwargs): - testloader = unittest.TestLoader() - testnames = testloader.getTestCaseNames(testclass) suite = unittest.TestSuite() - for name in testnames: + for name in get_test_case_names(testclass): suite.addTest(testclass(bitcoind, emulator, interface, name, *args, **kwargs)) return suite diff --git a/test/test_keepkey.py b/test/test_keepkey.py index ff9cf84c0..fdec81287 100755 --- a/test/test_keepkey.py +++ b/test/test_keepkey.py @@ -27,6 +27,7 @@ TestSignMessage, TestSignTx, ) +from test_selector import get_test_case_names from hwilib._cli import process_commands from hwilib.devices.keepkey import ( @@ -136,10 +137,8 @@ def __init__(self, emulator, interface='library', methodName='runTest'): @staticmethod def parameterize(testclass, emulator, interface='library'): - testloader = unittest.TestLoader() - testnames = testloader.getTestCaseNames(testclass) suite = unittest.TestSuite() - for name in testnames: + for name in get_test_case_names(testclass): suite.addTest(testclass(emulator, interface, name)) return suite diff --git a/test/test_selector.py b/test/test_selector.py new file mode 100644 index 000000000..f77519e36 --- /dev/null +++ b/test/test_selector.py @@ -0,0 +1,75 @@ +#! /usr/bin/env python3 + +import unittest + +from typing import List, Optional, Set, Type + + +_test_selectors: Optional[List[str]] = None +_matched_selectors: Set[str] = set() + + +def set_test_selectors(selectors: Optional[List[str]]) -> None: + """Select device tests by method name.""" + global _test_selectors, _matched_selectors + _test_selectors = selectors or None + _matched_selectors = set() + + +def get_test_case_names(testclass: Type[unittest.TestCase]) -> List[str]: + """Return the selected test methods from ``testclass``.""" + testnames = list(unittest.TestLoader().getTestCaseNames(testclass)) + if _test_selectors is None: + return testnames + + selected = [] + for testname in testnames: + if testname in _test_selectors: + _matched_selectors.add(testname) + selected.append(testname) + return selected + + +def get_unmatched_test_selectors() -> List[str]: + """Return selectors that did not match any constructed device suite.""" + if _test_selectors is None: + return [] + return [selector for selector in _test_selectors if selector not in _matched_selectors] + + +class TestTestSelector(unittest.TestCase): + class FirstTests(unittest.TestCase): + def test_alpha(self) -> None: + pass + + def test_shared(self) -> None: + pass + + class SecondTests(unittest.TestCase): + def test_beta(self) -> None: + pass + + def test_shared(self) -> None: + pass + + def tearDown(self) -> None: + set_test_selectors(None) + + def test_no_selectors(self) -> None: + set_test_selectors(None) + self.assertEqual( + get_test_case_names(self.FirstTests), + ["test_alpha", "test_shared"], + ) + + def test_method_selector(self) -> None: + set_test_selectors(["test_shared"]) + self.assertEqual(get_test_case_names(self.FirstTests), ["test_shared"]) + self.assertEqual(get_test_case_names(self.SecondTests), ["test_shared"]) + self.assertEqual(get_unmatched_test_selectors(), []) + + def test_selector_list_and_unmatched(self) -> None: + set_test_selectors(["test_alpha", "test_beta", "test_missing"]) + self.assertEqual(get_test_case_names(self.FirstTests), ["test_alpha"]) + self.assertEqual(get_test_case_names(self.SecondTests), ["test_beta"]) + self.assertEqual(get_unmatched_test_selectors(), ["test_missing"]) diff --git a/test/test_trezor.py b/test/test_trezor.py index b61c1f00a..0438ccb6f 100755 --- a/test/test_trezor.py +++ b/test/test_trezor.py @@ -26,6 +26,7 @@ TestSignMessage, TestSignTx, ) +from test_selector import get_test_case_names from hwilib._cli import process_commands from hwilib.devices.trezor import TrezorClient @@ -130,10 +131,8 @@ def __init__(self, emulator, interface='library', methodName='runTest'): @staticmethod def parameterize(testclass, emulator, interface='library'): - testloader = unittest.TestLoader() - testnames = testloader.getTestCaseNames(testclass) suite = unittest.TestSuite() - for name in testnames: + for name in get_test_case_names(testclass): suite.addTest(testclass(emulator, interface, name)) return suite