From e79c6b810dd971c9f3b823205f2fdcfecead3652 Mon Sep 17 00:00:00 2001 From: highlander Date: Wed, 26 Aug 2026 17:18:39 -0600 Subject: [PATCH 1/2] test(bitcoin-only): gate unsupported 7.15 handlers --- tests/test_msg_bip85.py | 1 + tests/test_msg_mayachain_signtx.py | 1 + 2 files changed, 2 insertions(+) diff --git a/tests/test_msg_bip85.py b/tests/test_msg_bip85.py index 4a0b2b89..1020d280 100644 --- a/tests/test_msg_bip85.py +++ b/tests/test_msg_bip85.py @@ -20,6 +20,7 @@ class TestMsgBip85(common.KeepKeyTest): def setUp(self): super().setUp() self.requires_firmware("7.15.0") + self.requires_fullFeature() def test_bip85_12word_flow(self): """12-word derivation: verify device goes through display flow and returns Success.""" diff --git a/tests/test_msg_mayachain_signtx.py b/tests/test_msg_mayachain_signtx.py index c774ede2..3d8952fc 100644 --- a/tests/test_msg_mayachain_signtx.py +++ b/tests/test_msg_mayachain_signtx.py @@ -264,6 +264,7 @@ def test_mayachain_sign_tx_memos(self): signs, and each signature is bound to its exact memo bytes — a memo substitution changes the sign-doc digest and fails verification.""" self.requires_firmware("7.9.1") + self.requires_fullFeature() self.setup_mnemonic_nopin_nopassphrase() memos = [ From c697a25115ea859ab5b0a89f77dd2c77e61ab889 Mon Sep 17 00:00:00 2001 From: highlander Date: Wed, 26 Aug 2026 17:49:31 -0600 Subject: [PATCH 2/2] test(report): respect Bitcoin-only feature boundaries --- scripts/generate-test-report.py | 21 ++++++++++-- tests/test_report_variant_validation.py | 45 +++++++++++++++++++++++++ 2 files changed, 63 insertions(+), 3 deletions(-) create mode 100644 tests/test_report_variant_validation.py diff --git a/scripts/generate-test-report.py b/scripts/generate-test-report.py index 5469ac3b..bd4ddf85 100644 --- a/scripts/generate-test-report.py +++ b/scripts/generate-test-report.py @@ -3243,6 +3243,13 @@ def screenshot_filter(fw_version): 'test_msg_solana_lut_attestation': '7.15.0', } +# These modules are mandatory only on the multi-chain product. Their handlers +# are intentionally absent from KK_BITCOIN_ONLY, so a capability-gated skip is +# evidence of the product boundary there, not missing release coverage. +FULL_FEATURE_ONLY_MUST_RUN_MODULES = { + 'test_msg_solana_lut_attestation', +} + def screenshot_audit(fw_version, screenshot_root, junit_path=None): """Which SECTIONS tests DECLARED screens but captured none? @@ -3283,7 +3290,7 @@ def screenshot_audit(fw_version, screenshot_root, junit_path=None): return (len(missing) == 0, missing) -def validate_junit(fw_version, results): +def validate_junit(fw_version, results, variant='full'): """Check SECTIONS tests against JUnit results. Returns (passed, failed_list). A test is considered failed if it appears in SECTIONS for this firmware version @@ -3299,7 +3306,12 @@ def validate_junit(fw_version, results): status = _lookup(results, mod, meth) if status in ('fail', 'error'): failures.append((tid, mod, meth, status)) - elif status == 'skip' and ver_ge(fw_version, MUST_RUN_MODULES.get(mod, '99.0.0')): + must_run = not ( + variant == 'bitcoin-only' and + mod in FULL_FEATURE_ONLY_MUST_RUN_MODULES + ) + if (status == 'skip' and must_run and + ver_ge(fw_version, MUST_RUN_MODULES.get(mod, '99.0.0'))): failures.append((tid, mod, meth, 'skipped-but-required')) elif not status: failures.append((tid, mod, meth, 'missing')) @@ -3320,6 +3332,9 @@ def main(): help='Print pytest -k expression for tests needing screenshots, then exit') p.add_argument('--validate-junit', action='store_true', help='Validate JUnit results against SECTIONS, exit non-zero on failures') + p.add_argument('--variant', choices=('full', 'bitcoin-only'), + default=os.environ.get('KK_FIRMWARE_VARIANT', 'full'), + help='Product variant whose required report coverage is validated') args = p.parse_args() fw = args.fw_version @@ -3347,7 +3362,7 @@ def main(): print('ERROR: --validate-junit requires --junit=', file=sys.stderr) sys.exit(2) results = parse_junit(args.junit) - ok, failures = validate_junit(fw, results) + ok, failures = validate_junit(fw, results, args.variant) if ok: print(f'SECTIONS validation passed: all tests for fw {fw} are pass or skip') sys.exit(0) diff --git a/tests/test_report_variant_validation.py b/tests/test_report_variant_validation.py new file mode 100644 index 00000000..73b25c08 --- /dev/null +++ b/tests/test_report_variant_validation.py @@ -0,0 +1,45 @@ +import importlib.util +import os +import unittest + + +REPORT_SCRIPT = os.path.join( + os.path.dirname(__file__), '..', 'scripts', 'generate-test-report.py') +SPEC = importlib.util.spec_from_file_location('generate_test_report', + REPORT_SCRIPT) +REPORT = importlib.util.module_from_spec(SPEC) +SPEC.loader.exec_module(REPORT) + + +def catalog_results_with_solana_lut_skipped(): + results = {} + for _, _, min_fw, _, _, tests in REPORT.SECTIONS: + if not REPORT.ver_ge('7.15.0', min_fw): + continue + for _, module, method, _, _, _ in tests: + results['%s::%s' % (module, method)] = 'pass' + for key in list(results): + if key.startswith('test_msg_solana_lut_attestation::'): + results[key] = 'skip' + return results + + +class TestReportVariantValidation(unittest.TestCase): + + def test_full_product_requires_solana_lut_coverage(self): + ok, failures = REPORT.validate_junit( + '7.15.0', catalog_results_with_solana_lut_skipped(), 'full') + self.assertFalse(ok) + self.assertEqual(4, len(failures)) + self.assertTrue(all(item[3] == 'skipped-but-required' + for item in failures)) + + def test_bitcoin_only_accepts_absent_solana_lut_handlers(self): + result = REPORT.validate_junit( + '7.15.0', catalog_results_with_solana_lut_skipped(), + 'bitcoin-only') + self.assertEqual((True, []), result) + + +if __name__ == '__main__': + unittest.main()