From 1bc34890cb8618b2fffd446a63924974e0fe15b5 Mon Sep 17 00:00:00 2001 From: antuz123 <43998049+antuz123@users.noreply.github.com> Date: Fri, 23 Oct 2026 03:02:59 +0530 Subject: [PATCH] Retain bounded native startup and onboarding failure facts --- .github/native/hosted_native.py | 72 +++- .github/native/product_failure_diagnosis.py | 264 +++++++++++++++ .github/native/product_measurements.py | 25 +- .github/native/startup_diagnosis.py | 13 +- .github/tests/test_diagnostic_privacy.py | 11 +- .github/tests/test_hosted_native.py | 30 ++ .github/tests/test_ios_startup_diagnostics.py | 187 +++++++++- .../tests/test_product_failure_diagnosis.py | 320 ++++++++++++++++++ .github/tests/test_startup_diagnosis.py | 49 ++- docs/product-failure-observation.md | 84 +++++ docs/r5-ios-startup-observation.md | 48 ++- 11 files changed, 1077 insertions(+), 26 deletions(-) create mode 100644 .github/native/product_failure_diagnosis.py create mode 100644 .github/tests/test_product_failure_diagnosis.py create mode 100644 docs/product-failure-observation.md diff --git a/.github/native/hosted_native.py b/.github/native/hosted_native.py index 593eea0db..1bed83718 100644 --- a/.github/native/hosted_native.py +++ b/.github/native/hosted_native.py @@ -7,6 +7,7 @@ import hashlib import importlib.util +import io import json import math import os @@ -22,6 +23,7 @@ from types import SimpleNamespace import urllib.request import xml.etree.ElementTree as ET +from datetime import datetime ROOT = Path(__file__).resolve().parents[2] @@ -31,6 +33,10 @@ MAESTRO_VERSION = 'cli-2.11.0' MAESTRO_SHA256 = '5384593cb4e7a106489e75a821d157dd43f4e438df6bc308b72e82c685e1283a' APP_IDS = {'IOS': 'io.hexawallet.hexakeeper.dev', 'ANDROID': 'io.hexawallet.keeper.development'} +IOS_LOG_MAX_INPUT_BYTES = 1024 * 1024 +IOS_LOG_MAX_RECORD_BYTES = 16 * 1024 +IOS_LOG_MAX_SCAN_LINES = 4096 +IOS_LOG_MAX_OWN_RECORDS = 4096 def save(data): @@ -167,13 +173,20 @@ def private_log_prefix(path): def ios_startup_diagnostics(payload, executable): - if not re.fullmatch(r'[A-Za-z0-9_][A-Za-z0-9_.-]*', executable): + if (not isinstance(executable, str) or + not re.fullmatch(r'[A-Za-z0-9_][A-Za-z0-9_.-]{0,127}', executable)): raise ValueError('Unsafe iOS executable selector.') + if not isinstance(payload, str): + raise TypeError('Expected compact log text.') # log(1) compact: ISO timestamp, abbreviated type, process[PID:TID], message. - # Only prefixed own-process records qualify; never retain continuation lines. - pattern = re.compile(r'^\d{4}-\d{2}-\d{2}[ T]\d{2}:\d{2}:\d{2}(?:\.\d+)?' - r'(?:[+-]\d{2}:?\d{2}|Z)?\s+(\S+)\s+' - r'([A-Za-z0-9_.-]+)\[\d+:[A-Fa-f0-9x]+\]\s+(.*)$') + # Framing/bounds adapted from #7115/eceb; retain this source's categories. + # Rendered text is not authenticated provenance. Never attach continuations. + pattern = re.compile(r'(?P\d{4}-\d{2}-\d{2}[ T]\d{2}:\d{2}:\d{2}(?:\.\d{1,9})?' + r'(?:[+-]\d{2}:?\d{2}|Z)?)\s+(?P[A-Za-z]+)\s+' + r'(?P[A-Za-z0-9_.-]{1,128})\[(?P[1-9]\d{0,9}):' + r'(?:0x)?[A-Fa-f0-9]{1,16}\]\s+(?P.*)') + severities = {'a', 'd', 'db', 'df', 'i', 'info', 'default', 'debug', + 'e', 'er', 'error', 'f', 'ft', 'fault'} patterns = { 'JS_TYPE': r'\bTypeError\b', 'JS_REFERENCE': r'\bReferenceError\b', 'JS_SYNTAX': r'\bSyntaxError\b', 'JS_RANGE': r'\bRangeError\b', @@ -187,16 +200,43 @@ def ios_startup_diagnostics(payload, executable): 'ATS_BLOCKED': r'App Transport Security.*(?:blocked|requires)|NSURLErrorDomain.*-1022\b', 'CONNECTION': r'ECONNREFUSED|Connection refused|NSURLErrorDomain.*-(?:1001|1003|1004|1009)\b|The Internet connection appears to be offline', } + encoded = payload[:IOS_LOG_MAX_INPUT_BYTES + 1].encode('utf-8', errors='replace') + truncated = len(payload) > IOS_LOG_MAX_INPUT_BYTES or len(encoded) > IOS_LOG_MAX_INPUT_BYTES + prefix = encoded[:IOS_LOG_MAX_INPUT_BYTES].decode('utf-8', errors='replace') + if truncated: + prefix = prefix.rsplit('\n', 1)[0] + '\n' if '\n' in prefix else '' selected, counts = [], {name: 0 for name in patterns} matched, errors, classified = 0, 0, 0 - for line in payload.splitlines(): - match = pattern.match(line) + malformed, oversized, scan_limit, record_limit = 0, 0, False, False + replacement = '\ufffd' in prefix + for index, raw in enumerate(io.StringIO(prefix)): + if index >= IOS_LOG_MAX_SCAN_LINES: + scan_limit = True + break + if matched >= IOS_LOG_MAX_OWN_RECORDS: + record_limit = True + break + line = raw.removesuffix('\n').removesuffix('\r') + if len(line) > IOS_LOG_MAX_RECORD_BYTES or len(line.encode('utf-8')) > IOS_LOG_MAX_RECORD_BYTES: + oversized += 1 + continue + match = pattern.fullmatch(line) if not match: + malformed += bool(re.match(r'^\d{4}-', line)) + continue + try: + datetime.fromisoformat(match['timestamp'].replace('Z', '+00:00')) + valid = (match['severity'].lower() in severities and int(match['pid']) <= 2147483647 + and not any(ord(char) < 32 and char != '\t' for char in line)) + except ValueError: + valid = False + if not valid: + malformed += 1 continue - severity, process, message = match.groups() - if process != executable: + if match['process'] != executable: continue matched += 1 + severity, message = match['severity'], match['message'] found = [name for name, regex in patterns.items() if re.search(regex, message, re.IGNORECASE)] if found or severity.lower() in ['e', 'f', 'er', 'ft', 'error', 'fault'] or re.search( r'(?:\b(?:error|exception|fatal)\b|\b[\w.$]*(?:Exception|Error)\b)', @@ -207,8 +247,14 @@ def ios_startup_diagnostics(payload, executable): classified += bool(found) for name in found: counts[name] += 1 + partial = any([truncated, malformed, oversized, replacement, scan_limit, record_limit]) return {'format': 'compact', 'matching_process_records': matched, 'errors': selected, 'observation_state': 'OWN_RECORDS_OBSERVED' if matched else 'UNKNOWN', + 'parse_status': 'UNKNOWN' if not matched else 'PARTIAL' if partial else 'PARSED', + 'cause': 'UNDETERMINED', 'input_truncated': truncated, + 'malformed_record_lines': malformed, 'oversized_record_lines': oversized, + 'replacement_character_observed': replacement, + 'scan_limit_reached': scan_limit, 'record_limit_reached': record_limit, 'error_records': errors, 'classified_error_records': classified, 'unclassified_error_records': errors - classified, 'categories': [name for name, count in counts.items() if count], @@ -428,12 +474,16 @@ def ios_details(): code = result.returncode except subprocess.TimeoutExpired: code = 124 - result = {'command': args, 'exit_code': code, 'errors': [], 'observation_state': 'UNKNOWN'} + result = {'command': args, 'exit_code': code, 'errors': [], 'observation_state': 'UNKNOWN', + 'query_status': 'SUCCESS' if code == 0 else 'TIMEOUT' if code == 124 else 'FAILED', + 'parse_status': 'UNKNOWN', 'cause': 'UNDETERMINED', 'input_truncated': None} if code == 0: try: payload, truncated = private_log_prefix(path) result.update(ios_startup_diagnostics(payload, executable)) - result['input_truncated'] = truncated + result['input_truncated'] |= truncated + if truncated: + result['parse_status'] = 'PARTIAL' if result['matching_process_records'] else 'UNKNOWN' except (ValueError, TypeError) as error: result['parse_error'] = type(error).__name__ return result diff --git a/.github/native/product_failure_diagnosis.py b/.github/native/product_failure_diagnosis.py new file mode 100644 index 000000000..1cda3f244 --- /dev/null +++ b/.github/native/product_failure_diagnosis.py @@ -0,0 +1,264 @@ +"""Read existing failed-sample files; export fixed observations, never raw text.""" +from itertools import islice +import json +import os +from pathlib import Path +import re +import stat +import unicodedata +import xml.etree.ElementTree as ET + +FILE_LIMIT = 1024 * 1024 +METRO_LIMIT = 256 * 1024 +SNAPSHOT_LIMIT = 8 +NODE_LIMIT = 10000 +FLOWS = ('product_first_wallet.yaml', 'product_retry.yaml') +IDS = {'btn_skip': 'onboarding_skip', 'view_startNewTile': 'start_new_tile', + 'view_recoverTile': 'recover_tile', 'btn_AppSettingsIcon': 'app_settings'} +TEXT = {'Start New': 'start_new_text', 'Welcome to Bitcoin Keeper': 'welcome', + 'Create a passcode': 'create_passcode', 'Confirm your passcode': 'confirm_passcode', + 'Enter your passcode': 'login', 'Welcome Back': 'welcome_back', + 'Unable to check your passcode. Please try again.': 'credential_error', 'Retry': 'retry', + 'Dismiss': 'dismiss_control', 'Minimize': 'minimize_control', + 'Allow': 'allow_control', "Don't Allow": 'dont_allow_control', + 'Don’t Allow': 'dont_allow_control'} +LOGBOX = ('Console Error', 'Uncaught Error', 'Syntax Error', 'Render Error', 'Failed to compile') +FLAGS = tuple(dict.fromkeys((*IDS.values(), *TEXT.values(), 'logbox_error_header', + 'logbox_warning_header', 'logbox_counter', 'logbox_controls', 'logbox_syntax_footer', + 'notification_prompt_phrase', 'notification_permission_controls'))) +SELECTORS = {**{f'id: {key}': value for key, value in IDS.items()}, + **{f'"{key}"': value for key, value in TEXT.items() + if value in ('create_passcode', 'confirm_passcode', 'login', 'retry')}} +FAULTS = {'JS_TYPE_ERROR': r'\bTypeError\b', 'JS_REFERENCE_ERROR': r'\bReferenceError\b', + 'JS_SYNTAX_ERROR': r'\bSyntaxError\b', 'MODULE_RESOLUTION_ERROR': r'Unable to resolve module|\bUnableToResolveError\b', + 'TRANSFORM_ERROR': r'\bTransformError\b', 'APP_NOT_REGISTERED': r'has not been registered', + 'NATIVE_MODULE_UNAVAILABLE': r'TurboModuleRegistry\.getEnforcing|requireNativeComponent|native module cannot be null', + 'MEMORY_EXHAUSTION': r'out of memory|heap limit|heap out of memory', + 'TRANSPORT_ERROR': r'\b(?:EADDRINUSE|ECONNREFUSED|ECONNRESET)\b', + 'FILESYSTEM_ERROR': r'\b(?:ENOENT|EACCES|ENOSPC)\b'} + + +def regular(path, root): + root, path = Path(root), Path(path) + if root.is_symlink() or not root.is_dir() or not path.resolve().is_relative_to(root.resolve()): + raise ValueError() + for part in (path, *path.parents): + if part.is_symlink(): + raise ValueError() + if part == root: + break + info = path.stat() + if not stat.S_ISREG(info.st_mode): + raise ValueError() + return info + + +def read(path, root): + regular(path, root) + with path.open('rb') as source: + payload = source.read(FILE_LIMIT + 1) + if len(payload) > FILE_LIMIT: + raise ValueError() + return payload + + +def unique_object(pairs): + result = {} + for key, value in pairs: + if key in result: + raise ValueError() + result[key] = value + return result + + +def junit(directory, flow): + result = {'state': 'UNKNOWN', 'failed_selector': None, 'failure_kind': None} + try: + if flow not in FLOWS: + return result + payload = read(directory / 'junit.xml', directory).decode('utf-8-sig') + if ' SNAPSHOT_LIMIT + for path in sorted(paths[:SNAPSHOT_LIMIT]): + try: + pending = [(json.loads(read(path, directory), object_pairs_hook=unique_object), 0)] + count, found = 0, dict.fromkeys(FLAGS, False) + while pending: + node, depth = pending.pop() + count += 1 + if count > NODE_LIMIT or depth > 128 or not isinstance(node, dict): + raise ValueError() + attrs, children = node.get('attributes', {}), node.get('children', []) + if (not isinstance(attrs, dict) or not isinstance(children, list) or + any(not isinstance(key, str) or not isinstance(value, str) for key, value in attrs.items())): + raise ValueError() + resource = attrs.get('resource-id') + if isinstance(resource, str): + resource = resource.removeprefix('io.hexawallet.keeper.development:id/') + if resource in IDS: + found[IDS[resource]] = True + for key in ('text', 'accessibilityText', 'title', 'value'): + value = attrs.get(key) + if not isinstance(value, str): + continue + value = value.strip() + if value in TEXT: + found[TEXT[value]] = True + found['logbox_error_header'] |= value in LOGBOX + found['logbox_warning_header'] |= value == 'Console Warning' + found['logbox_counter'] |= bool(re.fullmatch(r'Log [1-9]\d* of [1-9]\d*', value)) + found['logbox_syntax_footer'] |= value == 'This error cannot be dismissed.' + found['notification_prompt_phrase'] |= 'would like to send you notifications' in value.lower() + pending.extend((child, depth + 1) for child in children) + found['logbox_controls'] = found['dismiss_control'] and found['minimize_control'] + found['notification_permission_controls'] = (found['notification_prompt_phrase'] and + found['allow_control'] and found['dont_allow_control']) + result['snapshot_count'] += 1 + result['node_count'] += count + observed = {key: observed[key] or found[key] for key in FLAGS} + except (OSError, ValueError, TypeError, RecursionError): + result['rejected_snapshots'] += 1 + if result['snapshot_count']: + complete = not result['rejected_snapshots'] and not result['snapshot_limit_exceeded'] + result['state'] = 'COMPLETE' if complete else 'PARTIAL' + result['flags'] = {key: value if value or complete else None for key, value in observed.items()} + except OSError: + pass + return result + + +class MetroBinding: + def __init__(self, metro): + count = len(metro.events) + if (metro.phase != 'product' or not count or metro.events[-1].get('phase') != 'product' or + not any(child is metro.process for child in metro.owned)): + raise ValueError() + self.process, self.count, self.root = metro.process, count, Path(metro.private) + self.path = self.root / ('metro-%02d.log' % count) + regular(self.path, self.root) + with self.path.open('rb') as source: + info = os.fstat(source.fileno()) + self.identity, self.offset = (info.st_dev, info.st_ino), info.st_size + source.seek(max(0, self.offset - 1)) + self.partial_start = bool(self.offset and source.read(1) != b'\n') + + +def bind_metro(metro): + try: + return MetroBinding(metro) + except Exception: + return None + + +def metro_window(metro, binding): + result = {'state': 'UNKNOWN', 'scope': 'Bound product-sample console window; markers are not cause or request proof.', + 'process_state': 'UNKNOWN', 'exit_code': None} + try: + if (binding is None or metro.phase != 'product' or metro.process is not binding.process or + len(metro.events) != binding.count or metro.events[-1].get('phase') != 'product' or + Path(metro.private) != binding.root or not any(child is binding.process for child in metro.owned)): + return result + try: + code = binding.process.poll() + if code is None: + result['process_state'] = 'ALIVE' + elif type(code) is int: + result.update(process_state='EXITED', exit_code=code) + except Exception: + pass + info = regular(binding.path, binding.root) + if (info.st_dev, info.st_ino) != binding.identity or info.st_size < binding.offset: + return result + end, start = info.st_size, max(binding.offset, info.st_size - METRO_LIMIT) + with binding.path.open('rb') as source: + current = source.fileno() + opened = os.fstat(current) + if (opened.st_dev, opened.st_ino) != binding.identity: + return result + source.seek(start) + payload = source.read(end - start) + after = os.fstat(current) + current = regular(binding.path, binding.root) + if (len(payload) != end - start or min(after.st_size, current.st_size) < end or + (current.st_dev, current.st_ino) != binding.identity): + return result + partial_start = binding.partial_start or start > binding.offset + partial_end = bool(payload and not payload.endswith(b'\n')) + if partial_end: + payload = payload[:payload.rfind(b'\n') + 1] if b'\n' in payload else b'' + if partial_start: + payload = payload.split(b'\n', 1)[1] if b'\n' in payload else b'' + grew = max(after.st_size, current.st_size) > end + result.update(state='PARTIAL' if partial_start or partial_end or start > binding.offset or grew else 'COMPLETE', + bytes_appended=end - binding.offset, bytes_read=end - start, bytes_scanned=len(payload), + input_truncated=start > binding.offset, partial_start=partial_start, partial_end=partial_end, + growth_detected=grew, complete_lines=payload.count(b'\n')) + categories, headers, unknown, valid, invalid = set(), 0, 0, 0, 0 + for raw in payload.split(b'\n')[:-1]: + try: + line = raw.removesuffix(b'\r').decode('utf-8') + line = re.sub(r'\x1b\[[0-9;]*[A-Za-z]', '', line) + if any(unicodedata.category(char) in ('Cc', 'Zl', 'Zp') and char != '\t' for char in line): + raise ValueError() + except (UnicodeError, ValueError): + invalid += 1 + continue + valid += 1 + line = line.strip() + if not re.match(r'^(?:ERROR\b|FATAL ERROR:|\w*Error:|Unable to resolve module)', line, re.IGNORECASE): + continue + headers += 1 + found = [key for key, pattern in FAULTS.items() if re.search(pattern, line, re.IGNORECASE)] + categories.update(found) + unknown += not bool(found) + if invalid: + result['state'] = 'PARTIAL' if valid else 'UNKNOWN' + result.update(error_headers=headers, unclassified_error_headers=unknown, + valid_records=valid, invalid_records=invalid, + fault_categories=[key for key in FAULTS if key in categories]) + except Exception: + pass + return result + + +def collect(directory, flow, metro, binding=None): + directory = Path(directory) + return {'scope': 'Failed-sample files and bound owned Metro only; observations do not prove current visibility, route, tap delivery or cause.', + 'junit': junit(directory, flow), 'hierarchy': hierarchy(directory), + 'product_metro': metro_window(metro, binding)} diff --git a/.github/native/product_measurements.py b/.github/native/product_measurements.py index f89c5a118..e2950aca0 100644 --- a/.github/native/product_measurements.py +++ b/.github/native/product_measurements.py @@ -32,7 +32,7 @@ def native_module(name): - if name not in ('product_metro', 'product_credential_acceptance', + if name not in ('product_metro', 'product_failure_diagnosis', 'product_credential_acceptance', 'product_history_acceptance', 'product_vault_acceptance'): raise RuntimeError('PRODUCT_UI_FIXED_COMPANION_REQUIRED') spec = importlib.util.spec_from_file_location(name, Path(__file__).with_name(name + '.py')) @@ -66,6 +66,22 @@ def android_failure_observation(private, device): return {'status': 'UNAVAILABLE', 'scope': scope} +def prepare_failure_observation(metro): + try: + module = native_module('product_failure_diagnosis') + return module, module.bind_metro(metro) + except Exception: + return None + + +def sample_failure_observation(private, flow, metro, observer): + try: + module, binding = observer or (native_module('product_failure_diagnosis'), None) + return module.collect(private, flow, metro, binding) + except Exception: + return {'status': 'UNKNOWN', 'scope': 'Failed-sample observation unavailable; original failure retained.'} + + class Receipt: """Fixed vocabulary and numbers only; never store a request body/identifier.""" def __init__(self): @@ -375,8 +391,11 @@ def save(): else: h.command(['adb', '-s', device, 'install', '-r', artifact], 'Install exact product UI APK', timeout=120) h.command(['adb', '-s', device, 'shell', 'pm', 'clear', app], 'Clear only disposable development app') + flow = 'product_retry.yaml' if retry_fixture else 'product_first_wallet.yaml' + observer = None try: installed_before = key_companion.installed_proof(h, device) + observer = prepare_failure_observation(metro) h.command([data['maestro']['binary'], '--device', device, 'test', '--env', 'APP_ID=' + app, '--env', 'EXPECTED_INSTALLED_REGEX=' + expected_version['installed_regex'], @@ -384,8 +403,7 @@ def save(): '--flatten-debug-output', '--debug-output', str(output), '--test-output-dir', str(output), '--format', 'junit', '--output', str(report), - '.github/native/product_retry.yaml' if retry_fixture else - '.github/native/product_first_wallet.yaml'], + '.github/native/' + flow], 'Full-product controlled disconnect and Retry fixture' if retry_fixture else 'Full-product first-wallet and relaunch UI sample', timeout=600, error_details=False) h.successful_junit(report) @@ -403,6 +421,7 @@ def save(): result['samples'].append(outcome) except Exception: outcome = receipt.snapshot() + outcome['failure_observation'] = sample_failure_observation(output, flow, metro, observer) if h.PLATFORM == 'ANDROID': outcome['android_runtime_diagnostic'] = android_failure_observation(output, device) if retry_fixture: diff --git a/.github/native/startup_diagnosis.py b/.github/native/startup_diagnosis.py index 6acf3f48f..0bb7edf34 100644 --- a/.github/native/startup_diagnosis.py +++ b/.github/native/startup_diagnosis.py @@ -163,18 +163,27 @@ def observer(): text, truncated = _read(native, native.PRIVATE / 'startup-observer.jsonl') rows, invalid = [], 0 lines = text.splitlines() + ended_ms = int(time.time() * 1000) if native.PLATFORM == 'IOS' else None for line in lines[:1024]: if not line.strip(): continue try: row = json.loads(line) + if native.PLATFORM == 'IOS' and type(row) is not dict: + raise ValueError('Invalid observation shape.') when = row.get('time_ms') if type(when) is not int or when < 0 or when > 10**14: raise ValueError('Invalid timestamp.') - result = {'stage': 'UNKNOWN' if started_ms is None else 'APP_WINDOW' if when >= started_ms else 'BEFORE_APP'} + result = {'stage': 'UNKNOWN' if started_ms is None or (ended_ms is not None and when > ended_ms) + else 'APP_WINDOW' if when >= started_ms else 'BEFORE_APP'} kind = row.get('kind') if (kind == 'BUILD' and row.get('event') in ['bundle_build_started', 'bundle_build_done', 'bundle_build_failed']) or (kind == 'ERROR' and row.get('event') in ['bundling_error', 'initialize_failed', 'transformer_load_failed', 'hmr_client_error', 'cache_read_error', 'cache_write_error', 'worker_stderr_chunk']): result.update(kind=kind, event=row['event']) + if native.PLATFORM == 'IOS' and kind == 'BUILD': + platform = row.get('platform') + result['platform'] = platform if platform in ['android', 'ios', None] else 'OTHER' + result.update({key: row.get(key) if type(row.get(key)) is bool else None + for key in ['dev', 'minify']}) result['error_type'] = row.get('error_type') if row.get('error_type') in ['SyntaxError', 'TypeError', 'ReferenceError', 'RangeError', 'TransformError', 'UnableToResolveError', 'ResolutionError', 'AmbiguousModuleResolutionError', 'NotFoundError', 'ResourceNotFoundError', 'GraphNotFoundError', 'RevisionNotFoundError', 'OTHER', None] else 'OTHER' categories = row.get('categories', []) if native.PLATFORM == 'IOS' and (type(categories) is not list or @@ -185,6 +194,8 @@ def observer(): result.update(kind=kind, request=row['request']) if kind == 'REQUEST': options = row.get('options', {}) + if native.PLATFORM == 'IOS' and type(options) is not dict: + raise ValueError('Invalid request options shape.') result['options'] = {key: options.get(key) if type(options.get(key)) is bool or options.get(key) in [None, 'OTHER'] else 'OTHER' for key in ['dev', 'lazy', 'minify', 'modulesOnly', 'runModule', 'excludeSource']} for key, allowed in {'platform': ['android', 'ios'], 'sourcePaths': ['url-server', 'absolute'], 'engine': ['hermes', 'jsc'], 'app': ['EXPECTED']}.items(): result['options'][key] = options.get(key) if options.get(key) in [*allowed, None] else 'OTHER' diff --git a/.github/tests/test_diagnostic_privacy.py b/.github/tests/test_diagnostic_privacy.py index 923d7562d..86ac03b38 100644 --- a/.github/tests/test_diagnostic_privacy.py +++ b/.github/tests/test_diagnostic_privacy.py @@ -56,7 +56,16 @@ def test_arbitrary_private_text_cannot_be_exported_by_any_log_classifier(self): with self.subTest(route='ios', payload=payload[:70]): diagnostics = native.ios_startup_diagnostics( '2026-10-18 06:00:00.000 E hexa_keeper_dev[123:abc] ' + payload, 'hexa_keeper_dev') - self.assertEqual(diagnostics['errors'], ['IOS_STARTUP_ERROR']) + if '\x00' in payload or '\x1b' in payload: + # A malformed compact record cannot establish attribution + # even when its prefix names an error severity. + self.assertEqual(diagnostics['errors'], []) + self.assertEqual(diagnostics['parse_status'], 'UNKNOWN') + self.assertEqual(diagnostics['observation_state'], 'UNKNOWN') + self.assertGreater(diagnostics['malformed_record_lines'], 0) + else: + self.assertEqual(diagnostics['errors'], ['IOS_STARTUP_ERROR']) + self.assertNotIn('PRIVATE_SENTINEL', json.dumps(diagnostics)) def test_general_failure_categories_cover_install_blockers_without_excerpts(self): messages = [ diff --git a/.github/tests/test_hosted_native.py b/.github/tests/test_hosted_native.py index 95339e12b..5677780b0 100644 --- a/.github/tests/test_hosted_native.py +++ b/.github/tests/test_hosted_native.py @@ -358,12 +358,42 @@ def compact_log(args, stdout, **kwargs): self.assertTrue(diagnostics['input_truncated']) self.assertEqual(diagnostics['matching_process_records'], 1) self.assertEqual(diagnostics['errors'], ['IOS_STARTUP_ERROR']) + self.assertEqual(diagnostics['query_status'], 'SUCCESS') + self.assertEqual(diagnostics['parse_status'], 'PARTIAL') + self.assertEqual(diagnostics['cause'], 'UNDETERMINED') self.assertEqual(data['startup_diagnostics']['metro_progress'], {'bundle_marker_observed': True, 'bundle_marker_count': 2, 'prebundle_marker_count': 1}) self.assertNotIn('private unrelated', native.EVIDENCE.read_text()) self.assertNotIn('PRIVATE_SENTINEL', native.EVIDENCE.read_text()) self.assertNotIn('private-path', native.EVIDENCE.read_text()) + def test_ios_failed_queries_leave_valid_looking_partial_output_unknown(self): + native.PRIVATE.mkdir() + for code in [2, 124]: + with self.subTest(code=code): + native.save({'commands': [], 'smoke': 'FAIL', 'built_artifact': {'executable': 'hexa_keeper_dev'}}) + (native.PRIVATE / 'metro.log').write_text('normal startup\n') + def compact_log(args, stdout, **kwargs): + self.assertEqual(args[:4], ['xcrun', 'simctl', 'spawn', 'owned-simulator']) + self.assertIn('process == "hexa_keeper_dev"', args[-1]) + self.assertEqual(kwargs['timeout'], 25) + stdout.write(b'2026-10-22 09:22:33.123 E hexa_keeper_dev[123:abc] TypeError PRIVATE_SENTINEL\n') + if code == 124: + raise subprocess.TimeoutExpired(args, 25) + return subprocess.CompletedProcess(args, code) + with mock.patch.object(native, 'PLATFORM', 'IOS'), \ + mock.patch.object(subprocess, 'run', side_effect=compact_log): + native.record_startup_failure(native.PRIVATE / 'missing-report.xml', 'owned-simulator') + result = native.read()['startup_diagnostics']['ios_errors'] + self.assertEqual(result['query_status'], 'TIMEOUT' if code == 124 else 'FAILED') + self.assertEqual(result['observation_state'], 'UNKNOWN') + self.assertEqual(result['parse_status'], 'UNKNOWN') + self.assertIsNone(result['input_truncated']) + self.assertEqual(result['errors'], []) + self.assertNotIn('category_record_counts', result) + self.assertEqual(native.read()['smoke'], 'FAIL') + self.assertNotIn('PRIVATE_SENTINEL', native.EVIDENCE.read_text()) + def test_warmup_keeps_each_platform_body_private_and_does_not_supply_smoke_evidence(self): native.PRIVATE.mkdir() body = b'disposable development bundle PRIVATE_SENTINEL' diff --git a/.github/tests/test_ios_startup_diagnostics.py b/.github/tests/test_ios_startup_diagnostics.py index 8c9f9e732..463f2d011 100644 --- a/.github/tests/test_ios_startup_diagnostics.py +++ b/.github/tests/test_ios_startup_diagnostics.py @@ -3,6 +3,7 @@ import json from pathlib import Path import unittest +from unittest import mock spec = importlib.util.spec_from_file_location('ios_log_controls', Path(__file__).resolve().parents[1] / 'native/hosted_native.py') @@ -11,8 +12,16 @@ class IOSStartupDiagnosticsTests(unittest.TestCase): - def record(self, message, process='hexa_keeper_dev', severity='Df'): - return '2026-10-22 20:00:00.123+0000 %s %s[123:abc] %s' % (severity, process, message) + def record(self, message, process='hexa_keeper_dev', severity='Df', + timestamp='2026-10-22 20:00:00.123+0000', pid='123', tid='abc'): + return '%s %s %s[%s:%s] %s' % (timestamp, severity, process, pid, tid, message) + + def assert_private_and_undetermined(self, result): + encoded = json.dumps(result) + for value in ['PRIVATE_SENTINEL', '/private/seed.js', 'https://private.invalid/', + 'hexa_keeper_dev']: + self.assertNotIn(value, encoded) + self.assertEqual(result['cause'], 'UNDETERMINED') def test_known_js_native_bundle_and_network_markers_are_fixed_not_error_excerpts(self): cases = { @@ -74,6 +83,180 @@ def test_generic_errors_remain_unclassified_and_missing_records_remain_unknown(s self.assertEqual(absent['categories'], []) self.assertNotIn('PRIVATE_SENTINEL', json.dumps((result, absent))) + def test_supported_compact_timestamps_severity_and_thread_formats_keep_positive_records(self): + cases = [ + ('2026-01-15 20:41:18.941', 'Df', '11135', '3f6df'), + ('2026-10-22 09:22:33', 'Db', '123', 'abc'), + ('2026-10-22T09:22:33Z', 'I', '1', '0xabc'), + ('2026-10-22 09:22:33.123456+0000', 'E', '4321', '0x58ff'), + ('2026-10-22T09:22:33.123456789+00:00', 'F', '2147483647', 'ABCDEF0123456789'), + ('2026-10-22 09:22:33.1-0700', 'Df', '123', 'abc'), + ('2026-10-22T09:22:33-07:00', 'Df', '123', 'abc'), + ] + for timestamp, severity, pid, tid in cases: + with self.subTest(timestamp=timestamp, severity=severity): + result = native.ios_startup_diagnostics(self.record( + 'TypeError\tPRIVATE_SENTINEL', timestamp=timestamp, + severity=severity, pid=pid, tid=tid) + '\r\n', 'hexa_keeper_dev') + self.assertEqual(result['matching_process_records'], 1) + self.assertEqual(result['error_records'], 1) + self.assertEqual(result['category_record_counts']['JS_TYPE'], 1) + self.assertEqual(result['observation_state'], 'OWN_RECORDS_OBSERVED') + self.assertEqual(result['parse_status'], 'PARSED') + self.assertEqual(result['malformed_record_lines'], 0) + self.assert_private_and_undetermined(result) + + def test_invalid_record_framing_cannot_create_own_process_error_evidence(self): + bad_records = { + 'invalid month': self.record('TypeError PRIVATE_SENTINEL', timestamp='2026-99-22 09:22:33'), + 'invalid day': self.record('TypeError PRIVATE_SENTINEL', timestamp='2026-02-30 09:22:33'), + 'invalid time': self.record('TypeError PRIVATE_SENTINEL', timestamp='2026-10-22 25:22:33'), + 'unknown severity': self.record('TypeError PRIVATE_SENTINEL', severity='INJECTED'), + 'zero pid': self.record('TypeError PRIVATE_SENTINEL', pid='0'), + 'pid beyond signed range': self.record('TypeError PRIVATE_SENTINEL', pid='2147483648'), + 'oversized pid': self.record('TypeError PRIVATE_SENTINEL', pid='9999999999'), + 'nonhex thread': self.record('TypeError PRIVATE_SENTINEL', tid='xxx'), + 'nul': self.record('TypeError PRIVATE_SENTINEL\x00'), + 'embedded carriage return': self.record('TypeError PRIVATE_SENTINEL\rforged'), + 'vertical tab forged prefix': self.record('ordinary startup') + '\x0b' + + self.record('TypeError PRIVATE_SENTINEL'), + } + for kind, payload in bad_records.items(): + with self.subTest(kind=kind): + result = native.ios_startup_diagnostics(payload, 'hexa_keeper_dev') + self.assertEqual(result['matching_process_records'], 0) + self.assertEqual(result['error_records'], 0) + self.assertEqual(result['categories'], []) + self.assertEqual(result['errors'], []) + self.assertEqual(result['malformed_record_lines'], 1) + self.assertEqual(result['observation_state'], 'UNKNOWN') + self.assertEqual(result['parse_status'], 'UNKNOWN') + self.assert_private_and_undetermined(result) + + def test_record_byte_boundary_keeps_complete_records_and_rejects_oversized_utf8(self): + limit = 16 * 1024 + self.assertEqual(native.IOS_LOG_MAX_RECORD_BYTES, limit) + prefix = self.record('TypeError PRIVATE_SENTINEL ') + exact = prefix + 'x' * (limit - len(prefix.encode())) + result = native.ios_startup_diagnostics(exact + '\n', 'hexa_keeper_dev') + self.assertEqual(result['matching_process_records'], 1) + self.assertEqual(result['category_record_counts']['JS_TYPE'], 1) + self.assertEqual(result['oversized_record_lines'], 0) + self.assertEqual(result['parse_status'], 'PARSED') + self.assert_private_and_undetermined(result) + + multibyte = self.record('TypeError PRIVATE_SENTINEL ' + '\u79d8' * (limit // 2)) + self.assertLess(len(multibyte), limit) + self.assertGreater(len(multibyte.encode()), limit) + for payload in [exact + 'x', multibyte]: + with self.subTest(multibyte=payload is multibyte): + result = native.ios_startup_diagnostics( + payload + '\n' + self.record('ordinary startup'), 'hexa_keeper_dev') + self.assertEqual(result['matching_process_records'], 1) + self.assertEqual(result['error_records'], 0) + self.assertEqual(result['oversized_record_lines'], 1) + self.assertEqual(result['parse_status'], 'PARTIAL') + self.assert_private_and_undetermined(result) + + def test_one_mebibyte_prefix_discards_a_cut_record_even_when_its_error_marker_fits(self): + limit = 1024 * 1024 + self.assertEqual(native.IOS_LOG_MAX_INPUT_BYTES, limit) + first = self.record('No bundle URL present PRIVATE_SENTINEL') + '\n' + cut = self.record('TypeError PRIVATE_SENTINEL ') + padding = 'x' * (limit - len(first.encode()) - len(cut.encode()) - 9) + '\n' + payload = first + padding + cut + 'z' * 100 + self.assertGreater(len(payload.encode()), limit) + self.assertIn('TypeError', payload[:limit]) + result = native.ios_startup_diagnostics(payload, 'hexa_keeper_dev') + self.assertTrue(result['input_truncated']) + self.assertEqual(result['matching_process_records'], 1) + self.assertEqual(result['category_record_counts']['BUNDLE_LOAD'], 1) + self.assertEqual(result['category_record_counts']['JS_TYPE'], 0) + self.assertEqual(result['parse_status'], 'PARTIAL') + self.assert_private_and_undetermined(result) + + # A multibyte cut with no complete newline contributes no partial row. + result = native.ios_startup_diagnostics( + self.record('TypeError PRIVATE_SENTINEL ' + '\u79d8' * limit), 'hexa_keeper_dev') + self.assertTrue(result['input_truncated']) + self.assertEqual(result['matching_process_records'], 0) + self.assertEqual(result['errors'], []) + self.assertEqual(result['parse_status'], 'UNKNOWN') + self.assert_private_and_undetermined(result) + + def test_scan_and_own_record_limits_do_not_classify_unobserved_tail(self): + self.assertEqual(native.IOS_LOG_MAX_SCAN_LINES, 4096) + self.assertEqual(native.IOS_LOG_MAX_OWN_RECORDS, 4096) + with mock.patch.object(native, 'IOS_LOG_MAX_SCAN_LINES', 2): + result = native.ios_startup_diagnostics('notice\n' + + self.record('ordinary startup') + '\n' + + self.record('TypeError PRIVATE_SENTINEL'), 'hexa_keeper_dev') + self.assertTrue(result['scan_limit_reached']) + self.assertEqual(result['matching_process_records'], 1) + self.assertEqual(result['error_records'], 0) + self.assertEqual(result['parse_status'], 'PARTIAL') + self.assert_private_and_undetermined(result) + + with mock.patch.object(native, 'IOS_LOG_MAX_OWN_RECORDS', 1): + result = native.ios_startup_diagnostics('\n'.join([ + self.record('TypeError PRIVATE_SENTINEL', process='OtherApp'), + self.record('ordinary startup'), + self.record('TypeError PRIVATE_SENTINEL'), + ]), 'hexa_keeper_dev') + self.assertTrue(result['record_limit_reached']) + self.assertEqual(result['matching_process_records'], 1) + self.assertEqual(result['error_records'], 0) + self.assertEqual(result['parse_status'], 'PARTIAL') + self.assert_private_and_undetermined(result) + + def test_partial_input_retains_valid_positive_counts_without_claiming_complete_parse(self): + result = native.ios_startup_diagnostics( + self.record('TypeError PRIVATE_SENTINEL \ufffd'), 'hexa_keeper_dev') + self.assertTrue(result['replacement_character_observed']) + self.assertEqual(result['matching_process_records'], 1) + self.assertEqual(result['category_record_counts']['JS_TYPE'], 1) + self.assertEqual(result['errors'], ['IOS_STARTUP_ERROR']) + self.assertEqual(result['observation_state'], 'OWN_RECORDS_OBSERVED') + self.assertEqual(result['parse_status'], 'PARTIAL') + self.assert_private_and_undetermined(result) + + result = native.ios_startup_diagnostics(self.record('ordinary startup') + '\n' + + self.record('TypeError PRIVATE_SENTINEL', timestamp='2026-99-22 09:22:33'), + 'hexa_keeper_dev') + self.assertEqual(result['matching_process_records'], 1) + self.assertEqual(result['error_records'], 0) + self.assertEqual(result['malformed_record_lines'], 1) + self.assertEqual(result['parse_status'], 'PARTIAL') + self.assert_private_and_undetermined(result) + + def test_observed_size_2446_generic_records_remains_complete_and_unclassified(self): + payload = (self.record('PRIVATE_SENTINEL arbitrary application text', severity='E') + '\n') * 2446 + result = native.ios_startup_diagnostics(payload, 'hexa_keeper_dev') + self.assertEqual(result['matching_process_records'], 2446) + self.assertEqual(result['error_records'], 2446) + self.assertEqual(result['classified_error_records'], 0) + self.assertEqual(result['unclassified_error_records'], 2446) + self.assertFalse(any(result['category_record_counts'].values())) + self.assertFalse(result['input_truncated']) + self.assertFalse(result['scan_limit_reached']) + self.assertFalse(result['record_limit_reached']) + self.assertEqual(result['parse_status'], 'PARSED') + self.assert_private_and_undetermined(result) + + def test_empty_foreign_and_unframed_text_remain_unknown_without_private_output(self): + payloads = ['', 'PRIVATE_SENTINEL /private/seed.js https://private.invalid/', + self.record('TypeError PRIVATE_SENTINEL', process='OtherApp'), + ' ' + self.record('TypeError PRIVATE_SENTINEL'), + 'malformed PRIVATE_SENTINEL \ufffd'] + for payload in payloads: + with self.subTest(payload_present=bool(payload)): + result = native.ios_startup_diagnostics(payload, 'hexa_keeper_dev') + self.assertEqual(result['matching_process_records'], 0) + self.assertEqual(result['observation_state'], 'UNKNOWN') + self.assertEqual(result['parse_status'], 'UNKNOWN') + self.assertEqual(result['errors'], []) + self.assert_private_and_undetermined(result) + if __name__ == '__main__': unittest.main() diff --git a/.github/tests/test_product_failure_diagnosis.py b/.github/tests/test_product_failure_diagnosis.py new file mode 100644 index 000000000..55a1e9205 --- /dev/null +++ b/.github/tests/test_product_failure_diagnosis.py @@ -0,0 +1,320 @@ +import contextlib +import copy +import hashlib +import importlib.util +import json +from pathlib import Path +import tempfile +import types +import unittest +from unittest import mock + +NATIVE = Path(__file__).parents[1] / 'native' + + +def load(name): + spec = importlib.util.spec_from_file_location(name, NATIVE / (name + '.py')) + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +diagnosis = load('product_failure_diagnosis') +product = load('product_measurements') + + +class FileObservations(unittest.TestCase): + def setUp(self): + self.temp = tempfile.TemporaryDirectory() + self.addCleanup(self.temp.cleanup) + self.root = Path(self.temp.name) + self.flow = 'product_first_wallet.yaml' + + def report(self, message, **changes): + from xml.sax.saxutils import escape + body = ('' + + '' + escape(message) + '') + for old, new in changes.items(): + body = body.replace(old, new) + (self.root / 'junit.xml').write_text(body) + return diagnosis.junit(self.root, self.flow) + + def snapshot(self, node, number=1): + path = self.root / 'screen-hierarchy' / ('step-%d.json' % number) + path.parent.mkdir(exist_ok=True) + path.write_text(json.dumps(node)) + return path + + def test_pinned_junit_xml_text_exact_first_line_and_known_file_only(self): + for message, kind in [('Assertion is false: id: view_startNewTile is visible', 'ASSERT_VISIBLE'), + ('Element not found: Id matching regex: btn_skip', 'ELEMENT_NOT_FOUND')]: + value = self.report(message + '\nPRIVATE_SENTINEL diagnostic body') + self.assertEqual(value['failure_kind'], kind) + self.assertIn(value['failed_selector'], ('start_new_tile', 'onboarding_skip')) + self.assertNotIn('PRIVATE_SENTINEL', json.dumps(value)) + for message in ['PRIVATE_SENTINEL id: view_startNewTile', + 'Assertion is false: id: view_startNewTile is visible PRIVATE_SENTINEL', + 'Assertion is false: id: view_startNewTile is visible\u2028PRIVATE_SENTINEL', + 'Assertion is false: id: view_startNewTile is visible\u0085PRIVATE_SENTINEL', + 'PRIVATE_SENTINEL\nAssertion is false: id: view_startNewTile is visible', + 'Element not found: id: view_startNewTile']: + self.assertIsNone(self.report(message)['failed_selector']) + self.assertEqual(self.report('Assertion is false: id: view_startNewTile is visible', + **{'status="ERROR"': 'status="SUCCESS"'})['state'], 'UNKNOWN') + self.assertEqual(diagnosis.junit(self.root, 'private.yaml')['state'], 'UNKNOWN') + for malformed in [{'': '', '': ''}, + {'': ''}, + {'': '', '': ''}]: + result = self.report('Assertion is false: id: view_startNewTile is visible', + **malformed) + self.assertEqual(result['state'], 'UNKNOWN') + + def test_missing_malformed_entity_attribute_and_wrong_file_reports_are_unknown(self): + path = self.root / 'junit.xml' + for payload in [b'', b'', b']>', + ']>'.encode('utf-16'), + b'' + b'']: + path.write_bytes(payload) + self.assertEqual(diagnosis.junit(self.root, self.flow)['state'], 'UNKNOWN') + self.report('Assertion is false: id: view_startNewTile is visible') + self.assertEqual(diagnosis.junit(self.root, 'product_retry.yaml')['state'], 'UNKNOWN') + path.unlink() + self.assertEqual(diagnosis.junit(self.root, self.flow)['state'], 'UNKNOWN') + path.symlink_to(self.root / 'outside.xml') + self.assertEqual(diagnosis.junit(self.root, self.flow)['state'], 'UNKNOWN') + + def test_fixed_public_flags_leaf_omissions_and_no_arbitrary_values_escape(self): + self.snapshot({'children': [ + {'attributes': {'resource-id': 'view_startNewTile', 'text': 'PRIVATE_SENTINEL'}}, + {'attributes': {'resource-id': 'io.hexawallet.keeper.development:id/btn_skip'}}, + *[{'attributes': {'text': value}} for value in [ + 'Start New', 'Welcome to Bitcoin Keeper', 'Dismiss', 'Minimize', 'Render Error', + 'Unable to check your passcode. Please try again.', 'Retry', + '“PRIVATE_SENTINEL” Would Like to Send You Notifications', 'Allow', "Don't Allow"]]]}) + result = diagnosis.hierarchy(self.root) + self.assertEqual(result['state'], 'COMPLETE') + for name in ('start_new_tile', 'onboarding_skip', 'start_new_text', 'welcome', + 'credential_error', 'retry', 'logbox_controls', 'logbox_error_header', + 'notification_permission_controls'): + self.assertIs(result['flags'][name], True, name) + self.assertIs(result['flags']['recover_tile'], False) + self.assertNotIn('PRIVATE_SENTINEL', json.dumps(result)) + + def test_invalid_truncated_and_bounded_snapshots_do_not_clear_unknown_flags(self): + self.assertTrue(all(value is None for value in diagnosis.hierarchy(self.root)['flags'].values())) + self.snapshot({'attributes': {'resource-id': 'btn_skip'}}) + malformed = self.snapshot({'children': [42]}, 2) + result = diagnosis.hierarchy(self.root) + self.assertEqual(result['state'], 'PARTIAL') + self.assertIs(result['flags']['onboarding_skip'], True) + self.assertIsNone(result['flags']['start_new_tile']) + malformed.write_bytes(b' ' * (diagnosis.FILE_LIMIT + 1)) + self.assertEqual(diagnosis.hierarchy(self.root)['rejected_snapshots'], 1) + with mock.patch.object(diagnosis, 'NODE_LIMIT', 1): + self.snapshot({'children': [{}]}, 1) + self.assertEqual(diagnosis.hierarchy(self.root)['rejected_snapshots'], 2) + for number in range(3, 11): + self.snapshot({}, number) + self.assertTrue(diagnosis.hierarchy(self.root)['snapshot_limit_exceeded']) + + def test_isolated_permission_buttons_and_cross_snapshot_logbox_are_not_dialog_proof(self): + self.snapshot({'attributes': {'text': 'Allow'}, 'children': [{'attributes': {'text': 'Dismiss'}}]}) + self.snapshot({'attributes': {'text': "Don't Allow"}, 'children': [{'attributes': {'text': 'Minimize'}}]}, 2) + result = diagnosis.hierarchy(self.root) + self.assertIs(result['flags']['notification_permission_controls'], False) + self.assertIs(result['flags']['logbox_controls'], False) + + def test_duplicate_falsey_malformed_and_symlink_inputs_stay_unknown(self): + path = self.snapshot({}) + for payload in [b'{"attributes":{},"attributes":{"text":"PRIVATE_SENTINEL"}}', + b'{"attributes":null}', b'{"attributes":false}', b'{"attributes":[]}', + b'{"children":null}', b'{"children":false}', b'{"children":{}}', + b'{"attributes":{"text":42}}']: + with self.subTest(payload=payload): + path.write_bytes(payload) + value = diagnosis.hierarchy(self.root) + self.assertEqual(value['state'], 'UNKNOWN') + self.assertTrue(all(flag is None for flag in value['flags'].values())) + self.assertNotIn('PRIVATE_SENTINEL', json.dumps(value)) + path.unlink() + outside = self.root / 'outside.json' + outside.write_text('{}') + path.symlink_to(outside) + self.assertEqual(diagnosis.hierarchy(self.root)['state'], 'UNKNOWN') + + +class MetroObservations(unittest.TestCase): + def setUp(self): + self.temp = tempfile.TemporaryDirectory() + self.addCleanup(self.temp.cleanup) + self.root = Path(self.temp.name) + self.path = self.root / 'metro-01.log' + self.path.write_text('ERROR TypeError: PRIVATE_EARLIER\n') + self.process = types.SimpleNamespace(poll=lambda: None) + self.metro = types.SimpleNamespace(private=self.root, process=self.process, + phase='product', events=[{'phase': 'product'}], owned=[self.process]) + + def append(self, value): + with self.path.open('ab') as output: + output.write(value) + + def test_only_bound_sample_lines_and_fixed_categories_are_retained_even_after_exit(self): + bound = diagnosis.bind_metro(self.metro) + self.append(b'ERROR ReferenceError: PRIVATE_SENTINEL https://example.test/?secret=value\n') + self.process.poll = lambda: -9 + value = diagnosis.metro_window(self.metro, bound) + self.assertEqual(value['state'], 'COMPLETE') + self.assertEqual(value['fault_categories'], ['JS_REFERENCE_ERROR']) + self.assertEqual((value['process_state'], value['exit_code']), ('EXITED', -9)) + self.assertEqual(value['error_headers'], 1) + for forbidden in ('PRIVATE', 'example.test', 'secret', 'TYPE_ERROR'): + self.assertNotIn(forbidden, json.dumps(value)) + + def test_missing_replaced_shrunk_or_other_process_phase_event_cannot_be_adopted(self): + for change in ('missing', 'replaced', 'shrunk', 'process', 'phase', 'event', 'unowned'): + with self.subTest(change=change): + self.setUp() + bound = diagnosis.bind_metro(self.metro) + if change == 'missing': + self.path.unlink() + elif change == 'replaced': + replacement = self.root / 'replacement' + replacement.write_text('ERROR TypeError: PRIVATE_SENTINEL\n') + replacement.replace(self.path) + elif change == 'shrunk': + self.path.write_bytes(b'') + elif change == 'process': + self.metro.process = types.SimpleNamespace(poll=lambda: None) + elif change == 'phase': + self.metro.phase = 'arm' + elif change == 'event': + self.metro.events.append({'phase': 'product'}) + else: + self.metro.owned = [] + self.assertEqual(diagnosis.metro_window(self.metro, bound)['state'], 'UNKNOWN') + + def test_partial_start_end_limit_and_growth_are_disclosed(self): + self.path.write_bytes(b'PRIVATE_EARLIER_PARTIAL') + bound = diagnosis.bind_metro(self.metro) + self.append(b' TypeError\nERROR ReferenceError: PRIVATE_SENTINEL\nERROR SyntaxError: unfinished') + result = diagnosis.metro_window(self.metro, bound) + self.assertEqual(result['state'], 'PARTIAL') + self.assertTrue(result['partial_start']) + self.assertTrue(result['partial_end']) + self.assertEqual(result['fault_categories'], ['JS_REFERENCE_ERROR']) + with mock.patch.object(diagnosis, 'METRO_LIMIT', 18): + result = diagnosis.metro_window(self.metro, bound) + self.assertTrue(result['input_truncated']) + self.assertLessEqual(result['bytes_read'], 18) + original = diagnosis.os.fstat + calls = [] + def growing(fd): + info = original(fd) + calls.append(fd) + if len(calls) == 2: + return types.SimpleNamespace(st_size=info.st_size + 1) + return info + with mock.patch.object(diagnosis.os, 'fstat', side_effect=growing): + result = diagnosis.metro_window(self.metro, bound) + self.assertTrue(result['growth_detected']) + self.assertEqual(result['state'], 'PARTIAL') + + def test_malformed_utf8_controls_and_non_lf_separators_cannot_create_error_facts(self): + bound = diagnosis.bind_metro(self.metro) + for payload in [b'INFO not a newline\rERROR TypeError: PRIVATE_SENTINEL\n', + b'ERROR TypeError: \xff PRIVATE_SENTINEL\n', + b'ERROR TypeError: PRIVATE_SENTINEL\x00\n', + 'INFO\u2028ERROR TypeError: PRIVATE_SENTINEL\n'.encode()]: + with self.subTest(payload=payload): + self.path.write_bytes(self.path.read_bytes()[:bound.offset] + payload) + result = diagnosis.metro_window(self.metro, bound) + self.assertEqual(result['state'], 'UNKNOWN') + self.assertEqual(result['fault_categories'], []) + self.assertEqual(result['invalid_records'], 1) + self.assertEqual(result['error_headers'], 0) + self.assertNotIn('PRIVATE_SENTINEL', json.dumps(result)) + self.path.write_bytes(self.path.read_bytes()[:bound.offset] + + b'ERROR TypeError: PRIVATE_SENTINEL\x00\n\x1b[31mERROR ReferenceError: fixed\x1b[0m\r\n') + result = diagnosis.metro_window(self.metro, bound) + self.assertEqual(result['state'], 'PARTIAL') + self.assertEqual((result['valid_records'], result['invalid_records']), (1, 1)) + self.assertEqual(result['fault_categories'], ['JS_REFERENCE_ERROR']) + + +class FailureLifecycle(unittest.TestCase): + def test_original_error_identity_and_purge_survive_failed_observer_and_cleanup(self): + class PrimaryFailure(RuntimeError): + pass + with tempfile.TemporaryDirectory() as directory: + root, order, receipts = Path(directory), [], [] + private = root / 'private' + private.mkdir() + (root / 'release').mkdir() + manifest = b'{"name":"owned"}' + (root / 'package.json').write_bytes(manifest) + (root / 'release/version.json').write_text(json.dumps( + {'version': '2.6.4', 'androidVersionCode': 627, 'iosBuildNumber': 620})) + (root / '.env.ci').write_text('\n'.join([ + 'ENVIRONMENT=DEVELOPMENT', 'RELAY=http://127.0.0.1:8765/', + 'SIGNING_SERVER_TESTNET=https://127.0.0.1:9/', 'SIGNING_SERVER_MAINNET=https://127.0.0.1:9/', + 'CHANNEL_URL=https://127.0.0.1:9/', 'SENTRY_DNS=https://public@127.0.0.1:9/1', + 'ENC_KEY_STORAGE_IDENTIFIER=KEEPER-HOSTED-DISPOSABLE'])) + sha, artifact = 'a' * 40, {'sha256': 'b' * 64} + digest = hashlib.sha256(manifest).hexdigest() + state = {'build': 'PASS', 'source_sha': sha, 'built_artifact': artifact, + 'commands': [], 'maestro': {'binary': 'fake-maestro'}, 'nodeify_overlay': { + 'path': 'package.json', 'allowed_fields': ['browser', 'react-native', 'browserify'], + 'before_sha256': digest, 'after_sha256': digest}} + error = PrimaryFailure('PRIVATE_SENTINEL original') + def save(value): + receipts.append(copy.deepcopy(value['product_ui'])) + state.update(copy.deepcopy(value)) + def command(args, *_args, **_kwargs): + state['commands'].append({}) + if args == ['adb', 'get-serialno']: + return 'emulator-5554' + if args[0] == 'fake-maestro': + output = Path(args[args.index('--debug-output') + 1]) + (output / 'junit.xml').write_text('PRIVATE_SENTINEL') + raise error + return 'image' + def observe(output, *_args): + self.assertTrue((output / 'junit.xml').exists()) + order.append('observer') + raise OSError('PRIVATE_SENTINEL observation') + observer = types.SimpleNamespace(bind_metro=lambda _: None, collect=observe) + def close(): + order.append('close') + raise OSError('PRIVATE_SENTINEL cleanup') + manager = types.SimpleNamespace(start=lambda *_a, **_k: None, close=close, events=[]) + server = types.SimpleNamespace(serve_forever=lambda: None, shutdown=lambda: None, server_close=lambda: None) + def installed(_device): + proof = {'method': 'SINGLE_INSTALLED_BASE_APK_SHA256', 'sha256': 'c' * 64} + state.update(installed_artifact_verified=True, installed_artifact=proof) + return proof + h = types.SimpleNamespace(ROOT=root, PRIVATE=private, PLATFORM='ANDROID', APP_IDS={'ANDROID': 'owned.app'}, + read=lambda: copy.deepcopy(state), save=save, artifact_identity=lambda: artifact, command=command, + verify_installed_artifact=installed) + def module(name): + return observer if name == 'product_failure_diagnosis' else types.SimpleNamespace(OwnedMetro=lambda *_: manager) + with contextlib.ExitStack() as stack: + stack.enter_context(mock.patch.dict(product.os.environ, {'GITHUB_ACTIONS': 'true', + 'RUNNER_ENVIRONMENT': 'github-hosted', 'KEEPER_SOURCE_SHA': sha})) + stack.enter_context(mock.patch.object(product.subprocess, 'check_output', side_effect=[sha, manifest])) + stack.enter_context(mock.patch.object(product.subprocess, 'run', return_value=types.SimpleNamespace(returncode=0))) + stack.enter_context(mock.patch.object(product, 'native_module', side_effect=module)) + stack.enter_context(mock.patch.object(product, 'make_server', return_value=server)) + stack.enter_context(mock.patch.object(product, 'android_failure_observation', return_value={'status': 'UNAVAILABLE'})) + with self.assertRaises(PrimaryFailure) as failure: + product.run(h) + self.assertIs(failure.exception, error) + self.assertEqual(order, ['observer', 'close']) + self.assertEqual(list(private.iterdir()), []) + self.assertEqual(len(state['product_ui']['samples']), 1) + self.assertEqual(state['product_ui']['samples'][0]['failure_observation']['status'], 'UNKNOWN') + self.assertNotIn('PRIVATE_SENTINEL', json.dumps(receipts)) + + +if __name__ == '__main__': + unittest.main() diff --git a/.github/tests/test_startup_diagnosis.py b/.github/tests/test_startup_diagnosis.py index 938a035b9..5339e8945 100644 --- a/.github/tests/test_startup_diagnosis.py +++ b/.github/tests/test_startup_diagnosis.py @@ -117,7 +117,7 @@ def test_ios_timestamp_only_start_collects_metro_on_success_without_native_comma {'time_ms': 151, 'kind': 'RESPONSE', 'request': 'ENTRY_BUNDLE', 'outer_http_status': 200}, ] (self.root / 'startup-observer.jsonl').write_text('\n'.join(json.dumps(row) for row in rows)) - with mock.patch.object(diagnosis.time, 'time', return_value=0.150), \ + with mock.patch.object(diagnosis.time, 'time', side_effect=[0.150, 0.200]), \ mock.patch.object(diagnosis.subprocess, 'Popen') as popen, \ mock.patch.object(diagnosis.subprocess, 'run') as run: handle = diagnosis.start(native, device) @@ -254,6 +254,53 @@ def test_ios_supplied_falsey_nonobject_attributes_and_nonlist_children_stay_unkn self.assertIsNone(facts['hierarchy']['credential_error']) self.assertIn('hierarchy', facts['collection_errors']) + def test_ios_future_rows_are_unknown_and_build_options_remain_fixed_without_inheritance(self): + native = SimpleNamespace(PRIVATE=self.root, PLATFORM='IOS') + device = '12345678-1234-1234-1234-123456789ABC' + rows = [ + {'time_ms': 149, 'kind': 'BUILD', 'event': 'bundle_build_started', + 'platform': 'ios', 'dev': True, 'minify': False}, + {'time_ms': 150, 'kind': 'BUILD', 'event': 'bundle_build_done'}, + {'time_ms': 200, 'kind': 'BUILD', 'event': 'bundle_build_failed', + 'platform': 'PRIVATE_SENTINEL', 'dev': 1, 'minify': 'PRIVATE_SENTINEL'}, + {'time_ms': 201, 'kind': 'REQUEST', 'request': 'ENTRY_BUNDLE', 'options': {'platform': 'ios'}}, + ] + (self.root / 'startup-observer.jsonl').write_text('\n'.join(json.dumps(row) for row in rows)) + with mock.patch.object(diagnosis.time, 'time', return_value=0.200): + facts = diagnosis.collect(native, device, SimpleNamespace(started_ms=150))['metro'] + self.assertEqual([row['stage'] for row in facts['rows']], ['BEFORE_APP', 'APP_WINDOW', 'APP_WINDOW', 'UNKNOWN']) + self.assertEqual({key: facts['rows'][0][key] for key in ['platform', 'dev', 'minify']}, + {'platform': 'ios', 'dev': True, 'minify': False}) + self.assertEqual({key: facts['rows'][1][key] for key in ['platform', 'dev', 'minify']}, + {'platform': None, 'dev': None, 'minify': None}) + self.assertEqual({key: facts['rows'][2][key] for key in ['platform', 'dev', 'minify']}, + {'platform': 'OTHER', 'dev': None, 'minify': None}) + self.assertNotIn('PRIVATE_SENTINEL', json.dumps(facts)) + + def test_ios_row_and_options_shapes_are_invalid_and_android_schema_is_unchanged(self): + native = SimpleNamespace(PRIVATE=self.root, PLATFORM='IOS') + device = '12345678-1234-1234-1234-123456789ABC' + rows = [[], None, 'PRIVATE_SENTINEL', + {'time_ms': 150, 'kind': 'REQUEST', 'request': 'ENTRY_BUNDLE', 'options': []}, + {'time_ms': 150, 'kind': 'REQUEST', 'request': 'ENTRY_BUNDLE', 'options': 'PRIVATE_SENTINEL'}] + path = self.root / 'startup-observer.jsonl' + path.write_text('\n'.join(json.dumps(row) for row in rows)) + facts = diagnosis.collect(native, device)['metro'] + self.assertEqual(facts['invalid_rows'], len(rows)) + self.assertEqual(facts['rows'], []) + self.assertEqual(facts['coverage'], 'UNKNOWN') + self.assertNotIn('PRIVATE_SENTINEL', json.dumps(facts)) + path.write_text(json.dumps({'time_ms': 201, 'kind': 'BUILD', 'event': 'bundle_build_started', + 'platform': 'android', 'dev': True, 'minify': False})) + with mock.patch.object(diagnosis.time, 'time', return_value=0.200), \ + mock.patch.object(diagnosis.subprocess, 'run', return_value=SimpleNamespace(returncode=1, stdout=b'')): + facts = diagnosis.collect(self.native, 'emulator-5554', SimpleNamespace(started_ms=150))['metro'] + self.assertEqual(facts['rows'][0]['stage'], 'APP_WINDOW', 'the new end bound is iOS-only') + self.assertNotIn('platform', facts['rows'][0]) + self.assertNotIn('dev', facts['rows'][0]) + self.assertNotIn('minify', facts['rows'][0]) + self.assertNotIn('coverage', facts) + if __name__ == '__main__': unittest.main() diff --git a/docs/product-failure-observation.md b/docs/product-failure-observation.md new file mode 100644 index 000000000..d09be092f --- /dev/null +++ b/docs/product-failure-observation.md @@ -0,0 +1,84 @@ +# Failed product sample observation + +This preparation adds observation only. It changes no application code, +dependencies, Maestro flow, selector, deadline, Metro configuration or lifecycle, +and adds no native query, capture process, retry or validation window. + +Run `39846002094`, attempt `1`, source +`23f14e01f3437c213f1443c3167d96022f3a43bc` completed naturally. Android build, +startup smoke, persistence probe, five production-worker scenarios, exact source +receipts and installed APK bytes passed. Full-product sample 1 failed after the +host acknowledged the Skip command, before `start_new_visible`. That optional +marker prefix alone does not identify the failed Maestro command or an app +defect. Samples 2–5, full-product Retry/key-PIN checks and physical checks did not +run. The new reader has not run on a native host, and cannot add evidence to that +older source. Its cause remains undetermined. + +Artifact 12458149136 has API/upload/download ZIP SHA256 +`4d464ef5f8faa7de4f02292e683f0dad8242544843610da9371f2c675dc183f0`. +The only raw full-product duration was startup-to-passcode 21652.183ms; worker +helper timings do not satisfy the five full-product samples. Installed APK bytes +passed before and after the completed probes and before the failed product +sample; its post-product comparison did not run. + +The product coordinator calls `product_failure_diagnosis.py` on failure, before +removing that sample's private output. Observer/import failures yield UNKNOWN +and retain the original exception. Existing sample and outer cleanup still run; +raw JUnit, hierarchy, screenshots, app output and Metro logs remain private and +are removed by the existing cleanup. The Android post-failure four-command +collector is unchanged and retains its separate unknown-freshness scope. + +The reader exports three separate observations: + +- JUnit: only the pinned Maestro 2.11 XML-text failure format from the expected + single flow/test case, with status ERROR. A complete first line must equal a + fixed known assertion/element selector message. Other messages disclose only + ERROR_RECORDED or UNKNOWN. Failure attributes, custom labels, suffix matches, + unknown flow files, multiple failures and malformed XML cannot identify a + selector. No missing PIN marker is converted into a failed command. +- Hierarchy: fixed public IDs and strings for Skip, Start New, recovery/settings, + welcome/passcode/credential Retry, LogBox and notification permission controls. + This is explicitly ANY_RETAINED_SNAPSHOT, never the final post-Skip screen or + proof of current visibility, route, app ownership or tap delivery. Paired + LogBox/notification controls must occur within one snapshot. Allow/Don't Allow + alone does not establish a permission dialog. Modern LogBox resource IDs are + not treated as reliable iOS accessibility IDs. +- Product Metro: immediately before the original Maestro command, bind the + original owned Popen object, product phase, event count, regular log file + device/inode and EOF offset. On failure, require the same ownership, phase, + event and file identity; inspect only newly appended complete lines. Earlier + warmup and sample bytes are excluded. An exited owned Metro can still supply + its bound log; its numeric exit status is an observation, not cause proof. + +JUnit/hierarchy reads are limited to 1 MiB per file, eight snapshots, 10,000 nodes +per snapshot and depth 128. Oversized, duplicate-key, malformed, invalid-shape or +symlink inputs cannot establish absent UI. Partial coverage keeps unseen flags +null; true flags describe positive observations in a valid retained snapshot. +False flags describe only complete scanned snapshots. No arbitrary hierarchy +attribute, error text, URL, query string, filename or device text is exported. + +Metro reads at most the latest 256 KiB appended within the bound window, trims +partial starting/ending lines, and discloses byte counts, truncation, partial +boundaries and growth during reading. Missing, replaced, shrunk or differently +owned logs remain UNKNOWN. Records use LF boundaries, optional CRLF endings and +strict UTF-8; unsupported controls or invalid encoding increase fixed rejected +record counts and cannot establish an error category. Fixed console error categories do not establish +request success, bundle completion, runtime readiness, app health or root cause. + +Format evidence is in pinned Maestro `cli-2.11.0` sources, commit +`c8c361be1cc89b2e742f185466a302a22778be6b`: +`JUnitTestSuiteReporter.kt` writes `Failure.message` as XML text; +`TestSuiteInteractor.kt` uses the flow exception message; +`Orchestra.kt`, `Condition.kt` and `ElementSelector.kt` define the exact assertion +and element-selector descriptions. `ArtifactsGenerator.kt` serializes the root +`TreeNode`; the writer omits empty fields. App IDs/public text are defined by +`OnBoardingSlides.tsx`, `NewKeeperAppScreen.tsx` and English localization. +React Native 0.83.9 LogBox sources define its public footer/header strings; +Maestro's `PermissionButtonFinder.swift` supplies the notification prompt phrase. + +Host tests cover exact/negative formats, privacy sentinels, malformed/falsey and +duplicate fields, bounded/partial coverage, Metro source/window binding, and +preservation of the original failure plus purge when observation and cleanup +throw. The combined preparation passed 152 Node and 246 Python controls with no +failures or skips. These checks provide no native acceptance and select no +additional validation window. diff --git a/docs/r5-ios-startup-observation.md b/docs/r5-ios-startup-observation.md index 047639f36..51cdd0860 100644 --- a/docs/r5-ios-startup-observation.md +++ b/docs/r5-ios-startup-observation.md @@ -1,13 +1,15 @@ # iOS startup observation preparation -This isolated preparation starts from exact source -`23f14e01f3437c213f1443c3167d96022f3a43bc`. Its hosted run39846002094 attempt1 +The published #7116 preparation `76d1c949d91c5d885fc002f689a9ec321c16eafb` +starts from exact app source `23f14e01f3437c213f1443c3167d96022f3a43bc`. +That app source's hosted run39846002094 attempt1 passed driver readiness, unsigned development build, app installation and the pre-UI installed-byte comparison, then failed to find Create a passcode. Native persistence, production workers and product checks did not run on iOS. The retained metadata has36 Metro bundle markers and2446 generic own-process log -records; those facts do not identify the startup failure. The active combined -window and its serialized Android execution remain untouched. +records; those facts do not identify the startup failure. That combined window +and its serialized Android execution finished naturally. Their source and +results remain preserved. This changes observation only. Normal iOS startup uses the existing Metro middleware and CLI reporter observer already used for Android. Resolver, @@ -38,6 +40,33 @@ lost. No coverage value claims a complete startup trace. Malformed category containers and supplied non-object hierarchy attributes/non-list children remain invalid/UNKNOWN rather than producing diagnostic facts. +A subsequent diagnostic-truth adaptation uses published76 as its base and +adopts compatible validation semantics from #7115 exact source +`eceb5551a430a0e31cff7a32d70181ee55007b69`. It is not a startup-cause or app +correction. Pure differential controls showed that76 could credit impossible +dates, unsupported severity, PID0/oversized PID, malformed TID and C0 controls as +own-process errors; vertical-tab splitting could turn message text into another +record. The port validates a complete compact prefix, actual date, supported +severity, positive bounded PID and hexadecimal TID, rejects control characters +apart from tabs, and scans newline-delimited records only. Valid Apple compact +records without a timezone and with bare hexadecimal TIDs remain supported. + +The existing1MiB file-prefix bound is retained. The pure classifier also bounds +input to1MiB, individual records to16KiB and scanned lines/own records to4096. +It reports malformed/oversized/replacement/cap facts and UNKNOWN/PARTIAL/PARSED +parse state. PARSED describes supported bounded input, not a complete startup +trace, process liveness or authenticated log provenance. Reader and classifier +truncation are combined; failed or timed-out queries remain UNKNOWN and do not +classify their partial output. The original13 category names, counts, generic +fallback and observation state remain intact, including Hermes/ATS coverage. + +On iOS only, future-dated Metro rows have UNKNOWN window attribution. Fixed +BUILD platform/dev/minify options are retained as enums/booleans/unknown values; +done/failed events do not inherit options from an earlier started event. Row +and request-options shapes are checked before projection. The stronger76 +category-list validation, explicit512/1024 coverage and modern Login/LogBox +hierarchy flags remain intact. Android collector behavior/schema is unchanged. + The Login flags use exact23f English strings Enter your passcode and Welcome Back. Credential-error and Retry are separate public flags; Retry alone does not prove a credential error. Generic please wait/loading text belongs to the @@ -68,9 +97,14 @@ reporter forwarding, platform/window separation, missing/invalid data, public hierarchy positives/negatives, own-process log attribution and classification, one unchanged Maestro call, probe exclusion, original failure preservation and cleanup. They provide no native acceptance. Hosted iOS validation of this -preparation is NOT RUN; physical devices and signers are NOT RUN. The original -startup cause remains UNKNOWN. The final offline suites passed 152 Node controls +semantic port is NOT RUN; physical devices and signers are NOT RUN. The original +startup cause remains UNKNOWN. The published76 offline suites passed152 Node controls and 224 Python controls; independent review cleared malformed category shapes, malformed hierarchy fields and undisclosed row-cap coverage. Those are diagnostic accuracy checks, not native evidence. No additional native window is selected -by this preparation. +by this adaptation; publication and execution remain separately coordinated. + +The combined diagnostic preparation, including failed-product observations, +passed 152 Node and 246 Python controls with zero failures or skips. Independent +review cleared both the iOS parser adaptation and product reader. App source, +dependencies, flows, workflows and deadline arguments remain unchanged from 23f.