diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS new file mode 100644 index 0000000000..baadc36f3e --- /dev/null +++ b/.github/CODEOWNERS @@ -0,0 +1,2 @@ +# Request the three developers for all changes; one approval is required. +* @ben-kaufman @cakesoft-utkarsh @Parsh diff --git a/.github/ci/public_ci_mongo.py b/.github/ci/public_ci_mongo.py new file mode 100644 index 0000000000..352b0f67e6 --- /dev/null +++ b/.github/ci/public_ci_mongo.py @@ -0,0 +1,122 @@ +"""Own the existing public CI replica; failed queries never prove absence.""" +import json +import os +import re +import subprocess +import sys +import tempfile +import time + +IMAGE = 'mongo:7.0@sha256:9854f7139445d766a9523571d6f047530c45547460ffcf8259eb2bf4264632ca' +LABEL = 'keeper.r5.public-ci.owner' +PORT = '27058' +INIT = ('try { rs.status(); } catch (e) { if (e.code !== 94) throw e; ' + 'rs.initiate({_id: "keeperQA", members: [{_id: 0, host: "127.0.0.1:27058"}]}); } ' + 'quit(db.hello().isWritablePrimary ? 0 : 1)') + + +class ReplicaError(Exception): + pass + + +def identity(env): + if (env.get('GITHUB_ACTIONS') != 'true' or env.get('RUNNER_ENVIRONMENT') != 'github-hosted' or + env.get('GITHUB_REPOSITORY') != 'KeeperCommunity/bitcoin-keeper'): + raise ReplicaError('HOSTED_PUBLIC_CI_REQUIRED') + values = [env.get(key, '') for key in ('GITHUB_RUN_ID', 'GITHUB_RUN_ATTEMPT')] + if not all(isinstance(value, str) and re.fullmatch(r'[1-9][0-9]{0,19}', value) for value in values): + raise ReplicaError('INVALID_RUN_IDENTITY') + token = '-'.join(values) + return 'keeper-r5-public-ci-' + token, token + + +def docker(args, timeout=10): + try: + with tempfile.TemporaryFile() as output: + result = subprocess.run(['docker', *args], stdout=output, stderr=subprocess.DEVNULL, + timeout=timeout, check=False) + output.seek(0) + raw = output.read(4097) + if len(raw) > 4096: + raise ReplicaError('DOCKER_OUTPUT_UNKNOWN') + return result.returncode, raw.decode('utf-8').rstrip('\r\n') + except (OSError, subprocess.SubprocessError, UnicodeError): + raise ReplicaError('DOCKER_QUERY_FAILED') from None + + +def present(name): + code, value = docker(['container', 'ls', '--all', '--filter', 'name=^/' + name + '$', + '--format', '{{.Names}}']) + if code or value not in ('', name): + raise ReplicaError('DOCKER_INVENTORY_UNKNOWN') + return value == name + + +def owned(name, token, *, live=False): + template = '{{ index .Config.Labels "' + LABEL + '" }}|{{.Config.Image}}|{{.State.Running}}' + code, value = docker(['inspect', '--format', template, name]) + expected = token + '|' + IMAGE + '|' + if code or value not in (expected + 'true', expected + 'false'): + raise ReplicaError('CONTAINER_OWNER_OR_IMAGE_UNKNOWN') + if live and value != expected + 'true': + raise ReplicaError('OWNED_CONTAINER_NOT_RUNNING') + + +def start(name, token): + if present(name): + raise ReplicaError('CONTAINER_ALREADY_EXISTS') + # The deterministic name/label survive a committed start whose CLI reply is + # lost. The always-run cleanup independently discovers that exact resource. + code, _ = docker(['run', '--detach', '--name', name, '--label', LABEL + '=' + token, + '--publish', '127.0.0.1:27058:27058', IMAGE, + 'mongod', '--replSet', 'keeperQA', '--bind_ip_all', '--port', PORT], 120) + if code: + raise ReplicaError('CONTAINER_START_FAILED') + deadline = time.monotonic() + 60 + while time.monotonic() < deadline: + owned(name, token, live=True) + remaining = deadline - time.monotonic() + if remaining <= 0: + break + code, _ = docker(['exec', name, 'mongosh', '--port', PORT, '--quiet', '--eval', INIT], + min(10, remaining)) + if code == 0 and time.monotonic() <= deadline: + owned(name, token, live=True) + if time.monotonic() > deadline: + break + return 'writable' + time.sleep(min(2, max(0, deadline - time.monotonic()))) + raise ReplicaError('REPLICA_NOT_WRITABLE') + + +def cleanup(name, token): + if not present(name): + return 'absent' + owned(name, token) + code, removed = docker(['rm', '--force', '--volumes', name], 30) + if code or removed != name: + raise ReplicaError('CONTAINER_REMOVAL_FAILED') + if present(name): + raise ReplicaError('CONTAINER_REMAINS') + return 'removed' + + +def main(args=None, env=None): + args = sys.argv[1:] if args is None else args + phase = args[0] if len(args) == 1 and args[0] in ('start', 'cleanup') else 'INVALID' + try: + if phase == 'INVALID': + raise ReplicaError('INVALID_PHASE') + name, token = identity(os.environ if env is None else env) + disposition = (start if phase == 'start' else cleanup)(name, token) + print(json.dumps({'schema': 1, 'scope': 'PUBLIC_LEGACY_MATRIX_REPLICA_ONLY', + 'phase': phase, 'result': 'PASS', 'container': name, + 'disposition': disposition})) + return 0 + except ReplicaError as error: + print(json.dumps({'schema': 1, 'phase': phase, 'result': 'FAIL', 'error': str(error)})) + return 1 + + +if __name__ == '__main__': + raise SystemExit(main()) diff --git a/.github/ci/test_public_ci_mongo.py b/.github/ci/test_public_ci_mongo.py new file mode 100644 index 0000000000..6b1df6e442 --- /dev/null +++ b/.github/ci/test_public_ci_mongo.py @@ -0,0 +1,131 @@ +"""Fake Docker fault controls only; no container, database or native operation.""" +import contextlib +import importlib.util +import io +import json +from pathlib import Path +import subprocess +from types import SimpleNamespace +import unittest +from unittest.mock import patch + +SPEC = importlib.util.spec_from_file_location('public_ci_mongo', Path(__file__).with_name('public_ci_mongo.py')) +mongo = importlib.util.module_from_spec(SPEC) +SPEC.loader.exec_module(mongo) +ENV = {'GITHUB_ACTIONS': 'true', 'RUNNER_ENVIRONMENT': 'github-hosted', + 'GITHUB_REPOSITORY': 'KeeperCommunity/bitcoin-keeper', 'GITHUB_RUN_ID': '123', + 'GITHUB_RUN_ATTEMPT': '2', 'KEEPER_CONTAINER_NAME': 'foreign'} +NAME, TOKEN = mongo.identity(ENV) +OWNER = TOKEN + '|' + mongo.IMAGE + '|true' +TIMEOUT = subprocess.TimeoutExpired('PRIVATE_SENTINEL', 1) + + +class MongoFaults(unittest.TestCase): + def exercise(self, phase, answers=(), *, env=ENV, ticks=None): + answers, calls = iter(answers), [] + def run(args, **kwargs): + calls.append(args) + self.assertEqual(args[0], 'docker') + self.assertGreater(kwargs['timeout'], 0) + self.assertLessEqual(kwargs['timeout'], 120) + self.assertEqual(kwargs['stderr'], subprocess.DEVNULL) + value = next(answers) + if isinstance(value, BaseException): + raise value + code, raw = value + kwargs['stdout'].write(raw if isinstance(raw, bytes) else raw.encode()) + return SimpleNamespace(returncode=code) + output = io.StringIO() + clock = patch.object(mongo.time, 'monotonic', side_effect=ticks) if ticks else contextlib.nullcontext() + with patch.object(mongo.subprocess, 'run', side_effect=run), clock, \ + patch.object(mongo.time, 'sleep'), contextlib.redirect_stdout(output): + status = mongo.main([phase], env) + self.assertNotIn('PRIVATE_SENTINEL', output.getvalue()) + value = json.loads(output.getvalue()) + self.assertEqual(value['result'], 'FAIL' if status else 'PASS') + return status, value, calls + + def test_exact_known_absence_and_owned_removal(self): + self.assertEqual(self.exercise('cleanup', [(0, '')])[1]['disposition'], 'absent') + status, value, calls = self.exercise('cleanup', [(0, NAME), (0, OWNER), (0, NAME), (0, '')]) + self.assertEqual(status, 0) + self.assertEqual(value['disposition'], 'removed') + self.assertEqual(calls[2], ['docker', 'rm', '--force', '--volumes', NAME]) + + def test_stopped_owned_container_can_be_removed(self): + self.assertEqual(self.exercise('cleanup', [(0, NAME), (0, OWNER.replace('|true', '|false')), + (0, NAME), (0, '')])[0], 0) + + def test_inventory_query_failure_or_malformed_output_never_means_absent(self): + for value in [(1, ''), (0, 'different'), (0, ' ' + NAME), (0, NAME + '\n' + NAME), + (0, 'x' * 4097), (0, b'\xff'), TIMEOUT, OSError('PRIVATE_SENTINEL')]: + with self.subTest(value=type(value).__name__): + status, _, calls = self.exercise('cleanup', [value]) + self.assertEqual(status, 1) + self.assertEqual(len(calls), 1) + + def test_foreign_label_image_unknown_state_or_failed_inspection_never_removes(self): + for value in [(0, 'foreign|' + mongo.IMAGE + '|true'), (0, TOKEN + '|other|true'), + (0, OWNER + ' '), (0, TOKEN + '|' + mongo.IMAGE + '|unknown'), + (1, OWNER), TIMEOUT]: + with self.subTest(value=type(value).__name__): + status, _, calls = self.exercise('cleanup', [(0, NAME), value]) + self.assertEqual(status, 1) + self.assertFalse(any(call[1] == 'rm' for call in calls)) + + def test_removal_response_and_final_inventory_both_have_to_confirm(self): + for tail in [[(1, '')], [TIMEOUT], [(0, 'different')], [(0, NAME), (0, NAME)], + [(0, NAME), (1, '')], [(0, NAME), TIMEOUT]]: + with self.subTest(length=len(tail)): + self.assertEqual(self.exercise('cleanup', [(0, NAME), (0, OWNER), *tail])[0], 1) + + def test_invalid_host_repository_or_run_context_makes_no_docker_call(self): + for key, value in [('GITHUB_ACTIONS', 'false'), ('RUNNER_ENVIRONMENT', 'self-hosted'), + ('GITHUB_REPOSITORY', 'KeeperCommunity/Relay'), + *[(key, value) for key in ('GITHUB_RUN_ID', 'GITHUB_RUN_ATTEMPT') + for value in ('', '0', '01', '-1', '1/2', '1\n', '9' * 21, None, 1)]]: + with self.subTest(key=key, value=value): + self.assertEqual(self.exercise('cleanup', env={**ENV, key: value})[2], []) + self.assertEqual(self.exercise('other')[2], []) + + def test_start_collision_never_starts_or_removes(self): + status, _, calls = self.exercise('start', [(0, NAME)]) + self.assertEqual(status, 1) + self.assertEqual(len(calls), 1) + + def test_start_retains_existing_image_replica_port_and_verified_writable_probe(self): + status, value, calls = self.exercise('start', [(0, ''), (0, 'id'), (0, OWNER), (0, ''), (0, OWNER)]) + self.assertEqual(status, 0) + self.assertEqual(value['disposition'], 'writable') + self.assertIn('127.0.0.1:27058:27058', calls[1]) + self.assertIn(mongo.IMAGE, calls[1]) + self.assertIn(mongo.LABEL + '=' + TOKEN, calls[1]) + self.assertNotIn('foreign', calls[1]) + self.assertEqual(calls[3], ['docker', 'exec', NAME, 'mongosh', '--port', '27058', '--quiet', '--eval', mongo.INIT]) + + def test_foreign_or_stopped_owned_runtime_never_gets_initialization_probe(self): + for value in [(0, 'foreign'), (1, OWNER), (0, OWNER.replace('|true', '|false')), TIMEOUT]: + status, _, calls = self.exercise('start', [(0, ''), (0, 'id'), value]) + self.assertEqual(status, 1) + self.assertFalse(any(call[1] == 'exec' for call in calls)) + + def test_every_probe_and_final_readiness_revalidate_live_ownership(self): + status, _, calls = self.exercise('start', [(0, ''), (0, 'id'), (0, OWNER), (1, ''), + (0, OWNER), (0, ''), (0, OWNER)]) + self.assertEqual(status, 0) + self.assertEqual([call[1] for call in calls], ['container', 'run', 'inspect', 'exec', 'inspect', 'exec', 'inspect']) + self.assertEqual(self.exercise('start', [(0, ''), (0, 'id'), (0, OWNER), (0, ''), (0, 'foreign')])[0], 1) + + def test_start_commit_with_lost_reply_is_discoverable_by_separate_cleanup(self): + for value in [(1, ''), TIMEOUT]: + self.assertEqual(self.exercise('start', [(0, ''), value])[0], 1) + self.assertEqual(self.exercise('cleanup', [(0, NAME), (0, OWNER), (0, NAME), (0, '')])[0], 0) + + def test_writable_reply_after_deadline_cannot_pass(self): + self.assertEqual(self.exercise('start', [(0, ''), (0, 'id'), (0, OWNER), (0, '')], + ticks=[0, 0, 0, 61, 61, 61])[0], 1) + self.assertEqual(self.exercise('start', [(0, ''), (0, 'id')], ticks=[0, 61])[0], 1) + + +if __name__ == '__main__': + unittest.main() diff --git a/.github/native/.gitignore b/.github/native/.gitignore new file mode 100644 index 0000000000..ecb842d705 --- /dev/null +++ b/.github/native/.gitignore @@ -0,0 +1 @@ +.generated/ diff --git a/.github/native/actions_software_evidence.cjs b/.github/native/actions_software_evidence.cjs new file mode 100644 index 0000000000..dff21ff767 --- /dev/null +++ b/.github/native/actions_software_evidence.cjs @@ -0,0 +1,324 @@ +'use strict'; +// Read-only installed PR-route provenance. No dispatch, status, release or pin access. +const crypto = require('node:crypto'); +const path = require('node:path'); +const { execFileSync } = require('node:child_process'); +const { preflight, fingerprint } = require('./release_preflight.cjs'); +const { parseStrictJson } = require('./strict_json.cjs'); +const { SCOPES: COMPANION_SCOPES } = require('./public_companion_receipts.cjs'); +const REPOSITORY = 'KeeperCommunity/bitcoin-keeper', REPOSITORY_ID = 470904030; +const OWNER = 'KeeperCommunity', REPO = 'bitcoin-keeper'; +const WORKFLOW = '.github/workflows/keeper-release-acceptance.yml'; +const WORKER = '.github/workflows/keeper-native-platform.yml'; +const MAX_ARCHIVE = 1024 * 1024, MAX_MEMBER = 512 * 1024, MAX_AGE = 86400000; +const SHA = /^[a-f0-9]{40}$/, proofs = new WeakSet(); +const id = value => Number.isSafeInteger(value) && value > 0; +const digest = bytes => crypto.createHash('sha256').update(bytes).digest('hex'); +const demand = (value, code) => { if (!value) throw new Error(code); }; +const repository = value => value?.id === REPOSITORY_ID && value.full_name === REPOSITORY; +const args = { owner: OWNER, repo: REPO, request: { timeout: 30000 } }; +const PUBLIC_CHECKS = Object.freeze({ + 'native-build': 'development native build', + startup: 'ordinary app passcode startup smoke', + 'native-storage': 'CI entrypoint native Realm/MMKV process persistence', + 'production-worker-controls': 'CI-extracted production workers; synthetic boundaries are not complete application UI', + 'five-product-observations': 'five raw development full-product startup/first-wallet observations; no release-build budget', + 'controlled-setup-disconnect-retry': 'ordinary app loopback disconnect before commit; not physical WAN offline', + 'installed-version-history': 'native installed tuple and retained initial history; not authentic historical migration', + 'key-management-discoverability': 'ordinary app Keys Manage Keys entry and return', + 'unused-key-and-pin-ui': 'unused Mobile Key delete cancel wrong PIN and cold persistence; not protected backup or physical biometrics', + ...COMPANION_SCOPES, +}); + +function complete(response, key) { + const value = response?.data; + demand(response?.status === 200 && !/rel="next"/.test(response.headers?.link || '') && + Number.isSafeInteger(value?.total_count) && value.total_count > 0 && value.total_count <= 100 && + Array.isArray(value[key]) && value[key].length === value.total_count, 'SOFTWARE_API_INCOMPLETE'); + return value[key]; +} +function data(response) { + demand(response?.status === 200 && response.data && !/rel="next"/.test(response.headers?.link || ''), 'SOFTWARE_API_INVALID'); + return response.data; +} +function checkApp(value) { return value?.app?.id === 15368 && value.app.slug === 'github-actions'; } +function selectedSteps(platform) { + return ['Build development app', ...(platform === 'IOS' ? ['Run disposable startup smoke', + 'Run disposable native persistence probe', 'Run R5 production worker native scenarios', + 'Measure five full-product startup and first-wallet runs'] : ['Run Android startup and native persistence probes']), + 'Retain safe Phase A evidence']; +} +function exactPR(value, original) { + return value?.number === original.number && value.base?.ref === 'sprint' && + SHA.test(value.base.sha || '') && repository(value.base.repo) && repository(value.head?.repo) && + value.head.sha === original.head.sha && value.head.ref === original.head.ref; +} +function linkedPR(value, original) { + // Actions run associations use a reduced repository shape (id/url/name), + // unlike pulls.get. Global repository IDs bind to the separately verified + // complete repository and current PR; full_name is not present here. + return value?.number === original.number && value.head?.sha === original.head.sha && + value.head.ref === original.head.ref && value.head.repo?.id === REPOSITORY_ID && + value.base?.ref === 'sprint' && value.base.sha === original.base.sha && value.base.repo?.id === REPOSITORY_ID; +} +function publicRequest(context, input) { + const pr = context.payload?.pull_request; + demand(context.eventName === 'pull_request' && context.repo?.owner === OWNER && context.repo.repo === REPO && + id(pr?.number) && pr.number !== 7045 && pr.draft === false && exactPR(pr, pr) && SHA.test(pr.head.sha || '') && + !pr.labels?.some(label => ['keeper/native-diagnostic', 'keeper/legacy-native'].includes(label.name)), 'SOFTWARE_PUBLIC_ROUTE_REQUIRED'); + demand(input?.ios === 'success' && input.android === 'success' && id(context.runId) && id(input.attempt), 'SOFTWARE_EXACT_ATTEMPT_REQUIRED'); + const mergeRef = `refs/pull/${pr.number}/merge`; + demand(SHA.test(input.workflowSha || '') && input.workflowRef === `${REPOSITORY}/${WORKFLOW}@${mergeRef}`, + 'SOFTWARE_WORKFLOW_CONTROL_REQUIRED'); + return { sourceSha: pr.head.sha, workflowSha: input.workflowSha, workflowRef: input.workflowRef, + runId: context.runId, attempt: input.attempt, mergeRef, pr }; +} +function fresh(run, now) { + const updated = Date.parse(run.updated_at); + demand(Number.isFinite(now) && Number.isFinite(updated) && updated <= now && updated >= now - MAX_AGE, + 'SOFTWARE_RUN_STALE'); +} + +async function origin(github, context, request, now) { + const [repo, pr, branch, workflow] = await Promise.all([ + github.rest.repos.get(args).then(data), + github.rest.pulls.get({ ...args, pull_number: request.pr.number }).then(data), + github.rest.repos.getBranch({ ...args, branch: 'sprint' }).then(data), + github.rest.actions.getWorkflow({ ...args, workflow_id: 'keeper-release-acceptance.yml' }).then(data), + ]); + demand(repository(repo) && repo.default_branch === 'sprint' && exactPR(pr, request.pr) && pr.draft === false && + !pr.labels?.some(label => ['keeper/native-diagnostic', 'keeper/legacy-native'].includes(label.name)) && + branch.protected === true && branch.name === 'sprint' && branch.commit?.sha === pr.base.sha, + 'SOFTWARE_PR_SOURCE_NOT_CURRENT'); + // Keep the existing app/current-base/original-run-base installation boundary. + demand((await preflight(github, context)).trusted === true, 'SOFTWARE_REVIEWED_INSTALLATION_REQUIRED'); + demand(id(workflow.id) && workflow.path === WORKFLOW && workflow.state === 'active', 'SOFTWARE_WORKFLOW_NOT_INSTALLED'); + const runs = complete(await github.rest.actions.listWorkflowRuns({ ...args, workflow_id: workflow.id, + head_sha: request.sourceSha, branch: request.pr.head.ref, event: 'pull_request', per_page: 100 }), 'workflow_runs'); + demand(runs.every(run => id(run.id)) && Math.max(...runs.map(run => run.id)) === request.runId, 'SOFTWARE_LATEST_RUN_REQUIRED'); + const run = data(await github.rest.actions.getWorkflowRun({ ...args, run_id: request.runId })); + demand(run.id === request.runId && run.run_attempt === request.attempt && run.workflow_id === workflow.id && + run.path === WORKFLOW && run.event === 'pull_request' && run.head_sha === request.sourceSha && + run.head_branch === request.pr.head.ref && repository(run.repository) && repository(run.head_repository) && id(run.check_suite_id) && + (run.status === 'in_progress' || (run.status === 'completed' && run.conclusion === 'success')) && + runs.some(value => value.id === run.id && value.run_attempt === run.run_attempt && value.head_sha === run.head_sha), + 'SOFTWARE_RUN_NOT_EXACT_CURRENT'); + const linked = run.pull_requests?.filter(value => value.number === request.pr.number) || []; + demand(linked.length === 1 && linkedPR(linked[0], request.pr), + 'SOFTWARE_RUN_PR_BINDING_INVALID'); + // A local reusable workflow is loaded from the caller's merge control commit, + // not from the separately checked-out app head. Neither receipt selects this SHA. + demand(Array.isArray(run.referenced_workflows) && run.referenced_workflows.length === 1 && + run.referenced_workflows[0].path === `${REPOSITORY}/${WORKER}@${request.workflowSha}` && + run.referenced_workflows[0].ref === request.mergeRef && run.referenced_workflows[0].sha === request.workflowSha, + 'SOFTWARE_REUSABLE_CONTROL_BINDING_INVALID'); + fresh(run, now); + const [app, control, sourceTree, controlTree, suite] = await Promise.all([ + github.rest.git.getCommit({ ...args, commit_sha: request.sourceSha }).then(data), + github.rest.git.getCommit({ ...args, commit_sha: request.workflowSha }).then(data), + github.rest.git.getTree({ ...args, tree_sha: request.sourceSha, recursive: '1' }).then(data), + github.rest.git.getTree({ ...args, tree_sha: request.workflowSha, recursive: '1' }).then(data), + github.rest.checks.getSuite({ ...args, check_suite_id: run.check_suite_id }).then(data), + ]); + demand(app.sha === request.sourceSha && SHA.test(app.tree?.sha || '') && control.sha === request.workflowSha && + SHA.test(control.tree?.sha || '') && sourceTree.sha === app.tree.sha && controlTree.sha === control.tree.sha && + fingerprint(sourceTree) === fingerprint(controlTree), 'SOFTWARE_CONTROL_TREE_INVALID'); + demand(suite.id === run.check_suite_id && suite.head_sha === request.sourceSha && checkApp(suite) && + ['in_progress', 'completed'].includes(suite.status), 'SOFTWARE_CHECK_SUITE_INVALID'); + return { pr, workflow, run, appTree: app.tree.sha, controlTree: control.tree.sha }; +} + +async function platformJobs(github, request, run) { + const jobs = complete(await github.rest.actions.listJobsForWorkflowRunAttempt({ ...args, run_id: request.runId, + attempt_number: request.attempt, per_page: 100 }), 'jobs'); + const selected = {}; + for (const platform of ['IOS', 'ANDROID']) { + const name = `${platform.toLowerCase()} / Native ${platform} build and probes`; + const matches = jobs.filter(job => job.name === name); + demand(matches.length === 1, 'SOFTWARE_PLATFORM_JOB_MISSING'); + const job = matches[0]; + demand(id(job.id) && job.run_id === request.runId && job.run_attempt === request.attempt && job.head_sha === request.sourceSha && + job.status === 'completed' && job.conclusion === 'success', 'SOFTWARE_PLATFORM_JOB_INVALID'); + for (const name of selectedSteps(platform)) { + const steps = job.steps?.filter(step => step.name === name) || []; + demand(steps.length === 1 && steps[0].status === 'completed' && steps[0].conclusion === 'success', 'SOFTWARE_PLATFORM_STEP_INVALID'); + } + const match = job.check_run_url?.match(/^https:\/\/api\.github\.com\/repos\/KeeperCommunity\/bitcoin-keeper\/check-runs\/([1-9]\d*)$/); + demand(match && id(Number(match[1])), 'SOFTWARE_CHECK_BINDING_INVALID'); + const check = data(await github.rest.checks.get({ ...args, check_run_id: Number(match[1]) })); + demand(check.id === Number(match[1]) && check.name === job.name && check.check_suite?.id === run.check_suite_id && + check.head_sha === request.sourceSha && check.status === 'completed' && check.conclusion === 'success' && checkApp(check), + 'SOFTWARE_CHECK_ORIGIN_INVALID'); + selected[platform] = { job_id: job.id, check_run_id: check.id }; + } + return selected; +} + +async function boundedBytes(response) { + let reader; + try { + demand(response?.ok === true && response.body?.getReader, 'SOFTWARE_ARCHIVE_READ_FAILED'); + const length = response.headers.get('content-length'); + demand(length === null || (/^\d+$/.test(length) && Number(length) <= MAX_ARCHIVE), 'SOFTWARE_ARCHIVE_TOO_LARGE'); + reader = response.body.getReader(); + const chunks = []; + let size = 0; + for (;;) { + const { done, value } = await reader.read(); + if (done) break; + size += value.length; + demand(size <= MAX_ARCHIVE, 'SOFTWARE_ARCHIVE_TOO_LARGE'); + chunks.push(Buffer.from(value)); + } + demand(length === null || Number(length) === size, 'SOFTWARE_ARCHIVE_SIZE_MISMATCH'); + return Buffer.concat(chunks); + } finally { + // Header/status rejection happens before a reader exists. Close that body + // too; a cancellation failure must prevent an apparently successful read. + if (reader) await reader.cancel(); + else if (response?.body?.cancel) await response.body.cancel(); + } +} +async function readArchive(github, artifactId, fetchImpl) { + demand(id(artifactId), 'SOFTWARE_ARTIFACT_IDENTITY_INVALID'); + const apiURL = `https://api.github.com/repos/${REPOSITORY}/actions/artifacts/${artifactId}/zip`; + let intercepted = false; + const response = await github.rest.actions.downloadArtifact({ ...args, artifact_id: artifactId, archive_format: 'zip', + request: { timeout: 30000, parseSuccessResponseBody: false, + // The pinned github-script client does not forward request.redirect. + // Its supported fetch hook controls the authenticated API hop instead. + fetch: async (url, init) => { + demand(!intercepted && url === apiURL && init?.method === 'GET' && init.body == null, + 'SOFTWARE_ARCHIVE_API_REQUEST_INVALID'); + intercepted = true; + const apiResponse = await fetchImpl(url, { ...init, redirect: 'manual', signal: AbortSignal.timeout(30000) }); + try { + demand(apiResponse?.status === 302, 'SOFTWARE_ARCHIVE_REDIRECT_REQUIRED'); + return apiResponse; + } finally { + // No SDK JSON/text/arrayBuffer parser may consume an API error or + // accidentally followed ZIP body before our streaming size checks. + if (apiResponse?.body) await apiResponse.body.cancel(); + } + } } }); + demand(intercepted && response.status === 302, 'SOFTWARE_ARCHIVE_REDIRECT_REQUIRED'); + const location = new URL(response.headers.location); + demand(location.protocol === 'https:' && !location.username && !location.password && !location.hash && + (!location.port || location.port === '443') && + (location.hostname.endsWith('.blob.core.windows.net') || location.hostname.endsWith('.actions.githubusercontent.com')), + 'SOFTWARE_ARCHIVE_HOST_INVALID'); + // No GitHub token or API headers reach the signed storage endpoint. + return boundedBytes(await fetchImpl(location, { redirect: 'error', signal: AbortSignal.timeout(30000) })); +} +// Transport only: returns original untrusted ZIP bytes, never an issued proof. +// A separately reviewed fixed-artifact host probe can exercise this exact adapter. +async function readPublicArtifactBytes(github, artifactId, { fetchImpl = fetch } = {}) { + try { return await readArchive(github, artifactId, fetchImpl); } + catch (error) { + throw new Error(/^SOFTWARE_[A-Z0-9_]{1,80}$/.test(error?.message || '') ? error.message : 'SOFTWARE_ARCHIVE_READ_FAILED'); + } +} +function unpack(archive, platform) { + const result = execFileSync('python3', ['-B', path.join(__dirname, 'software_artifact_zip.py'), platform], { + input: archive, timeout: 10000, maxBuffer: MAX_MEMBER * 2, stdio: ['pipe', 'pipe', 'pipe'], + // Only the parser's executable search path is needed; no runner credentials. + env: { PATH: process.env.PATH, PYTHONDONTWRITEBYTECODE: '1', PYTHONNOUSERSITE: '1' }, + }); + const members = JSON.parse(result.toString('utf8')), name = `${platform.toLowerCase()}/metadata.json`; + demand(members && Object.keys(members).length === 1 && typeof members[name] === 'string', 'SOFTWARE_ARCHIVE_MEMBERS_INVALID'); + const bytes = Buffer.from(members[name], 'base64'); + demand(bytes.length > 0 && bytes.length <= MAX_MEMBER && bytes.toString('base64') === members[name], 'SOFTWARE_ARCHIVE_MEMBERS_INVALID'); + return bytes; +} +function artifactIdentity(info, name, request, now) { + demand(id(info?.id) && info.name === name && info.expired === false && id(info.size_in_bytes) && info.size_in_bytes <= MAX_ARCHIVE && + /^sha256:[a-f0-9]{64}$/.test(info.digest || '') && Date.parse(info.expires_at) > now && + info.workflow_run?.id === request.runId && info.workflow_run.head_sha === request.sourceSha && + info.workflow_run.repository_id === REPOSITORY_ID && info.workflow_run.head_repository_id === REPOSITORY_ID, + 'SOFTWARE_ARTIFACT_IDENTITY_INVALID'); +} +async function workerSources(github, sourceSha, expected) { + return Promise.all(Object.entries(expected).map(async ([name, names]) => { + const file = data(await github.rest.repos.getContent({ ...args, path: name, ref: sourceSha })); + demand(file.path === name && file.type === 'file' && file.encoding === 'base64' && SHA.test(file.sha || '') && + id(file.size) && file.size <= MAX_MEMBER && typeof file.content === 'string' && /^[A-Za-z0-9+/=\r\n]+$/.test(file.content), + 'SOFTWARE_COMMITTED_WORKER_UNAVAILABLE'); + const bytes = Buffer.from(file.content, 'base64'); + const blob = crypto.createHash('sha1').update(`blob ${bytes.length}\0`).update(bytes).digest('hex'); + demand(bytes.length === file.size && bytes.toString('base64') === file.content.replace(/[\r\n]/g, '') && blob === file.sha, + 'SOFTWARE_COMMITTED_WORKER_UNAVAILABLE'); + return { path: name, sha256: digest(bytes), names: [...names] }; + })); +} + +async function collectPublicSoftwareEvidence(github, context, input, { clock = Date.now, fetchImpl = fetch } = {}) { + try { + const request = publicRequest(context, input); + const first = await origin(github, context, request, clock()); + const jobs = await platformJobs(github, request, first.run); + // Import the reviewed P1 validator only after source and installed route checks. + const { validatePublicSoftware, WORKER_SOURCES } = require('./public_software_receipts.cjs'); + const sources = await workerSources(github, request.sourceSha, WORKER_SOURCES); + const artifacts = complete(await github.rest.actions.listWorkflowRunArtifacts({ ...args, run_id: request.runId, per_page: 100 }), 'artifacts'); + const platforms = {}, identities = []; + for (const platform of ['IOS', 'ANDROID']) { + const name = `keeper-release-native-${platform}-${request.sourceSha}-${request.runId}-${request.attempt}`; + const matches = artifacts.filter(artifact => artifact.name === name); + demand(matches.length === 1 && id(matches[0].id), 'SOFTWARE_ARTIFACT_MISSING_OR_AMBIGUOUS'); + const info = data(await github.rest.actions.getArtifact({ ...args, artifact_id: matches[0].id })); + demand(info.id === matches[0].id, 'SOFTWARE_ARTIFACT_CHANGED'); + artifactIdentity(info, name, request, clock()); + const archive = await readPublicArtifactBytes(github, info.id, { fetchImpl }); + demand(archive.length === info.size_in_bytes && `sha256:${digest(archive)}` === info.digest, 'SOFTWARE_ARTIFACT_DIGEST_MISMATCH'); + const bytes = unpack(archive, platform); + const metadata = parseStrictJson(bytes, MAX_MEMBER); + const validated = validatePublicSoftware(metadata, { platform, sourceSha: request.sourceSha, workflowSha: request.workflowSha, + workflowRef: request.workflowRef, runId: request.runId, runAttempt: request.attempt, workerSources: sources }); + platforms[platform] = Object.freeze({ ...jobs[platform], artifact_id: info.id, archive_sha256: digest(archive), + receipt_sha256: digest(bytes), installed_artifact: validated.installed_artifact, + executed_checks: Object.freeze(Object.fromEntries(Object.entries(PUBLIC_CHECKS).map(([id, scope]) => + [id, Object.freeze({ status: 'PASS', scope })]))) }); + identities.push(info); + } + // A partial rerun, later pending/failing run, source/base movement, expired + // artifact or changed check ownership invalidates collection after download. + const final = await origin(github, context, request, clock()); + demand(first.pr.base.sha === final.pr.base.sha && first.appTree === final.appTree && first.controlTree === final.controlTree && + first.workflow.id === final.workflow.id && first.run.check_suite_id === final.run.check_suite_id, + 'SOFTWARE_ORIGIN_CHANGED_DURING_COLLECTION'); + demand(JSON.stringify(await platformJobs(github, request, final.run)) === JSON.stringify(jobs), 'SOFTWARE_JOBS_CHANGED_DURING_COLLECTION'); + const finalArtifacts = complete(await github.rest.actions.listWorkflowRunArtifacts({ ...args, run_id: request.runId, per_page: 100 }), 'artifacts'); + for (const info of identities) { + demand(finalArtifacts.filter(value => value.name === info.name).length === 1 && + finalArtifacts.some(value => value.name === info.name && value.id === info.id), 'SOFTWARE_ARTIFACT_CHANGED'); + const current = data(await github.rest.actions.getArtifact({ ...args, artifact_id: info.id })); + artifactIdentity(current, info.name, request, clock()); + demand(current.id === info.id && current.digest === info.digest && current.size_in_bytes === info.size_in_bytes, + 'SOFTWARE_ARTIFACT_CHANGED'); + } + // Artifact readbacks are network work too. Recheck latest source/attempt + // after them, rather than treating the earlier run snapshot as current. + const last = await origin(github, context, request, clock()); + demand(last.pr.base.sha === first.pr.base.sha && last.appTree === first.appTree && last.controlTree === first.controlTree && + last.workflow.id === first.workflow.id && last.run.check_suite_id === first.run.check_suite_id, + 'SOFTWARE_ORIGIN_CHANGED_DURING_COLLECTION'); + for (const info of identities) artifactIdentity(info, info.name, request, clock()); + fresh(last.run, clock()); + const proof = Object.freeze({ schema: 1, kind: 'keeper-public-software-evidence', selected_route: 'public-reusable', + repository: REPOSITORY, source_sha: request.sourceSha, source_tree: first.appTree, + workflow_sha: request.workflowSha, workflow_ref: request.workflowRef, control_tree: first.controlTree, + run_id: request.runId, run_attempt: request.attempt, platforms: Object.freeze(platforms), software_complete: false, + private_integration: 'NOT VERIFIED: frozen C2/D contract and separately authenticated private route required', + installed_trust: 'UNVERIFIED', release_ready: false, authorization_eligible: false, release_authorized: false, execution_allowed: false }); + proofs.add(proof); + return proof; + } catch (error) { + throw new Error(/^SOFTWARE_[A-Z0-9_]{1,80}$/.test(error?.message || '') ? error.message : 'SOFTWARE_EVIDENCE_INVALID'); + } +} +function collectPrivateSoftwareEvidence() { + // No WIP schema, caller manifest or private workflow name supplies trust. + throw new Error('SOFTWARE_PRIVATE_CONTRACT_NOT_FROZEN'); +} +module.exports = { collectPublicSoftwareEvidence, collectPrivateSoftwareEvidence, + isPublicSoftwareEvidence: value => proofs.has(value), readPublicArtifactBytes, PUBLIC_CHECKS }; diff --git a/.github/native/android_startup_diagnosis.py b/.github/native/android_startup_diagnosis.py new file mode 100644 index 0000000000..8f9a30b375 --- /dev/null +++ b/.github/native/android_startup_diagnosis.py @@ -0,0 +1,328 @@ +"""Read-only Android startup facts. No raw device or application text escapes.""" +from pathlib import Path +import importlib.util +import re +import shutil +import subprocess + +APP = 'io.hexawallet.keeper.development' +LIMIT = 256 * 1024 +TYPES = ('TypeError', 'SyntaxError', 'ReferenceError', 'RangeError', 'EvalError', + 'URIError', 'AggregateError', 'TransformError', 'UnableToResolveError', + 'InternalError', 'RuntimeException', 'IllegalStateException', + 'IllegalArgumentException', 'NullPointerException', 'SecurityException', + 'UnsatisfiedLinkError', 'NoClassDefFoundError', 'ClassNotFoundException', + 'OutOfMemoryError', 'Error') +FAULTS = { + 'JS_TYPE_ERROR': r'\bTypeError\b|undefined is not a function', + 'JS_SYNTAX_ERROR': r'\bSyntaxError\b', + 'JS_REFERENCE_ERROR': r'\bReferenceError\b', + 'JS_RANGE_ERROR': r'\bRangeError\b', + 'MODULE_RESOLUTION_ERROR': r'Unable to resolve module|\bUnableToResolveError\b', + 'TRANSFORM_ERROR': r'\bTransformError\b', + 'APP_NOT_REGISTERED': r'has not been registered', + 'NATIVE_MODULE_UNAVAILABLE': r'TurboModuleRegistry\.getEnforcing|requireNativeComponent|native module cannot be null', + 'NATIVE_LIBRARY_LOAD_ERROR': r'\bUnsatisfiedLinkError\b|dlopen failed|could not load library', + 'NATIVE_FATAL_SIGNAL': r'Fatal signal \d+\b', + 'JAVA_FATAL_EXCEPTION': r'FATAL EXCEPTION:', + 'MEMORY_EXHAUSTION': r'\bOutOfMemoryError\b|out of memory', + 'TRANSPORT_ERROR': r'\b(?:EADDRINUSE|ECONNREFUSED|ECONNRESET)\b', + 'FILESYSTEM_ERROR': r'\b(?:ENOENT|EACCES|ENOSPC)\b', + 'WORKER_HEAP_EXHAUSTION': r'JavaScript heap out of memory|Reached heap limit|Ineffective mark-compacts', + 'PROCESS_KILLED_MARKER': r'\bSIGKILL\b|worker process.*terminated|worker.*killed', +} +EXIT_REASONS = {0: 'UNKNOWN', 1: 'EXIT_SELF', 2: 'SIGNALED', 3: 'LOW_MEMORY', + 4: 'JAVA_CRASH', 5: 'NATIVE_CRASH', 6: 'ANR', 7: 'INITIALIZATION_FAILURE', + 8: 'PERMISSION_CHANGE', 9: 'EXCESSIVE_RESOURCE_USAGE', + 10: 'USER_REQUESTED', 11: 'USER_STOPPED', 12: 'DEPENDENCY_DIED', + 13: 'OTHER', 14: 'FREEZER', 15: 'PACKAGE_STATE_CHANGE', 16: 'PACKAGE_UPDATED'} + + +def safe_error(error): + name = type(error).__name__ + return name if name in {'OSError', 'FileNotFoundError', 'PermissionError', + 'TimeoutExpired', 'ValueError', 'TypeError'} else 'OTHER_ERROR' + + +def bounded(path, private): + if path.is_symlink() or not path.resolve().is_relative_to(private.resolve()): + raise ValueError('Expected a private regular file.') + size = path.stat().st_size + with path.open('rb') as source: + prefix = source.read(LIMIT) + tail = b'' + if size > LIMIT: + source.seek(max(LIMIT, size - LIMIT)) + tail = source.read(LIMIT) + truncated = size > len(prefix) + len(tail) + if truncated: + prefix = prefix.rsplit(b'\n', 1)[0] if b'\n' in prefix else b'' + tail = tail.split(b'\n', 1)[1] if b'\n' in tail else b'' + return (prefix + (b'\n' if truncated else b'') + tail).decode(errors='replace'), { + 'bytes_observed': size, 'bytes_read': len(prefix) + len(tail), + 'input_truncated': truncated, 'read_scope': 'PREFIX_AND_TAIL' if truncated else 'FULL'} + + +def faults(payload): + return {'exception_types': [kind for kind in TYPES if re.search(r'\b' + kind + r'\b', payload)], + 'fault_categories': [name for name, pattern in FAULTS.items() + if re.search(pattern, payload, re.IGNORECASE)]} + + +def processes(payload, usable=True): + lines = [line.strip() for line in payload.splitlines() if line.strip()] + result = {'state': 'UNKNOWN', 'matching_process_count': None, 'parsed_records': 0, + 'unparsed_records': 0} + owned = set() + seen = set() + if not usable or not lines or lines[0].split() != ['PID', 'NAME']: + return result, owned + for line in lines[1:]: + match = re.fullmatch(r'([1-9]\d*)\s+(\S+)', line) + if not match or int(match[1]) in seen: + result['unparsed_records'] += 1 + continue + seen.add(int(match[1])) + result['parsed_records'] += 1 + if match[2] == APP or match[2].startswith(APP + ':'): + owned.add(int(match[1])) + if result['unparsed_records'] == 0 and result['parsed_records'] > 0: + result.update(state='PRESENT' if owned else 'ABSENT', matching_process_count=len(owned)) + else: + owned.clear() # Malformed inventory cannot establish process attribution. + return result, owned + + +def foreground(payload, usable=True): + relations = set() + parsed, unparsed = 0, 0 + if usable: + for line in payload.splitlines(): + if not re.search(r'\b(?:topResumedActivity|mResumedActivity)\s*[:=]', line): + continue + if re.search(r'[:=]\s*(?:null|None)\s*$', line): + relations.add('NONE') + parsed += 1 + elif match := re.search(r'ActivityRecord\{.*?\bu\d+\s+([A-Za-z0-9_.]+)/(?:[^\s}]+)', line): + relations.add('OWN_APP' if match[1] == APP else 'OTHER_APP') + parsed += 1 + else: + unparsed += 1 + return {'relation': next(iter(relations)) if len(relations) == 1 and not unparsed else 'UNKNOWN', + 'parsed_records': parsed, 'unparsed_records': unparsed, + 'scope': 'RESUMED_ACTIVITY_ONLY_NOT_WINDOW_VISIBILITY'} + + +def exit_history(payload, usable=True): + reasons = [] + matched = 0 + if usable: + for line in payload.splitlines(): + match = re.search(r'\bprocess=(\S+)\s+reason=(\d+)\s', line) + if match and (match[1] == APP or match[1].startswith(APP + ':')): + matched += 1 + reason = EXIT_REASONS.get(int(match[2]), 'UNRECOGNIZED_REASON') + if reason not in reasons: + reasons.append(reason) + return {'matching_records': matched, 'reason_categories': reasons, + 'parse_status': 'PARSED' if matched else 'UNKNOWN', + 'freshness': 'UNKNOWN', 'scope': 'APP_HISTORY_NOT_PROOF_OF_CURRENT_FAILURE_CAUSE'} + + +def android_log(payload, owned): + result = {'parsed_error_records': 0, 'own_error_records': 0, 'unclassified_own_error_records': 0, + 'diagnostic_marker_records': 0, + 'unparsed_nonempty_lines': 0, 'exception_types': [], 'fault_categories': [], + 'attribution': 'CURRENT_OWN_PID_MATCH_HISTORY_FRESHNESS_UNKNOWN' if owned + else 'UNKNOWN_NO_CURRENT_OWN_PROCESS'} + pattern = r'^\d{2}-\d{2} \d{2}:\d{2}:\d{2}\.\d+\s+(\d+)\s+\d+\s+[EF]\s+[^:]+:\s*(.*)$' + for line in payload.splitlines(): + # Reserved development diagnostics are not runtime error records. Keep + # this message-start filter independent of the optional receipt parser; + # an unavailable observer cannot turn its own markers into app faults. + if re.fullmatch(r'[0-9]{2}-[0-9]{2} [0-9]{2}:[0-9]{2}:[0-9]{2}\.[0-9]{3}' + r' +[1-9][0-9]{0,9} +[1-9][0-9]{0,9} +E +ReactNativeJS: ' + r'KEEPER_(?:ONBOARDING|STARTUP)_BOUNDARY\|[^\n]*', line): + result['diagnostic_marker_records'] += 1 + continue + match = re.match(pattern, line) + if not match: + if line.strip() and not line.startswith('--------- beginning of '): + result['unparsed_nonempty_lines'] += 1 + continue + result['parsed_error_records'] += 1 + if int(match[1]) not in owned: + continue + result['own_error_records'] += 1 + observed = faults(match[2]) + if not any(observed.values()): + result['unclassified_own_error_records'] += 1 + for key in observed: + result[key] = list(dict.fromkeys(result[key] + observed[key])) + return result + + +def onboarding_receipt(payload, owned, *, usable, full_input): + try: + spec = importlib.util.spec_from_file_location('onboarding_failure_receipts', + Path(__file__).with_name('onboarding_failure_receipts.py')) + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module.parse(payload, owned, usable=usable, full_input=full_input) + except Exception: + return {'state': 'UNKNOWN', 'scope': 'OPTIONAL_DEV_MARKER_PARSER_UNAVAILABLE', + 'cause': 'UNDETERMINED', 'freshness': 'GUEST_CLOCK_UNVERIFIED'} + + +def startup_boundary_receipt(payload, owned, *, usable, full_input): + try: + spec = importlib.util.spec_from_file_location('startup_boundary_receipts', + Path(__file__).with_name('startup_boundary_receipts.py')) + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module.parse(payload, owned, usable=usable, full_input=full_input) + except Exception: + return {'state': 'UNKNOWN', 'scope': 'OPTIONAL_DEV_MARKER_PARSER_UNAVAILABLE', + 'cause': 'UNDETERMINED', 'freshness': 'GUEST_CLOCK_UNVERIFIED'} + + +def metro(payload): + selected = [] + headers, remaining = 0, 0 + for raw in payload.splitlines(): + line = re.sub(r'\x1b\[[0-9;]*[A-Za-z]', '', raw).strip() + if re.match(r'^(?:ERROR\b|FATAL ERROR:|(?:\w*Error):|Unable to resolve module)', line, re.IGNORECASE): + headers += 1 + remaining = 4 + elif remaining: + remaining -= 1 + else: + continue + if re.match(r'^(?:(?:>\s*)?\d+\s*\||\^|at\s)', line): + continue + selected.append(line) + return {'observed_error_headers': headers, 'scanned_error_context_lines': len(selected), + 'unclassified_error_context_lines': sum(not faults(line)['fault_categories'] for line in selected), + **faults('\n'.join(selected)), 'scope': 'ERROR_MARKERS_NOT_BUNDLE_OR_STARTUP_ACCEPTANCE'} + + +def host_memory(root=Path('/')): + result = {'scope': 'HOST_SNAPSHOT_AND_HISTORICAL_CGROUP_EVENTS_NOT_FAILURE_CAUSE', + 'meminfo': {'lookup_status': 'UNKNOWN'}, 'cgroup': {}} + + def small(path): + with path.open('rb') as source: + value = source.read(8193) + if len(value) > 8192: + raise ValueError('Memory record exceeded size limit.') + return value.decode('ascii').strip() + + try: + payload = small(root / 'proc/meminfo') + matches = re.findall(r'^(MemTotal|MemAvailable|SwapTotal|SwapFree):\s+(\d+) kB$', payload, re.MULTILINE) + fields = dict(matches) + recognized = sum(bool(re.match(r'^(?:MemTotal|MemAvailable|SwapTotal|SwapFree)\b', line)) + for line in payload.splitlines()) + if len(fields) == len(matches) == recognized == 4: + result['meminfo'] = {'lookup_status': 'SUCCESS', + 'bytes': {name: int(value) * 1024 for name, value in fields.items()}} + except Exception as error: + result['meminfo']['error_type'] = safe_error(error) + for name in ['memory.current', 'memory.max', 'memory.events']: + observed = {'lookup_status': 'UNKNOWN'} + try: + value = small(root / 'sys/fs/cgroup' / name) + if name == 'memory.events': + matches = re.findall(r'^(oom|oom_kill) (\d+)$', value, re.MULTILINE) + events = dict(matches) + recognized = sum(bool(re.match(r'^(?:oom|oom_kill)\b', line)) for line in value.splitlines()) + if len(events) == len(matches) == recognized == 2: + observed = {'lookup_status': 'SUCCESS', 'counts': {k: int(v) for k, v in events.items()}} + elif value.isascii() and value.isdecimal(): + observed = {'lookup_status': 'SUCCESS', 'bytes': int(value)} + elif name == 'memory.max' and value == 'max': + observed = {'lookup_status': 'SUCCESS', 'limit': 'UNLIMITED'} + except Exception as error: + observed['error_type'] = safe_error(error) + result['cgroup'][name] = observed + return result + + +def collect(private, device): + private = Path(private) + if not re.fullmatch(r'emulator-\d+', device): + raise ValueError('Expected the owned emulator selector.') + work = private / 'android-startup-diagnosis' + work.mkdir(mode=0o700) + result = {'diagnostic_only': True, 'cause': 'UNDETERMINED', 'host_memory': host_memory(), 'commands': [], + 'collection_errors': [], 'cleanup_status': 'UNKNOWN'} + + def capture(label, arguments): + path = work / (label + '.log') + facts = {'label': label, 'lookup_status': 'UNKNOWN', 'exit_code': None, + 'bytes_observed': None, 'bytes_read': 0, 'input_truncated': None, 'read_scope': 'NOT_READ'} + payload = '' + try: + with path.open('xb') as output: + proc = subprocess.run(['adb', '-s', device, *arguments], stdout=output, + stderr=subprocess.STDOUT, timeout=15, check=False) + facts['exit_code'] = proc.returncode + payload, observed = bounded(path, private) + facts.update(observed) + if proc.returncode == 0: + facts['lookup_status'] = 'SUCCESS' + else: + payload = '' + except Exception as error: + facts['error_type'] = safe_error(error) + payload = '' + finally: + path.unlink(missing_ok=True) + result['commands'].append(facts) + return payload, facts.get('lookup_status') == 'SUCCESS' and not facts.get('input_truncated', True) + + try: + payload, usable = capture('processes', ['shell', 'ps', '-A', '-o', 'PID,NAME']) + result['processes'], owned = processes(payload, usable) + payload, usable = capture('foreground', ['shell', 'dumpsys', 'activity', 'activities']) + result['foreground_activity'] = foreground(payload, usable) + payload, usable = capture('exit-history', ['shell', 'dumpsys', 'activity', 'exit-info', APP]) + result['exit_history'] = exit_history(payload, usable) + payload, _ = capture('runtime-errors', ['logcat', '-d', '-b', 'main', '-b', 'crash', + '-t', '1200', '-v', 'threadtime', '*:E']) + result['runtime_errors'] = android_log(payload, owned) + result['runtime_errors'].update( + lookup_status=result['commands'][-1]['lookup_status'], + input_truncated=result['commands'][-1]['input_truncated']) + # logcat applies -t to the raw ring before priority/tag filtering. An + # earlier fixed marker can fall outside that tail despite few E rows. + # Keep runtime errors unchanged; query only the reserved marker prefix + # separately. -d/15s timeout, 65 matching entries and bounded() limit + # collection. The 65th entry makes the existing 64-marker parser report + # PARTIAL. Ownership/freshness/positive-only scope remains unchanged. + marker_payload, _ = capture('onboarding-markers', [ + 'logcat', '-d', '-b', 'main', '-v', 'threadtime', + '--regex', r'^KEEPER_ONBOARDING_BOUNDARY\|', '--max-count', '65', + 'ReactNativeJS:E', '*:S']) + result['onboarding_boundary_observation'] = onboarding_receipt(marker_payload, owned, + usable=result['commands'][-1]['lookup_status'] == 'SUCCESS', + full_input=result['commands'][-1]['input_truncated'] is False) + startup_payload, _ = capture('startup-markers', [ + 'logcat', '-d', '-b', 'main', '-v', 'threadtime', + '--regex', r'^KEEPER_STARTUP_BOUNDARY\|', '--max-count', '41', + 'ReactNativeJS:V', '*:S']) + result['startup_boundary_observation'] = startup_boundary_receipt(startup_payload, owned, + usable=result['commands'][-1]['lookup_status'] == 'SUCCESS', + full_input=result['commands'][-1]['input_truncated'] is False) + try: + payload, observed = bounded(private / 'metro.log', private) + result['metro'] = {'lookup_status': 'SUCCESS', **observed, **metro(payload)} + except Exception as error: + result['collection_errors'].append({'stage': 'metro', 'error_type': safe_error(error)}) + finally: + try: + shutil.rmtree(work) + result['cleanup_status'] = 'DELETED' + except Exception as error: + result['collection_errors'].append({'stage': 'cleanup', 'error_type': safe_error(error)}) + return result diff --git a/.github/native/canonical_metadata.py b/.github/native/canonical_metadata.py new file mode 100644 index 0000000000..c61c143b14 --- /dev/null +++ b/.github/native/canonical_metadata.py @@ -0,0 +1,35 @@ +#!/usr/bin/env python3 +"""Read one bounded metadata.json member without extracting paths or echoing errors.""" +import io +import stat +import sys +import zipfile + + +def read_metadata(raw): + if len(raw) > 1048576: + raise ValueError('archive size') + with zipfile.ZipFile(io.BytesIO(raw)) as archive: + members = archive.infolist() + if len(members) != 1: + raise ValueError('single metadata file required') + member = members[0] + mode = member.external_attr >> 16 + if (member.filename != 'metadata.json' or member.flag_bits & 1 or + stat.S_ISLNK(mode) or not 0 < member.file_size <= 524288): + raise ValueError('metadata member rejected') + with archive.open(member) as source: + result = source.read(524289) + if len(result) != member.file_size: + raise ValueError('metadata size mismatch') + return result + + +if __name__ == '__main__': + try: + with open(sys.argv[1], 'rb') as source: + result = read_metadata(source.read(1048577)) + sys.stdout.buffer.write(result) + except Exception: + sys.stderr.write('Canonical metadata archive rejected.\n') + raise SystemExit(1) diff --git a/.github/native/canonical_native.cjs b/.github/native/canonical_native.cjs new file mode 100644 index 0000000000..bd4102817b --- /dev/null +++ b/.github/native/canonical_native.cjs @@ -0,0 +1,354 @@ +'use strict'; +// Read-only canonical provenance/receipts. Never signs, posts statuses or approves release. +const crypto = require('node:crypto'); +const fs = require('node:fs'); +const os = require('node:os'); +const path = require('node:path'); +const { execFileSync } = require('node:child_process'); +const { REQUIRED, acceptance } = require('./release_acceptance.cjs'); +const { publicCompanionChecks } = require('./public_companion_receipts.cjs'); +const { validatePublicSoftware, WORKER_SOURCES, PUBLIC_SOFTWARE_SCOPE } = require('./public_software_receipts.cjs'); +const { parseStrictJson } = require('./strict_json.cjs'); +const REPOSITORY = 'KeeperCommunity/bitcoin-keeper'; +const REPOSITORY_ID = 470904030; +const WORKFLOW = '.github/workflows/keeper-canonical-native.yml'; +const WORKER = '.github/workflows/keeper-canonical-native-platform.yml'; +const APP_ID = 15368; +const SHA = /^[a-f0-9]{40}$/; +const digest = bytes => crypto.createHash('sha256').update(bytes).digest('hex'); +const demand = (condition, code) => { if (!condition) throw new Error(code); }; +const artifactName = (platform, request) => `keeper-canonical-native-${platform}-${request.sourceSha}-${request.runId}-${request.attempt}`; +const attestationName = request => `keeper-canonical-native-attestation-${request.sourceSha}-${request.runId}-${request.attempt}`; +const publicProofs = new WeakSet(); +const publicAuthorities = new WeakMap(); +function freeze(value) { + if (value && typeof value === 'object') { Object.values(value).forEach(freeze); Object.freeze(value); } + return value; +} + +function requestFromEnvironment(context, env) { + const request = { sourceSha: env.KEEPER_SOURCE_SHA, workflowSha: env.GITHUB_WORKFLOW_SHA, + workflowRef: env.GITHUB_WORKFLOW_REF, pin: env.KEEPER_CANONICAL_NATIVE_DRIVER_SHA, + runId: Number(env.GITHUB_RUN_ID), attempt: Number(env.GITHUB_RUN_ATTEMPT) }; + demand(context.eventName === 'workflow_dispatch' && context.ref === 'refs/heads/sprint' && + `${context.repo.owner}/${context.repo.repo}` === REPOSITORY, 'CANONICAL_DISPATCH_REF_REQUIRED'); + demand(SHA.test(request.sourceSha || '') && request.sourceSha === context.sha && + request.workflowSha === request.sourceSha && request.pin === request.sourceSha && + request.workflowRef === `${REPOSITORY}/${WORKFLOW}@refs/heads/sprint`, 'INSTALLED_DRIVER_PIN_REQUIRED'); + demand(Number.isSafeInteger(request.runId) && request.runId === context.runId && request.runId > 0 && + Number.isSafeInteger(request.attempt) && request.attempt > 0, 'EXACT_RUN_ATTEMPT_REQUIRED'); + return request; +} + +function githubApp(check) { + return check?.app?.id === APP_ID && check.app.slug === 'github-actions'; +} + +function pinReaderFromEnvironment(github, env) { + const token = env.KEEPER_CANONICAL_PIN_READ_TOKEN; + // The collector spawns the bounded ZIP reader. Do not pass this credential to + // that subprocess or to subsequent tooling through the process environment. + delete env.KEEPER_CANONICAL_PIN_READ_TOKEN; + demand(typeof token === 'string' && token.trim().length > 0, 'LIVE_DRIVER_PIN_IDENTITY_REQUIRED'); + // Keep the normal GITHUB_TOKEN client for Actions/checks/contents. This + // repository-selected Variables:read identity is used only for the live pin. + return new github.constructor({ auth: token }); +} + +function privateReaderFromEnvironment(github, env) { + const token = env.KEEPER_CANONICAL_PRIVATE_ACTIONS_READ_TOKEN; + delete env.KEEPER_CANONICAL_PRIVATE_ACTIONS_READ_TOKEN; + // Missing private identity preserves incomplete readiness; it never borrows + // the public Actions client or its token to produce parent facts. + if (token === undefined || token === '') return undefined; + demand(typeof token === 'string' && token.trim().length > 0, 'PRIVATE_ACTIONS_READ_IDENTITY_INVALID'); + try { return new github.constructor({ auth: token }); } + catch { throw new Error('PRIVATE_ACTIONS_READ_IDENTITY_INVALID'); } +} + +async function origin(github, context, request, { admission = false, pinReader } = {}) { + demand(typeof pinReader?.rest?.actions?.getRepoVariable === 'function', 'LIVE_DRIVER_PIN_IDENTITY_REQUIRED'); + const args = context.repo; + const [repository, branch, workflow, run, pin] = await Promise.all([ + github.rest.repos.get(args).then(r => r.data), + github.rest.repos.getBranch({ ...args, branch: 'sprint' }).then(r => r.data), + github.rest.actions.getWorkflow({ ...args, workflow_id: path.basename(WORKFLOW) }).then(r => r.data), + github.rest.actions.getWorkflowRun({ ...args, run_id: request.runId }).then(r => r.data), + pinReader.rest.actions.getRepoVariable({ ...args, name: 'KEEPER_CANONICAL_NATIVE_DRIVER_SHA', + headers: { 'cache-control': 'no-cache' } }).then(r => r.data) + .catch(() => { throw new Error('LIVE_DRIVER_PIN_REVOKED_OR_UNREADABLE'); }), + ]); + demand(pin?.name === 'KEEPER_CANONICAL_NATIVE_DRIVER_SHA' && pin.value === request.sourceSha && pin.value === request.pin, + 'LIVE_DRIVER_PIN_REVOKED_OR_UNREADABLE'); + demand(repository.id === REPOSITORY_ID && repository.full_name === REPOSITORY && repository.default_branch === 'sprint', 'REPOSITORY_MISMATCH'); + demand(branch.protected === true && branch.commit?.sha === request.sourceSha, 'CANONICAL_HEAD_MOVED_OR_UNPROTECTED'); + demand(workflow.path === WORKFLOW && workflow.state === 'active', 'CANONICAL_WORKFLOW_NOT_INSTALLED'); + demand(run.id === request.runId && run.run_attempt === request.attempt && run.workflow_id === workflow.id && + run.event === 'workflow_dispatch' && run.path?.split('@')[0] === WORKFLOW && + run.head_sha === request.sourceSha && run.head_branch === 'sprint' && + run.head_repository?.id === REPOSITORY_ID && run.head_repository.full_name === REPOSITORY && + Array.isArray(run.pull_requests) && run.pull_requests.length === 0 && + (run.status === 'in_progress' || (run.status === 'completed' && run.conclusion === 'success')), + 'CANONICAL_RUN_MISMATCH'); + const suite = (await github.rest.checks.getSuite({ ...args, check_suite_id: run.check_suite_id })).data; + demand(githubApp(suite) && suite.head_sha === request.sourceSha && suite.id === run.check_suite_id, 'FORGED_ACTIONS_APP'); + const listing = (await github.rest.actions.listWorkflowRuns({ ...args, workflow_id: workflow.id, + branch: 'sprint', event: 'workflow_dispatch', head_sha: request.sourceSha, per_page: 100 })).data; + demand(Number.isSafeInteger(listing.total_count) && listing.total_count <= 100 && + listing.total_count === listing.workflow_runs?.length && + listing.workflow_runs.every(item => item.head_sha === request.sourceSha) && + Math.max(...listing.workflow_runs.map(item => item.id)) === request.runId, + 'NEWER_OR_INCOMPLETE_CANONICAL_RUNS'); + if (admission) { + const active = await Promise.all(['in_progress', 'queued', 'waiting', 'pending', 'requested'].map(async status => + (await github.rest.actions.listWorkflowRunsForRepo({ ...args, status, per_page: 100 })).data)); + const nativePaths = [WORKFLOW, '.github/workflows/keeper-hosted-native.yml', + '.github/workflows/keeper-native-diagnostic.yml', '.github/workflows/keeper-release-acceptance.yml', + '.github/workflows/keeper-one-time-native.yml', '.github/workflows/keeper-one-time-native-diagnostic.yml', + '.github/workflows/android-diagnostic-build.yml', '.github/workflows/keeper-protected-release-workers.yml']; + for (const page of active) { + demand(page.total_count <= 100 && page.total_count === page.workflow_runs?.length, 'CAPACITY_OBSERVATION_INCOMPLETE'); + demand(!page.workflow_runs.some(item => item.id !== request.runId && nativePaths.includes(item.path?.split('@')[0])), + 'ANOTHER_NATIVE_RUN_ACTIVE_OR_QUEUED'); + } + } + return { run, workflow }; +} + +function requiredSteps(platform) { + return ['Verify canonical installed source', 'Record runner and toolchain', 'Install immutable native dependencies', + 'Build development app', ...(platform === 'IOS' ? ['Check disposable iOS driver readiness', + 'Run disposable startup smoke', 'Run disposable native persistence probe', + 'Run R5 production worker native scenarios', 'Measure full-product startup wallet and Retry flows'] + : ['Run Android startup persistence worker and product flows'])]; +} + +async function platformJobs(github, context, request, run) { + const jobs = await github.paginate(github.rest.actions.listJobsForWorkflowRunAttempt, + { ...context.repo, run_id: request.runId, attempt_number: request.attempt, per_page: 100 }); + const result = {}; + for (const platform of ['IOS', 'ANDROID']) { + const matches = jobs.filter(job => job.name === `${platform.toLowerCase()} / Canonical ${platform} build and features`); + demand(matches.length === 1, 'CANONICAL_PLATFORM_JOB_MISSING_OR_AMBIGUOUS'); + const job = matches[0]; + demand(Number.isSafeInteger(job.id) && job.id > 0 && !Object.values(result).some(previous => previous.id === job.id) && + job.run_id === request.runId && job.run_attempt === request.attempt && job.head_sha === request.sourceSha && + job.status === 'completed' && job.conclusion === 'success', 'CANONICAL_PLATFORM_NOT_EXACT_SUCCESS'); + for (const name of requiredSteps(platform)) { + const steps = job.steps?.filter(step => step.name === name) ?? []; + demand(steps.length === 1 && steps[0].status === 'completed' && steps[0].conclusion === 'success', 'CANONICAL_REQUIRED_STEP_NOT_SUCCESS'); + } + const match = job.check_run_url?.match(/^https:\/\/api\.github\.com\/repos\/KeeperCommunity\/bitcoin-keeper\/check-runs\/([0-9]+)$/); + demand(match, 'CANONICAL_JOB_CHECK_LINK_INVALID'); + const check = (await github.rest.checks.get({ ...context.repo, check_run_id: Number(match[1]) })).data; + demand(check.id === Number(match[1]) && check.name === job.name && githubApp(check) && + check.head_sha === request.sourceSha && check.check_suite?.id === run.check_suite_id && + check.status === 'completed' && check.conclusion === 'success', 'CANONICAL_JOB_CHECK_UNTRUSTED'); + result[platform] = job; + } + return result; +} + +function validateMetadata(metadata, request, platform, workerSources) { + const apps = { IOS: 'io.hexawallet.hexakeeper.dev', ANDROID: 'io.hexawallet.keeper.development' }; + demand(metadata.schema === 1 && metadata.platform === platform && metadata.source_sha === request.sourceSha && + metadata.workflow_sha === request.workflowSha && metadata.workflow_ref === request.workflowRef && + metadata.run_id === String(request.runId) && metadata.run_attempt === String(request.attempt) && + metadata.app_id === apps[platform] && metadata.diagnostic_only === false, 'CANONICAL_RECEIPT_IDENTITY_MISMATCH'); + const status = value => value === 'PASS' ? 'PASS' : 'NOT VERIFIED'; + const checks = { + 'native-build': { status: status(metadata.build), scope: 'development native build' }, + 'startup': { status: status(metadata.smoke), scope: 'ordinary app passcode startup smoke' }, + 'native-storage': { status: status(metadata.native_probe), scope: 'CI entrypoint native Realm/MMKV process persistence' }, + 'production-worker-controls': { status: status(metadata.r5_features), scope: 'CI-extracted production workers; synthetic boundaries are not complete application UI' }, + 'five-product-observations': { status: 'NOT VERIFIED', scope: 'ordinary app debug UI observations; no release performance budget' }, + 'controlled-setup-disconnect-retry': { status: 'NOT VERIFIED', scope: 'ordinary app loopback disconnect before commit; not physical WAN offline' }, + 'installed-version-history': { status: 'NOT VERIFIED', scope: 'native installed tuple and retained initial history; not stale-history recovery' }, + 'key-management-discoverability': { status: 'NOT VERIFIED', scope: 'ordinary app Keys Manage Keys entry and return' }, + 'unused-key-and-pin-ui': { status: 'NOT VERIFIED', scope: 'unused Mobile Key delete cancel wrong PIN and cold persistence; not live-Vault references or physical biometrics' }, + }; + const samples = metadata.product_ui?.samples; + if (metadata.product_ui?.status === 'PASS' && metadata.product_ui.requested_samples === 5 && + Array.isArray(samples) && samples.length === 5 && samples.every((sample, index) => + sample.sample === index + 1 && sample.status === 'PASS' && sample.create_new_app_calls === 1 && + sample.visible_wallet_count_before_relaunch === 1 && sample.visible_wallet_count_after_relaunch === 1 && + ['startup_to_passcode_ms', 'start_new_to_ready_ms'].every(key => + Number.isFinite(sample.raw_durations?.[key]) && sample.raw_durations[key] >= 0))) { + checks['five-product-observations'].status = 'PASS'; + if (samples.every(sample => sample.installed_version_header_and_initial_history === 'PASS')) + checks['installed-version-history'].status = 'PASS'; + if (samples.every(sample => sample.key_management_entry_and_exit === 'PASS')) + checks['key-management-discoverability'].status = 'PASS'; + } + const retry = metadata.product_ui?.controlled_disconnect_retry; + if (retry?.status === 'PASS' && retry.scenario === 'CONTROLLED_DISCONNECT_BEFORE_COMMIT_NOT_WAN_OFFLINE' && + retry.performance_sample === false && retry.create_new_app_calls === 2 && + retry.transport_requests_dropped_without_commit === 1 && retry.acknowledged_create_responses === 1 && + retry.unexpected_create_attempts === 0 && retry.visible_wallet_count_after_relaunch === 1) + checks['controlled-setup-disconnect-retry'].status = 'PASS'; + const key = metadata.product_ui?.key_management_ui; + if (key?.status === 'PASS' && key.entrypoint === 'index.js' && key.network === 'TESTNET' && + key.scenario === 'UNUSED_MOBILE_KEY_UI_DELETE_AND_NATIVE_PIN' && key.new_account_requests_during_key_flow === 0 && + key.visible_eligible_keys_after_deletion === 0 && key.unrelated_visible_wallet_count_after_relaunch === 1) + checks['unused-key-and-pin-ui'].status = 'PASS'; + const parent = acceptance({ trusted: checks['native-build'].status === 'PASS' && checks.startup.status === 'PASS' && + checks['native-storage'].status === 'PASS', ios: 'success', android: 'success' }); + const cases = REQUIRED.map(id => ({ id, status: parent.cases.find(item => item.id === id)?.status ?? 'NOT IMPLEMENTED' })); + Object.assign(checks, publicCompanionChecks(metadata, platform)); + checks['public-raw-software-evidence'] = { status: 'NOT VERIFIED', scope: PUBLIC_SOFTWARE_SCOPE }; + try { + validatePublicSoftware(metadata, { platform, sourceSha: request.sourceSha, + workflowSha: request.workflowSha, workflowRef: request.workflowRef, + runId: request.runId, runAttempt: request.attempt, workerSources }); + checks['public-raw-software-evidence'].status = 'PASS'; + } catch { /* A label or payload-owned source hash cannot satisfy this subcheck. */ } + // Parent obligations remain explicit. Scoped companion UI does not establish + // private bytes, protocol/account isolation, WAN offline, recovery or a budget. + return { cases, executed_checks: checks }; +} + +async function collectPublic(github, context, request, readZip, { pinReader } = {}) { + requestFromEnvironment(context, { KEEPER_SOURCE_SHA: request.sourceSha, GITHUB_WORKFLOW_SHA: request.workflowSha, + GITHUB_WORKFLOW_REF: request.workflowRef, KEEPER_CANONICAL_NATIVE_DRIVER_SHA: request.pin, + GITHUB_RUN_ID: String(request.runId), GITHUB_RUN_ATTEMPT: String(request.attempt) }); + const { run } = await origin(github, context, request, { pinReader }); + const jobs = await platformJobs(github, context, request, run); + const jobAuthorities = Object.fromEntries(['IOS', 'ANDROID'].map(platform => [platform, + { id: jobs[platform].id, name: jobs[platform].name, check_run_url: jobs[platform].check_run_url }])); + // Compare full-file worker hashes with fixed committed inputs, independently + // of the downloaded receipt. No artifact ingestion or private-route admission. + const workerSources = await Promise.all(Object.entries(WORKER_SOURCES).map(async ([member, names]) => { + const file = (await github.rest.repos.getContent({ ...context.repo, path: member, ref: request.sourceSha })).data; + demand(file?.type === 'file' && file.path === member && file.encoding === 'base64' && + Number.isSafeInteger(file.size) && file.size > 0 && file.size <= 524288 && + typeof file.content === 'string' && /^[A-Za-z0-9+/=\r\n]+$/.test(file.content), 'CANONICAL_WORKER_SOURCE_UNAVAILABLE'); + const bytes = Buffer.from(file.content, 'base64'); + demand(bytes.length === file.size, 'CANONICAL_WORKER_SOURCE_UNAVAILABLE'); + return { path: member, names: [...names], sha256: digest(bytes) }; + })); + const page = (await github.rest.actions.listWorkflowRunArtifacts({ ...context.repo, run_id: request.runId, per_page: 100 })).data; + demand(page.total_count <= 100 && page.total_count === page.artifacts?.length, 'CANONICAL_ARTIFACT_LIST_INCOMPLETE'); + const platforms = {}, artifactAuthorities = []; + for (const platform of ['IOS', 'ANDROID']) { + const matches = page.artifacts.filter(item => item.name === artifactName(platform, request)); + demand(matches.length === 1, 'CANONICAL_ARTIFACT_MISSING_OR_AMBIGUOUS'); + const artifact = matches[0]; + demand(!artifact.expired && artifact.size_in_bytes > 0 && artifact.size_in_bytes <= 1048576 && + artifact.workflow_run?.id === request.runId && artifact.workflow_run.head_sha === request.sourceSha && + /^sha256:[a-f0-9]{64}$/.test(artifact.digest || ''), 'CANONICAL_ARTIFACT_IDENTITY_INVALID'); + // Capture primitive authority fields independently of response object identity. + // Every selected artifact is rechecked after all later platform/tree reads. + artifactAuthorities.push({ id: artifact.id, name: artifact.name, digest: artifact.digest, + size: artifact.size_in_bytes, runId: artifact.workflow_run.id, sourceSha: artifact.workflow_run.head_sha, + expiresAt: artifact.expires_at ?? null }); + const archive = Buffer.from((await github.rest.actions.downloadArtifact({ ...context.repo, + artifact_id: artifact.id, archive_format: 'zip' })).data); + demand(archive.length <= 1048576 && archive.length === artifact.size_in_bytes && + `sha256:${digest(archive)}` === artifact.digest, 'CANONICAL_ARTIFACT_DIGEST_INVALID'); + const current = (await github.rest.actions.getArtifact({ ...context.repo, artifact_id: artifact.id })).data; + demand(current.id === artifact.id && current.name === artifact.name && !current.expired && + current.digest === artifact.digest && current.size_in_bytes === artifact.size_in_bytes && + current.workflow_run?.id === request.runId && current.workflow_run.head_sha === request.sourceSha, + 'CANONICAL_ARTIFACT_EXPIRED_OR_CHANGED'); + const raw = readZip(archive); + const metadata = parseStrictJson(raw); + platforms[platform] = { conclusion: 'success', artifact_id: artifact.id, + receipt_sha256: digest(raw), ...validateMetadata(metadata, request, platform, workerSources) }; + } + const commit = (await github.rest.git.getCommit({ ...context.repo, commit_sha: request.sourceSha })).data; + demand(commit.sha === request.sourceSha && SHA.test(commit.tree?.sha || ''), 'CANONICAL_SOURCE_TREE_UNVERIFIED'); + for (const original of artifactAuthorities) { + const current = (await github.rest.actions.getArtifact({ ...context.repo, artifact_id: original.id })).data; + demand(current.id === original.id && current.name === original.name && current.expired === false && + current.digest === original.digest && current.size_in_bytes === original.size && + current.workflow_run?.id === original.runId && current.workflow_run.head_sha === original.sourceSha, + 'CANONICAL_ARTIFACT_EXPIRED_OR_CHANGED'); + } + // The source/pin/run fence must be terminal: commit-tree and final artifact + // reads above can observe a revocation that occurred after initial admission. + await origin(github, context, request, { pinReader }); + const result = freeze({ schema: 1, kind: 'keeper-installed-public-software', repository: REPOSITORY, + source_sha: request.sourceSha, source_tree: commit.tree.sha, workflow_sha: request.workflowSha, + workflow_path: WORKFLOW, run_id: request.runId, run_attempt: request.attempt, + selected_route: 'canonical-hosted-native', platforms, worker_sources: workerSources }); + if (Object.values(platforms).every(item => item.cases.slice(0, 2).every(test => test.status === 'PASS') && + Object.values(item.executed_checks).every(test => test.status === 'PASS'))) { + publicProofs.add(result); + publicAuthorities.set(result, freeze({ request: { ...request }, jobs: jobAuthorities, + artifacts: artifactAuthorities, tree: commit.tree.sha, suiteId: run.check_suite_id, workflowId: run.workflow_id })); + } + return result; +} + +async function collectInstalledPublicEvidence(github, context, request, readZip, options = {}) { + const result = await collectPublic(github, context, request, readZip, options); + demand(publicProofs.has(result), 'CANONICAL_PUBLIC_SOFTWARE_INCOMPLETE'); + return result; +} + +async function revalidateInstalledPublicEvidence(github, context, request, proof, { pinReader, clock = Date.now } = {}) { + demand(publicProofs.has(proof), 'CANONICAL_PUBLIC_CAPABILITY_REQUIRED'); + const original = publicAuthorities.get(proof); + demand(['sourceSha', 'workflowSha', 'workflowRef', 'pin', 'runId', 'attempt'].every(key => + request[key] === original.request[key]), 'CANONICAL_PUBLIC_WINDOW_MISMATCH'); + const { run } = await origin(github, context, request, { pinReader }); + demand(run.check_suite_id === original.suiteId && run.workflow_id === original.workflowId, + 'CANONICAL_PUBLIC_AUTHORITY_CHANGED'); + const jobs = await platformJobs(github, context, request, run); + for (const platform of ['IOS', 'ANDROID']) { + const current = jobs[platform], previous = original.jobs[platform]; + demand(current.id === previous.id && current.name === previous.name && current.check_run_url === previous.check_run_url, + 'CANONICAL_PUBLIC_AUTHORITY_CHANGED'); + } + const commit = (await github.rest.git.getCommit({ ...context.repo, commit_sha: request.sourceSha })).data; + demand(commit.sha === request.sourceSha && commit.tree?.sha === original.tree, 'CANONICAL_SOURCE_TREE_UNVERIFIED'); + for (const previous of original.artifacts) { + const current = (await github.rest.actions.getArtifact({ ...context.repo, artifact_id: previous.id })).data; + const expiry = current.expires_at ?? null; + demand(current.id === previous.id && current.name === previous.name && current.expired === false && + current.digest === previous.digest && current.size_in_bytes === previous.size && + current.workflow_run?.id === previous.runId && current.workflow_run.head_sha === previous.sourceSha && + expiry === previous.expiresAt && (expiry === null || (typeof expiry === 'string' && Date.parse(expiry) > clock())), + 'CANONICAL_ARTIFACT_EXPIRED_OR_CHANGED'); + } + // Long private network/ZIP work cannot preserve stale public proof authority. + // This final public phase also rechecks source/workflow/current run and live pin. + await origin(github, context, request, { pinReader }); + return true; +} + +async function collect(github, context, request, output, readZip, { pinReader, privateGithub, clock, fetchImpl } = {}) { + fs.rmSync(path.join(output, 'keeper-canonical-native-attestation.json'), { force: true }); + const publicProof = await collectPublic(github, context, request, readZip, { pinReader }); + const { kind, worker_sources, ...publicFields } = publicProof; + // No default-token fallback: parent collection requires an explicitly supplied + // separate read identity and a genuinely installed, finite source profile. + let platforms = publicProof.platforms; + if (privateGithub !== undefined) { + const parents = require('./installed_parent_software.cjs'); + const privateProof = await parents.collectInstalledParentSoftware(privateGithub, github, context, request, + publicProof, { pinReader, clock, fetchImpl }); + platforms = parents.projectInstalledParentCases(publicProof, privateProof); + } + const complete = Object.values(platforms).every(item => item.cases.length === REQUIRED.length && + item.cases.every(test => test.status === 'PASS') && Object.values(item.executed_checks).every(test => test.status === 'PASS')); + const result = { ...publicFields, platforms, kind: 'keeper-canonical-native', feature_matrix_complete: complete }; + fs.mkdirSync(output, { recursive: true }); + fs.writeFileSync(path.join(output, complete ? 'keeper-canonical-native-attestation.json' : + 'keeper-canonical-native-readiness.json'), JSON.stringify(result, null, 2) + '\n'); + demand(complete, 'CANONICAL_REQUIRED_APP_MATRIX_INCOMPLETE'); + return result; +} + +function readMetadataZip(archive) { + const directory = fs.mkdtempSync(path.join(os.tmpdir(), 'keeper-canonical-receipt-')); + try { + const file = path.join(directory, 'receipt.zip'); fs.writeFileSync(file, archive, { mode: 0o600 }); + return execFileSync('python3', [path.join(__dirname, 'canonical_metadata.py'), file], { maxBuffer: 524288 }); + } finally { fs.rmSync(directory, { recursive: true, force: true }); } +} +module.exports = { REPOSITORY, REPOSITORY_ID, WORKFLOW, WORKER, APP_ID, requestFromEnvironment, + pinReaderFromEnvironment, privateReaderFromEnvironment, origin, requiredSteps, platformJobs, validateMetadata, collect, + artifactName, attestationName, readMetadataZip, collectInstalledPublicEvidence, + revalidateInstalledPublicEvidence, + isInstalledPublicEvidence: value => publicProofs.has(value) }; diff --git a/.github/native/contract-tools/package-lock.json b/.github/native/contract-tools/package-lock.json new file mode 100644 index 0000000000..255413586f --- /dev/null +++ b/.github/native/contract-tools/package-lock.json @@ -0,0 +1,28 @@ +{ + "name": "keeper-native-contract-tools", + "version": "1.0.0", + "lockfileVersion": 3, + "requires": true, + "packages": { + "": { + "name": "keeper-native-contract-tools", + "version": "1.0.0", + "dependencies": { + "typescript": "5.0.4" + } + }, + "node_modules/typescript": { + "version": "5.0.4", + "resolved": "https://registry.npmjs.org/typescript/-/typescript-5.0.4.tgz", + "integrity": "sha512-cW9T5W9xY37cc+jfEnaUvX91foxtHkza3Nw3wkoF4sSlKn0MONdkdEndig/qPBWXNkmplh3NzayQzCiHM4/hqw==", + "license": "Apache-2.0", + "bin": { + "tsc": "bin/tsc", + "tsserver": "bin/tsserver" + }, + "engines": { + "node": ">=12.20" + } + } + } +} diff --git a/.github/native/contract-tools/package.json b/.github/native/contract-tools/package.json new file mode 100644 index 0000000000..215add3f49 --- /dev/null +++ b/.github/native/contract-tools/package.json @@ -0,0 +1,8 @@ +{ + "name": "keeper-native-contract-tools", + "version": "1.0.0", + "private": true, + "dependencies": { + "typescript": "5.0.4" + } +} diff --git a/.github/native/credential_arm.js b/.github/native/credential_arm.js new file mode 100644 index 0000000000..feb36b8ceb --- /dev/null +++ b/.github/native/credential_arm.js @@ -0,0 +1,2 @@ +import { registerFixture } from './credential_fixture'; +registerFixture('arm'); diff --git a/.github/native/credential_fixture.js b/.github/native/credential_fixture.js new file mode 100644 index 0000000000..c703855e0e --- /dev/null +++ b/.github/native/credential_fixture.js @@ -0,0 +1,26 @@ +// Alternate fixture entrypoint only; ordinary app source is never patched. +import React, { useEffect, useState } from 'react'; +import { AppRegistry, SafeAreaView, Text } from 'react-native'; +import * as Keychain from 'react-native-keychain'; +import config from 'react-native-config'; +import { Storage } from '../../src/storage'; +import { name as appName } from '../../app.json'; +import { runFixture } from './credential_fixture_core'; +// Selected by the two fixed entry wrappers, never by production app state. +export function registerFixture(phase) { + let attempt; + function CredentialFixture() { + const [stage, setStage] = useState('running'); + useEffect(() => { + let active = true; + if (!attempt) attempt = runFixture({ phase, Keychain, storage: Storage, config, + development: __DEV__ }).catch(() => 'failed'); + attempt.then((value) => { if (active) setStage(value); }); + return () => { active = false; }; + }, []); + return + Keeper disposable credential fixture {stage} + ; + } + AppRegistry.registerComponent(appName, () => CredentialFixture); +} diff --git a/.github/native/credential_fixture.yaml b/.github/native/credential_fixture.yaml new file mode 100644 index 0000000000..37a89a4c14 --- /dev/null +++ b/.github/native/credential_fixture.yaml @@ -0,0 +1,11 @@ +appId: ${APP_ID} +name: Device-only credential fixture phase +--- +- launchApp +- extendedWaitUntil: + visible: + id: credential-fixture-${FIXTURE_STAGE} + timeout: 60000 +- assertNotVisible: + id: credential-fixture-failed +- stopApp diff --git a/.github/native/credential_fixture_core.js b/.github/native/credential_fixture_core.js new file mode 100644 index 0000000000..b986c35552 --- /dev/null +++ b/.github/native/credential_fixture_core.js @@ -0,0 +1,71 @@ +/* CI-only fixture. Secret bytes remain in native Keychain or this process. */ +const BACKUP_SERVICE = 'keeper-ci-malformed-credential-recovery-v1'; +const MALFORMED = '{"keeperCiMalformedCredential":'; +class FixtureFailure extends Error { + constructor(code) { super(code); this.code = code; } +} +function check(value, code) { if (!value) throw new FixtureFailure(code); } +function owner(storage) { + const root = JSON.parse(storage.getString('persist:root')); + const accounts = JSON.parse(root.account).allAccounts; + const current = JSON.parse(root.storage).appId; + check(Array.isArray(accounts) && accounts.length === 1, 'sole-account-required'); + const account = accounts[0]; + check(account.isDefault === true && account.accountIdentifier === '' && + typeof account.appId === 'string' && account.appId.length > 0 && + account.appId === current && typeof account.realmId === 'string' && account.realmId.length > 0 && + typeof account.hash === 'string' && account.hash.length > 0, 'default-owner-required'); + return { appId: account.appId, realmId: account.realmId, hash: account.hash }; +} +function validOriginal(value, current, username) { + check(value && value.username === username && typeof value.password === 'string', 'credential-required'); + const parsed = JSON.parse(value.password); + check(parsed && parsed.hash === current.hash && typeof parsed.enc_key === 'string' && + parsed.enc_key.length > 0, 'matching-credential-required'); +} +async function runFixture({ phase, Keychain, storage, config, development }) { + check(development === true && config.ENVIRONMENT === 'DEVELOPMENT' && + config.RELAY === 'http://127.0.0.1:8765/' && + config.ENC_KEY_STORAGE_IDENTIFIER === 'KEEPER-HOSTED-DISPOSABLE', 'disposable-build-required'); + check(phase === 'arm' || phase === 'restore', 'phase-rejected'); + const current = owner(storage); + const defaultOptions = { service: undefined }; + const backupOptions = { service: BACKUP_SERVICE, accessible: Keychain.ACCESSIBLE.WHEN_UNLOCKED }; + const username = config.ENC_KEY_STORAGE_IDENTIFIER; + if (phase === 'arm') { + // Successfully empty is never labelled a read fault or turned into a test account. + check((await Keychain.getGenericPassword({ service: BACKUP_SERVICE })) === false, 'backup-already-present'); + const original = await Keychain.getGenericPassword(defaultOptions); + validOriginal(original, current, username); + const payload = JSON.stringify({ schema: 1, owner: current, + original: { username: original.username, password: original.password } }); + check(await Keychain.setGenericPassword(username, payload, backupOptions), 'backup-write-failed'); + const saved = await Keychain.getGenericPassword({ service: BACKUP_SERVICE }); + check(saved && saved.username === username && saved.password === payload, 'backup-readback-failed'); + check(await Keychain.setGenericPassword(username, MALFORMED, + { ...defaultOptions, accessible: Keychain.ACCESSIBLE.WHEN_UNLOCKED }), 'malformed-write-failed'); + const malformed = await Keychain.getGenericPassword(defaultOptions); + check(malformed && malformed.username === username && malformed.password === MALFORMED, + 'malformed-readback-failed'); + return 'armed'; + } + const saved = await Keychain.getGenericPassword({ service: BACKUP_SERVICE }); + check(saved && saved.username === username, 'backup-required'); + const payload = JSON.parse(saved.password); + check(payload && payload.schema === 1 && JSON.stringify(payload.owner) === JSON.stringify(current), + 'backup-owner-mismatch'); + validOriginal(payload.original, current, username); + const malformed = await Keychain.getGenericPassword(defaultOptions); + check(malformed && malformed.username === username && malformed.password === MALFORMED, + 'restore-source-changed'); + check(await Keychain.setGenericPassword(payload.original.username, payload.original.password, + { ...defaultOptions, accessible: Keychain.ACCESSIBLE.WHEN_UNLOCKED }), 'restore-write-failed'); + const restored = await Keychain.getGenericPassword(defaultOptions); + check(restored && restored.username === payload.original.username && + restored.password === payload.original.password, 'restore-readback-failed'); + check(await Keychain.resetGenericPassword({ service: BACKUP_SERVICE }), 'backup-cleanup-failed'); + check((await Keychain.getGenericPassword({ service: BACKUP_SERVICE })) === false, + 'backup-cleanup-readback-failed'); + return 'restored'; +} +module.exports = { BACKUP_SERVICE, MALFORMED, FixtureFailure, runFixture }; diff --git a/.github/native/credential_metro.cjs b/.github/native/credential_metro.cjs new file mode 100644 index 0000000000..b9c2f3686e --- /dev/null +++ b/.github/native/credential_metro.cjs @@ -0,0 +1,21 @@ +// Only the owned hosted runner starts this config, while the ordinary app is stopped. +const path = require('node:path'); +const base = require('../../metro.config'); +const phase = process.env.KEEPER_CREDENTIAL_FIXTURE_PHASE; +if (!['arm', 'restore'].includes(phase)) throw new Error('CREDENTIAL_FIXTURE_PHASE_REQUIRED'); +const enhance = base.server?.enhanceMiddleware; +module.exports = { + ...base, + projectRoot: path.resolve(__dirname, '../..'), + server: { + ...base.server, + enhanceMiddleware(middleware, server) { + const inner = enhance ? enhance(middleware, server) : middleware; + return (req, res, next) => { + req.url = req.url.replace(/^\/index\.(bundle|map)(?=\?|$)/, + `/.github/native/credential_${phase}.$1`); + return inner(req, res, next); + }; + }, + }, +}; diff --git a/.github/native/credential_restore.js b/.github/native/credential_restore.js new file mode 100644 index 0000000000..714e28e026 --- /dev/null +++ b/.github/native/credential_restore.js @@ -0,0 +1,2 @@ +import { registerFixture } from './credential_fixture'; +registerFixture('restore'); diff --git a/.github/native/driver_diagnosis.py b/.github/native/driver_diagnosis.py new file mode 100644 index 0000000000..ad7db34b7f --- /dev/null +++ b/.github/native/driver_diagnosis.py @@ -0,0 +1,403 @@ +#!/usr/bin/env python3 +"""Setup-only iOS driver diagnosis; export fixed facts, never private log text. + +This deliberately keeps the pinned Maestro and existing hierarchy launch path. +There is no Keeper install/build, dependency install, functional correction or +timeout change. Capture happens before hosted_native.driver deletes its device. +""" + +import hashlib +import http.client +import io +import json +from concurrent.futures import ThreadPoolExecutor, TimeoutError as FutureTimeoutError +import os +from pathlib import Path +import plistlib +import re +import socket +import subprocess +import sys +import time +import zipfile + +import hosted_native as native + + +LIMIT = 1024 * 1024 +POLL_SNAPSHOT_SECONDS = 60 +RUNNER_ID = 'dev.mobile.maestro-driver-iosUITests.xctrunner' +RUNNER_NAME = 'maestro-driver-iosUITests-Runner' +CATEGORIES = { + 'DYLD_LIBRARY_MISSING': r'library not loaded:|reason: tried:|image not found', + 'DYLD_SYMBOL_MISSING': r'symbol not found:|symbol missing|missing symbol', + 'DYLD_INCOMPATIBLE_LIBRARY': r'incompatible library version|incompatible architecture|wrong architecture', + 'DYLD_LIBRPAC_REFERENCE': r'library not loaded:.*librpac|reason: tried:.*librpac', + 'DYLD_SWIFT_COMPATIBILITY_REFERENCE': r'library not loaded:.*swiftcompatibility|reason: tried:.*swiftcompatibility', + 'TEST_BUNDLE_LOAD_FAILED': r'failed to load (?:the )?test bundle|could not load (?:the )?test bundle|bundle.*couldn.t be loaded', + 'TEST_RUNNER_LAUNCH_FAILED': r'failed to launch|could not launch|application failed to launch|failed to start test runner', + 'TEST_RUNNER_INSTALL_FAILED': r'failed to install|could not install|applicationverificationfailed|installd.*error', + 'TEST_MANAGER_CONNECTION_FAILED': r'failed to establish.*connection|failed to connect to testmanagerd|lost connection to the test manager', + 'TEST_RUNNER_WAIT_TIMEOUT': r'timed out waiting for.*(?:test runner|testing process|testmanagerd)|test runner.*failed to initialize', + 'RUNTIME_OR_DEPLOYMENT_MISMATCH': r'requires (?:ios|a newer version)|minimum.*(?:ios|deployment)|built for.*(?:newer|later)|unsupported os version', + 'XCODE_COMPONENT_MISSING': r'xcode.*(?:component|platform).*(?:missing|not installed)|unable to find.*(?:platform|runtime)', + 'DESTINATION_UNAVAILABLE': r'unable to find a destination|ineligible destinations|device.*(?:not available|unavailable|disconnected)', + 'CODE_SIGNING_FAILED': r'code sign(?:ing)?.*(?:failed|error)|invalid code signature|codesign.*failed', + 'PERMISSION_DENIED': r'permission denied|operation not permitted|not authorized', + 'DISK_SPACE_EXHAUSTED': r'no space left on device|not enough (?:free )?(?:disk )?space', + 'PROCESS_CRASHED': r'exception type:|termination reason:|crashed thread:|"exception"\s*:\s*\{|"termination"\s*:\s*\{', + 'XCODE_TEST_EXECUTE_FAILED': r'\*\* test execute failed \*\*', + 'XCODE_TEST_FAILED': r'\*\* test failed \*\*|testing failed:', + 'XCODE_TEST_STARTED': r'test suite .* started|testing started|test case .* started', + 'XCODE_TEST_SUCCEEDED': r'\*\* test (?:execute )?succeeded \*\*', + 'DRIVER_SERVER_STARTED': r'(?:server|listening).*(?:7001|22087)|server started|server is running', + 'XCODE_COMMAND_INVOCATION': r'^Command line invocation:', + 'XCODE_USER_DEFAULTS': r'^User defaults from command line:', + 'XCODE_DESTINATION_PREPARATION': r'Preparing.*destination|Preparing.*simulator|destination.*prepar', + 'XCODE_RESULT_BUNDLE': r'Test session results, code coverage, and logs:', + 'LICENSE_NOT_ACCEPTED': r'license.*(?:not accepted|agree)|requires.*license', + 'FIRST_LAUNCH_REQUIRED': r'first launch.*(?:required|not complete)|runFirstLaunch|install.*additional components', + 'ERROR_DOMAIN_CORE_SIMULATOR': r'com\.apple\.CoreSimulator\.SimError', + 'ERROR_DOMAIN_DVT_DEVICE': r'DVTDevice\w*ErrorDomain|DVTiPhoneSimulator\w*ErrorDomain', + 'ERROR_DOMAIN_IDE_TEST': r'IDETest\w*ErrorDomain|IDEXCTest\w*ErrorDomain', + 'ERROR_DOMAIN_XPC': r'NSXPCConnection\w*ErrorDomain|NSCocoaErrorDomain[^\n]{0,80}Code=409[79]', +} + +STACK_MARKERS = { + 'XPC_SYNC_WAIT': r'xpc_connection_send_message_with_reply_sync', + 'MACH_WAIT': r'mach_msg|mach_msg2_trap', + 'DISPATCH_WAIT': r'dispatch_sync|dispatch_semaphore_wait|dispatch_group_wait', + 'CORE_SIMULATOR': r'CoreSimulator|SimDevice|SimServiceContext', + 'CORE_DEVICE': r'CoreDevice|CoreDeviceService', + 'DVT_DEVICE': r'DVTDevice|DVTiPhoneSimulator|DVTPlatform', + 'DVT_PLUGIN': r'DVTPlugIn|DVTPlugInManager', + 'IDE_TEST': r'IDETest|IDEXCTest|XCTest', + 'DEVICE_INSTALL': r'installApplication|installApp|installBundle', + 'DESTINATION_LOOKUP': r'findDestination|resolveDestination|availableDestinations|eligibleDestinations', + 'LOCK_WAIT': r'__psynch_mutexwait|__ulock_wait|pthread_mutex_lock', +} + + +def classify(payload): + return [name for name, pattern in CATEGORIES.items() + if re.search(pattern, payload, re.IGNORECASE | re.MULTILINE)] + + +def safe_error(error): + return type(error).__name__ if type(error).__name__ in { + 'OSError', 'FileNotFoundError', 'PermissionError', 'TimeoutExpired', + 'ValueError', 'TypeError', 'KeyError', 'BadZipFile', 'InvalidFileException', + 'ConnectionRefusedError', 'TimeoutError', 'RemoteDisconnected', + } else 'OTHER_ERROR' + + +def bounded(path): + if path.is_symlink() or not path.resolve().is_relative_to(native.PRIVATE.resolve()): + raise ValueError('Expected a private regular log.') + size = path.stat().st_size + with path.open('rb') as source: + payload = source.read(LIMIT) + return payload.decode('utf-8', errors='replace'), { + 'bytes': size, 'prefix_bytes': len(payload), 'truncated': size > LIMIT, + 'prefix_sha256': hashlib.sha256(payload).hexdigest(), + 'categories': classify(payload.decode('utf-8', errors='replace')), + } + + +def capture(args, label, timeout=20): + """Keep output private; return bounded text only to in-process classifiers.""" + path = native.PRIVATE / ('driver-diagnosis-' + label + '.log') + result = {'label': label} + try: + with path.open('wb') as output: + proc = subprocess.Popen(args, stdout=output, stderr=subprocess.STDOUT, + start_new_session=True) + try: + result['exit_code'] = proc.wait(timeout=timeout) + except subprocess.TimeoutExpired: + import signal + os.killpg(proc.pid, signal.SIGKILL) + proc.wait() + result['exit_code'] = 124 + payload, facts = bounded(path) + result.update(facts) + return payload, result + except Exception as error: + result['error_type'] = safe_error(error) + return '', result + + +def numeric(value): + value = str(value) + return value if re.fullmatch(r'\d+(?:\.\d+){0,3}', value) else None + + +def version(value): + return tuple(int(piece) for piece in value.split('.')) if value else () + + +def process_inventory_known(payload, result): + rows = [line for line in payload.splitlines() if line.strip()] + return (bool(rows) and result.get('exit_code') == 0 and not result.get('truncated', False) and + all(re.fullmatch(r'\s*\d+\s+\S+\s+\S.*', line) for line in rows)) + + +def bundled_runner(metadata=None): + """Read exact checksum-verified release JAR metadata without extracting code.""" + jars = list((native.PRIVATE / 'maestro').rglob('maestro-ios-driver.jar')) + if len(jars) != 1: + raise ValueError('Expected one pinned iOS driver JAR.') + result = {'jar_sha256': hashlib.sha256(jars[0].read_bytes()).hexdigest(), 'bundles': []} + with zipfile.ZipFile(jars[0]) as jar: + name = 'driver-iPhoneSimulator/Debug-iphonesimulator/maestro-driver-iosUITests-Runner.zip' + if jar.getinfo(name).file_size > 64 * 1024 * 1024: + raise ValueError('Unexpected bundled runner size.') + with zipfile.ZipFile(io.BytesIO(jar.read(name))) as runner: + names = { + 'runner': RUNNER_NAME + '.app/Info.plist', + 'test_bundle': RUNNER_NAME + '.app/PlugIns/maestro-driver-iosUITests.xctest/Info.plist', + } + for kind, name in names.items(): + if runner.getinfo(name).file_size > LIMIT: + raise ValueError('Unexpected bundled plist size.') + data = plistlib.loads(runner.read(name)) + sdk = re.fullmatch(r'iphonesimulator(\d+(?:\.\d+){0,2})(?:\.internal)?', + str(data.get('DTSDKName', ''))) + result['bundles'].append({ + 'kind': kind, 'xcode_numeric': numeric(data.get('DTXcode')), + 'sdk_version': sdk.group(1) if sdk else None, + 'minimum_os_version': numeric(data.get('MinimumOSVersion')), + }) + selected_sdk = numeric((metadata if metadata is not None else native.read()).get('toolchain', {}).get('sdk')) + result['selected_sdk_version'] = selected_sdk + for item in result['bundles']: + item['built_with_newer_sdk_than_selected'] = ( + version(item['sdk_version']) > version(selected_sdk)) if selected_sdk and item['sdk_version'] else None + item['minimum_os_exceeds_selected_sdk'] = ( + version(item['minimum_os_version']) > version(selected_sdk)) if selected_sdk and item['minimum_os_version'] else None + return result + + +def baseline(device): + """Check existing simulator app-service readiness before Maestro touches it.""" + facts = {'scope': 'Read-only simulator controls before hierarchy command.', 'commands': []} + ps, result = capture(['ps', '-axo', 'pid=,stat=,args='], 'baseline-processes', 10) + facts['commands'].append(result) + frontend = [line for line in ps.splitlines() if '/Simulator.app/Contents/MacOS/Simulator' in line] + known = process_inventory_known(ps, result) + facts['simulator_frontend_process_count'] = len(frontend) if known else None + selected = os.environ.get('DEVELOPER_DIR', '') + # Export only a boolean comparison, never the process arguments or path. + facts['selected_simulator_frontend_running'] = any( + selected + '/Applications/Simulator.app/Contents/MacOS/Simulator' in line for line in frontend + ) if selected and known else None + for args, label in [ + (['xcrun', 'simctl', 'listapps', device], 'baseline-listapps'), + (['xcrun', 'simctl', 'get_app_container', device, 'com.apple.Preferences', 'app'], + 'baseline-preferences-container'), + ]: + _, result = capture(args, label, 10) + facts['commands'].append(result) + return facts + + +def collect(device, metadata=None): + facts = {'scope': 'Before owned simulator cleanup; driver setup only.', + 'collection_errors': [], 'commands': [], 'logs': []} + metadata = metadata if metadata is not None else native.read() + + def command(args, label, timeout=20): + payload, result = capture(args, label, timeout) + facts['commands'].append(result) + return payload, result + + try: + facts['bundled_runner'] = bundled_runner(metadata) + except Exception as error: + facts['collection_errors'].append({'stage': 'bundled_runner', 'error_type': safe_error(error)}) + + ps, process_result = command(['ps', '-axo', 'pid=,stat=,args='], 'processes') + owned = [line for line in ps.splitlines() + if 'xcodebuild' in line and device in line and 'test-without-building' in line] + processes_known = process_inventory_known(ps, process_result) + facts['owned_xcodebuild_process_count'] = len(owned) if processes_known else None + facts['owned_xcodebuild_alive'] = bool(owned) if processes_known else None + facts['owned_xcodebuild_zombie_count'] = sum( + len(line.split(None, 2)) >= 2 and 'Z' in line.split(None, 2)[1] for line in owned) if processes_known else None + pids = [int(line.split()[0]) for line in owned if re.match(r'^\s*\d+\s', line)] if processes_known else [] + # Sample only the exact owned-device xcodebuild; never all runner processes. + if len(pids) == 1: + sample_path = native.PRIVATE / 'driver-xcodebuild.sample' + command(['sample', str(pids[0]), '3', '1', '-file', str(sample_path)], 'xcodebuild-sample', 15) + try: + payload, result = bounded(sample_path) + result['stack_categories'] = [name for name, pattern in STACK_MARKERS.items() + if re.search(pattern, payload, re.IGNORECASE)] + facts['xcodebuild_sample'] = result + except Exception as error: + facts['collection_errors'].append({'stage': 'xcodebuild_sample', 'error_type': safe_error(error)}) + + command(['xcodebuild', '-checkFirstLaunchStatus'], 'xcode-first-launch-status', 20) + command(['xcrun', 'simctl', 'spawn', device, '/usr/bin/true'], 'simulator-spawn-control', 10) + command(['xcrun', 'simctl', 'listapps', device], 'simulator-listapps', 20) + command(['log', 'show', '--style', 'compact', '--last', '5m', '--predicate', + 'process == "xcodebuild" OR process == "CoreSimulatorService"'], 'host-xcode-simulator-events', 25) + + _, container = command(['xcrun', 'simctl', 'get_app_container', device, RUNNER_ID, 'app'], 'runner-installed') + # Generic simctl failures do not establish that an app is absent. + facts['runner_installed'] = True if container.get('exit_code') == 0 else None + launchctl, launch_result = command(['xcrun', 'simctl', 'spawn', device, 'launchctl', 'list'], 'runner-running') + runner_lines = [line for line in launchctl.splitlines() if RUNNER_ID in line] + launch_rows = [line for line in launchctl.splitlines() if line.strip()] + launch_known = (bool(launch_rows) and launch_result.get('exit_code') == 0 and not launch_result.get('truncated', False) and + all(re.fullmatch(r'\s*(?:\d+|-)\s+-?\d+\s+\S+\s*', line) + or re.fullmatch(r'\s*PID\s+Status\s+Label\s*', line) for line in launch_rows)) + facts['runner_launchctl_entries'] = len(runner_lines) if launch_known else None + facts['runner_running'] = any(re.match(r'^\s*[1-9]\d*\s', line) for line in runner_lines) if launch_known else None + facts['runner_last_exit_codes'] = [int(match.group(1)) for line in runner_lines + if (match := re.match(r'^\s*(?:\d+|-)\s+(-?\d+)\s', line))][:4] if launch_known else [] + + paths = [] + for index, entry in enumerate(metadata['commands'], 1): + if entry['label'] == 'Check disposable iOS driver readiness': + paths.append(('hierarchy-command', native.PRIVATE / ('command-%03d.log' % index))) + state = native.PRIVATE / 'maestro-preflight-state' / 'maestro' / 'tests' + for pattern, kind in [('**/maestro.log', 'maestro'), ('**/xctest_runner_*.log', 'xctest')]: + paths.extend((kind, path) for path in sorted(state.glob(pattern))[:4]) + port = None + for kind, path in paths[:9]: + try: + payload, result = bounded(path) + facts['logs'].append({'kind': kind, **result}) + match = re.search(r'Running XcUITest with `xcodebuild test-without-building` with (\d+) and config:', payload) + if match and 0 < int(match.group(1)) <= 65535: + port = int(match.group(1)) + except Exception as error: + facts['collection_errors'].append({'stage': kind, 'error_type': safe_error(error)}) + facts['driver_port'] = port + if port is not None: + try: + with socket.create_connection(('127.0.0.1', port), timeout=2): + facts['driver_tcp_connects'] = True + except OSError: + facts['driver_tcp_connects'] = False + conn = http.client.HTTPConnection('127.0.0.1', port, timeout=3) + try: + conn.request('GET', '/status') + facts['driver_status_http_code'] = conn.getresponse().status + except Exception as error: + facts['driver_status_error_type'] = safe_error(error) + finally: + conn.close() + + predicate = 'process == "' + RUNNER_NAME + '" OR process == "testmanagerd"' + command(['xcrun', 'simctl', 'spawn', device, 'log', 'show', '--style', 'compact', + '--last', '5m', '--predicate', predicate], 'simulator-runner-events', 25) + crash_dirs = [Path.home() / 'Library/Logs/DiagnosticReports', + Path.home() / 'Library/Developer/CoreSimulator/Devices' / device / 'data/Library/Logs/CrashReporter'] + crash_count = 0 + for directory in crash_dirs: + if not directory.is_dir() or directory.is_symlink(): + continue + for path in sorted(directory.glob('*'), key=lambda item: item.name): + if crash_count >= 8: + break + if not path.is_file() or path.is_symlink() or not path.name.startswith((RUNNER_NAME, 'testmanagerd', 'xctest')): + continue + try: + if time.time() - path.stat().st_mtime > 600: + continue + with path.open('rb') as source: + payload = source.read(LIMIT) + facts.setdefault('recent_crash_reports', []).append({ + 'prefix_bytes': len(payload), 'truncated': path.stat().st_size > LIMIT, + 'categories': classify(payload.decode('utf-8', errors='replace')), + }) + crash_count += 1 + except Exception as error: + facts['collection_errors'].append({'stage': 'crash_report', 'error_type': safe_error(error)}) + return facts + + +def run(): + if native.PLATFORM != 'IOS' or os.environ.get('KEEPER_DIAGNOSTIC') != 'true': + raise ValueError('Expected explicit iOS diagnostic environment.') + os.environ['MAESTRO_CLI_NO_ANALYTICS'] = '1' + native.record() + native.install_maestro() + original = native.command + + def observed(args, label, **kwargs): + if label != 'Check disposable iOS driver readiness': + return original(args, label, **kwargs) + # Device is supplied by the unchanged owned-device driver path. + device = args[args.index('--device') + 1] + if not re.fullmatch(r'[A-Fa-f0-9]{8}(?:-[A-Fa-f0-9]{4}){3}-[A-Fa-f0-9]{12}', device): + raise ValueError('Expected owned simulator UUID.') + metadata = native.read() + # command() appends exactly this entry when its worker starts. Keep the + # index locally so collection never races its metadata file writes. + metadata['commands'].append({'label': label}) + baseline_facts = {'collection_errors': []} + try: + baseline_facts = baseline(device) + except Exception as error: + baseline_facts['collection_errors'].append({'stage': 'baseline', 'error_type': safe_error(error)}) + facts = {'collection_errors': []} + + def snapshot(phase, future): + started_while_running = not future.done() + try: + result = collect(device, metadata) + except Exception as error: + result = {'collection_errors': [{'stage': 'pre_cleanup', 'error_type': safe_error(error)}]} + result.update(snapshot_phase=phase, + hierarchy_running_before_collection=started_while_running, + hierarchy_running_after_collection=not future.done()) + return result + + primary_failed = False + try: + with ThreadPoolExecutor(max_workers=1) as executor: + future = executor.submit(original, args, label, **kwargs) + try: + result = future.result(timeout=POLL_SNAPSHOT_SECONDS) + except FutureTimeoutError: + # Sample the owned xcodebuild before Maestro's exception + # cleanup can delete its test products and derived data. + facts = snapshot('DURING_DRIVER_POLLING' if not future.done() else 'AFTER_DRIVER_COMMAND', future) + return future.result() + except Exception: + facts = snapshot('AFTER_DRIVER_COMMAND', future) + raise + else: + facts = snapshot('AFTER_DRIVER_COMMAND', future) + return result + except Exception: + primary_failed = True + raise + finally: + # Executor exit waits for the bounded original command. Persist only + # now, after command() has finished its own metadata writes. + facts['baseline'] = baseline_facts + try: + data = native.read() + data['driver_diagnosis'] = facts + native.save(data) + except Exception: + if not primary_failed: + raise RuntimeError('Driver diagnostic receipt failed.') from None + + native.command = observed + try: + native.run_phase('driver') + finally: + native.command = original + + +if __name__ == '__main__': + try: + run() + except Exception: + print('iOS driver diagnosis failed; inspect fixed metadata categories.', file=sys.stderr) + sys.exit(1) diff --git a/.github/native/history_fixture.yaml b/.github/native/history_fixture.yaml new file mode 100644 index 0000000000..21a0a24ecf --- /dev/null +++ b/.github/native/history_fixture.yaml @@ -0,0 +1,9 @@ +appId: ${APP_ID} +name: Controlled native history fixture stage +--- +- launchApp +- extendedWaitUntil: + visible: + id: ${EXPECTED_FIXTURE_ID} + timeout: 60000 +- stopApp diff --git a/.github/native/history_product.yaml b/.github/native/history_product.yaml new file mode 100644 index 0000000000..ee45e8e12f --- /dev/null +++ b/.github/native/history_product.yaml @@ -0,0 +1,47 @@ +# Normal index.js. Controlled Realm history fixture; account version is unchanged. +appId: ${APP_ID} +name: Current history observation after ordinary cold login +--- +- launchApp +- extendedWaitUntil: + visible: Enter your passcode + timeout: 60000 +- repeat: + times: 4 + commands: + - tapOn: + id: key_1 +- tapOn: Proceed +- extendedWaitUntil: + visible: + id: home_visible_wallet_count_1 + timeout: 60000 +- assertVisible: Mobile Wallet +- tapOn: + id: home_tab_more +- scrollUntilVisible: + element: + id: btn_setting_App Settings + direction: DOWN + timeout: 30000 +- tapOn: + id: btn_setting_App Settings +- extendedWaitUntil: + visible: + id: btn_setting_Version History + timeout: 30000 +- tapOn: + id: btn_setting_Version History +- extendedWaitUntil: + visible: + id: text_installedAppVersionLabel + text: Installed app version + timeout: 30000 +- assertVisible: + id: text_installedAppVersion + text: ${EXPECTED_INSTALLED_REGEX} +- assertVisible: Current installation observed +- assertVisible: + text: ${EXPECTED_HISTORY_REGEX} +- assertNotVisible: Upgraded from.* +- stopApp diff --git a/.github/native/history_restore.js b/.github/native/history_restore.js new file mode 100644 index 0000000000..dde0f75ca1 --- /dev/null +++ b/.github/native/history_restore.js @@ -0,0 +1,2 @@ +import { registerFixture } from './version_history_fixture'; +registerFixture('restore'); diff --git a/.github/native/history_restore_metro.cjs b/.github/native/history_restore_metro.cjs new file mode 100644 index 0000000000..b1cfff05c2 --- /dev/null +++ b/.github/native/history_restore_metro.cjs @@ -0,0 +1,15 @@ +// Fixed CI-only entrypoint. Never selected by the ordinary production runner. +const path = require('node:path'); +const base = require('../../metro.config'); +const enhance = base.server?.enhanceMiddleware; +module.exports = { + ...base, projectRoot: path.resolve(__dirname, '../..'), + server: { ...base.server, enhanceMiddleware(middleware, server) { + const inner = enhance ? enhance(middleware, server) : middleware; + return (req, res, next) => { + req.url = req.url.replace(/^\/index\.(bundle|map)(?=\?|$)/, + '/.github/native/history_restore.$1'); + return inner(req, res, next); + }; + } }, +}; diff --git a/.github/native/history_seed_empty.js b/.github/native/history_seed_empty.js new file mode 100644 index 0000000000..7ca99cd608 --- /dev/null +++ b/.github/native/history_seed_empty.js @@ -0,0 +1,2 @@ +import { registerFixture } from './version_history_fixture'; +registerFixture('seed_empty'); diff --git a/.github/native/history_seed_empty_metro.cjs b/.github/native/history_seed_empty_metro.cjs new file mode 100644 index 0000000000..ee52bcbecd --- /dev/null +++ b/.github/native/history_seed_empty_metro.cjs @@ -0,0 +1,15 @@ +// Fixed CI-only entrypoint. Never selected by the ordinary production runner. +const path = require('node:path'); +const base = require('../../metro.config'); +const enhance = base.server?.enhanceMiddleware; +module.exports = { + ...base, projectRoot: path.resolve(__dirname, '../..'), + server: { ...base.server, enhanceMiddleware(middleware, server) { + const inner = enhance ? enhance(middleware, server) : middleware; + return (req, res, next) => { + req.url = req.url.replace(/^\/index\.(bundle|map)(?=\?|$)/, + '/.github/native/history_seed_empty.$1'); + return inner(req, res, next); + }; + } }, +}; diff --git a/.github/native/history_seed_stale.js b/.github/native/history_seed_stale.js new file mode 100644 index 0000000000..06b6fe74b2 --- /dev/null +++ b/.github/native/history_seed_stale.js @@ -0,0 +1,2 @@ +import { registerFixture } from './version_history_fixture'; +registerFixture('seed_stale'); diff --git a/.github/native/history_seed_stale_metro.cjs b/.github/native/history_seed_stale_metro.cjs new file mode 100644 index 0000000000..aeca577b73 --- /dev/null +++ b/.github/native/history_seed_stale_metro.cjs @@ -0,0 +1,15 @@ +// Fixed CI-only entrypoint. Never selected by the ordinary production runner. +const path = require('node:path'); +const base = require('../../metro.config'); +const enhance = base.server?.enhanceMiddleware; +module.exports = { + ...base, projectRoot: path.resolve(__dirname, '../..'), + server: { ...base.server, enhanceMiddleware(middleware, server) { + const inner = enhance ? enhance(middleware, server) : middleware; + return (req, res, next) => { + req.url = req.url.replace(/^\/index\.(bundle|map)(?=\?|$)/, + '/.github/native/history_seed_stale.$1'); + return inner(req, res, next); + }; + } }, +}; diff --git a/.github/native/history_verify_empty_first.js b/.github/native/history_verify_empty_first.js new file mode 100644 index 0000000000..f4635c1605 --- /dev/null +++ b/.github/native/history_verify_empty_first.js @@ -0,0 +1,2 @@ +import { registerFixture } from './version_history_fixture'; +registerFixture('verify_empty_first'); diff --git a/.github/native/history_verify_empty_first_metro.cjs b/.github/native/history_verify_empty_first_metro.cjs new file mode 100644 index 0000000000..44e0466a9f --- /dev/null +++ b/.github/native/history_verify_empty_first_metro.cjs @@ -0,0 +1,15 @@ +// Fixed CI-only entrypoint. Never selected by the ordinary production runner. +const path = require('node:path'); +const base = require('../../metro.config'); +const enhance = base.server?.enhanceMiddleware; +module.exports = { + ...base, projectRoot: path.resolve(__dirname, '../..'), + server: { ...base.server, enhanceMiddleware(middleware, server) { + const inner = enhance ? enhance(middleware, server) : middleware; + return (req, res, next) => { + req.url = req.url.replace(/^\/index\.(bundle|map)(?=\?|$)/, + '/.github/native/history_verify_empty_first.$1'); + return inner(req, res, next); + }; + } }, +}; diff --git a/.github/native/history_verify_empty_repeat.js b/.github/native/history_verify_empty_repeat.js new file mode 100644 index 0000000000..00d02f7610 --- /dev/null +++ b/.github/native/history_verify_empty_repeat.js @@ -0,0 +1,2 @@ +import { registerFixture } from './version_history_fixture'; +registerFixture('verify_empty_repeat'); diff --git a/.github/native/history_verify_empty_repeat_metro.cjs b/.github/native/history_verify_empty_repeat_metro.cjs new file mode 100644 index 0000000000..d940701829 --- /dev/null +++ b/.github/native/history_verify_empty_repeat_metro.cjs @@ -0,0 +1,15 @@ +// Fixed CI-only entrypoint. Never selected by the ordinary production runner. +const path = require('node:path'); +const base = require('../../metro.config'); +const enhance = base.server?.enhanceMiddleware; +module.exports = { + ...base, projectRoot: path.resolve(__dirname, '../..'), + server: { ...base.server, enhanceMiddleware(middleware, server) { + const inner = enhance ? enhance(middleware, server) : middleware; + return (req, res, next) => { + req.url = req.url.replace(/^\/index\.(bundle|map)(?=\?|$)/, + '/.github/native/history_verify_empty_repeat.$1'); + return inner(req, res, next); + }; + } }, +}; diff --git a/.github/native/history_verify_restored.js b/.github/native/history_verify_restored.js new file mode 100644 index 0000000000..03fa848bcd --- /dev/null +++ b/.github/native/history_verify_restored.js @@ -0,0 +1,2 @@ +import { registerFixture } from './version_history_fixture'; +registerFixture('verify_restored'); diff --git a/.github/native/history_verify_restored_metro.cjs b/.github/native/history_verify_restored_metro.cjs new file mode 100644 index 0000000000..5ff8b87336 --- /dev/null +++ b/.github/native/history_verify_restored_metro.cjs @@ -0,0 +1,15 @@ +// Fixed CI-only entrypoint. Never selected by the ordinary production runner. +const path = require('node:path'); +const base = require('../../metro.config'); +const enhance = base.server?.enhanceMiddleware; +module.exports = { + ...base, projectRoot: path.resolve(__dirname, '../..'), + server: { ...base.server, enhanceMiddleware(middleware, server) { + const inner = enhance ? enhance(middleware, server) : middleware; + return (req, res, next) => { + req.url = req.url.replace(/^\/index\.(bundle|map)(?=\?|$)/, + '/.github/native/history_verify_restored.$1'); + return inner(req, res, next); + }; + } }, +}; diff --git a/.github/native/history_verify_stale_first.js b/.github/native/history_verify_stale_first.js new file mode 100644 index 0000000000..ed24dd862b --- /dev/null +++ b/.github/native/history_verify_stale_first.js @@ -0,0 +1,2 @@ +import { registerFixture } from './version_history_fixture'; +registerFixture('verify_stale_first'); diff --git a/.github/native/history_verify_stale_first_metro.cjs b/.github/native/history_verify_stale_first_metro.cjs new file mode 100644 index 0000000000..b682b42781 --- /dev/null +++ b/.github/native/history_verify_stale_first_metro.cjs @@ -0,0 +1,15 @@ +// Fixed CI-only entrypoint. Never selected by the ordinary production runner. +const path = require('node:path'); +const base = require('../../metro.config'); +const enhance = base.server?.enhanceMiddleware; +module.exports = { + ...base, projectRoot: path.resolve(__dirname, '../..'), + server: { ...base.server, enhanceMiddleware(middleware, server) { + const inner = enhance ? enhance(middleware, server) : middleware; + return (req, res, next) => { + req.url = req.url.replace(/^\/index\.(bundle|map)(?=\?|$)/, + '/.github/native/history_verify_stale_first.$1'); + return inner(req, res, next); + }; + } }, +}; diff --git a/.github/native/history_verify_stale_repeat.js b/.github/native/history_verify_stale_repeat.js new file mode 100644 index 0000000000..057050ee9a --- /dev/null +++ b/.github/native/history_verify_stale_repeat.js @@ -0,0 +1,2 @@ +import { registerFixture } from './version_history_fixture'; +registerFixture('verify_stale_repeat'); diff --git a/.github/native/history_verify_stale_repeat_metro.cjs b/.github/native/history_verify_stale_repeat_metro.cjs new file mode 100644 index 0000000000..9c347d3dbc --- /dev/null +++ b/.github/native/history_verify_stale_repeat_metro.cjs @@ -0,0 +1,15 @@ +// Fixed CI-only entrypoint. Never selected by the ordinary production runner. +const path = require('node:path'); +const base = require('../../metro.config'); +const enhance = base.server?.enhanceMiddleware; +module.exports = { + ...base, projectRoot: path.resolve(__dirname, '../..'), + server: { ...base.server, enhanceMiddleware(middleware, server) { + const inner = enhance ? enhance(middleware, server) : middleware; + return (req, res, next) => { + req.url = req.url.replace(/^\/index\.(bundle|map)(?=\?|$)/, + '/.github/native/history_verify_stale_repeat.$1'); + return inner(req, res, next); + }; + } }, +}; diff --git a/.github/native/hosted_native.py b/.github/native/hosted_native.py new file mode 100644 index 0000000000..37831c453d --- /dev/null +++ b/.github/native/hosted_native.py @@ -0,0 +1,1529 @@ +#!/usr/bin/env python3 +"""Disposable hosted smoke harness; only allowlisted metadata leaves the runner. + +Native and Maestro output stays under RUNNER_TEMP, never in an uploaded glob. +The gate trusts protected workflow/job provenance, never this writable artifact. +""" + +import hashlib +import importlib.util +import json +import math +import os +from pathlib import Path +import platform +import re +import shutil +import signal +import stat +import subprocess +import sys +import tempfile +import time +from types import SimpleNamespace +import urllib.request +import xml.etree.ElementTree as ET + + +ROOT = Path(__file__).resolve().parents[2] +PLATFORM = os.environ.get('KEEPER_NATIVE_PLATFORM', '') +PRIVATE = Path(os.environ.get('RUNNER_TEMP', tempfile.gettempdir())) / 'keeper-native-private' +EVIDENCE = ROOT / 'artifacts' / 'native' / PLATFORM.lower() / 'metadata.json' +MAESTRO_VERSION = 'cli-2.11.0' +MAESTRO_SHA256 = '5384593cb4e7a106489e75a821d157dd43f4e438df6bc308b72e82c685e1283a' +APP_IDS = {'IOS': 'io.hexawallet.hexakeeper.dev', 'ANDROID': 'io.hexawallet.keeper.development'} + + +def save(data): + EVIDENCE.parent.mkdir(parents=True, exist_ok=True) + EVIDENCE.write_text(json.dumps(data, indent=2) + '\n') + + +def read(): + return json.loads(EVIDENCE.read_text()) + + +def failure_details(log): + """Classify observed tool failures; never return any part of a log line. + + Redaction cannot recognize arbitrary wallet, user or application text. Each + result must come from this fixed vocabulary, including the fallback. + """ + categories = ( + (r'lockfile needs to be updated|could not find compatible versions|^The sandbox is not in sync', + 'DEPENDENCY_LOCK_MISMATCH'), + (r'no space left on device', 'DISK_SPACE_EXHAUSTED'), + (r'command not found', 'REQUIRED_TOOL_UNAVAILABLE'), + (r'requires a higher minimum deployment target', 'IOS_DEPLOYMENT_TARGET_UNSUPPORTED'), + (r'^e: ', 'KOTLIN_COMPILER_ERROR'), + (r'^\[!\]', 'COCOAPODS_ERROR'), + (r'^FAILURE:|^A problem occurred|^Execution failed', 'BUILD_FAILURE'), + (r'^Could not ', 'DEPENDENCY_OR_TOOL_FAILURE'), + (r'\berror:|^error |^> (?!Task\b)|^Caused by:', 'TOOL_ERROR'), + ) + selected = [] + for raw in log.splitlines(): + line = re.sub(r'\x1b\[[0-9;]*[A-Za-z]', '', raw).strip() + if re.match(r'^\d+\s*\|', line): + continue + for pattern, category in categories: + if re.search(pattern, line, re.IGNORECASE): + if category not in selected: + selected.append(category) + break + return selected + + +def maestro_failure_details(log): + """Retain only fixed command labels, selectors and tool failure categories.""" + selected = [] + for raw in log.splitlines(): + line = re.sub(r'\x1b\[[0-9;]*[A-Za-z]', '', raw).strip() + if line.endswith('FAILED'): + if line.startswith(('Launch app', 'Clear state', 'Stop app')) and any(app in line for app in APP_IDS.values()): + selected.append('Maestro app launch/stop command FAILED') + elif line.startswith(('Assert', 'Wait')) and 'Create a passcode' in line: + selected.append('Maestro assertion: Create a passcode visible FAILED') + elif line.startswith(('Assert', 'Wait')) and 'Confirm your passcode' in line: + selected.append('Maestro assertion: Confirm your passcode state FAILED') + elif line.startswith('Tap on'): + match = re.search(r'\bkey_[1-4]\b', line) + if match: + selected.append('Maestro tap %s FAILED' % match.group()) + elif line.startswith(('Assertion is false:', 'Element not found:')): + # Emit only a category and fixed selector; even a first-line suffix + # can contain a hierarchy or arbitrary device text. + category = 'assertion failed' if line.startswith('Assertion is false:') else 'element not found' + for target in ['Create a passcode', 'Confirm your passcode', 'key_1', 'key_2', 'key_3', 'key_4']: + if target in line: + selected.append('Maestro %s: %s' % (category, target)) + break + elif line.startswith(('Unable to launch app', 'Failed to launch app')): + selected.append('Maestro app launch failed') + elif line.startswith(('Unable to connect', 'Failed to connect')): + selected.append('Maestro connection failed') + elif line.startswith('Failed to start'): + selected.append('Maestro driver start failed') + elif line.startswith(('No devices found', 'No connected devices', 'Unable to find device')): + selected.append('Maestro device unavailable') + if len(selected) >= 10: + break + return list(dict.fromkeys(selected))[:10] + + +def android_startup_errors(log): + categories = {'AndroidRuntime': 'ANDROID_RUNTIME_ERROR', + 'ReactNativeJS': 'REACT_NATIVE_JS_ERROR', 'ReactNative': 'REACT_NATIVE_ERROR'} + selected = [] + for line in log.splitlines(): + match = re.match(r'^[EF]/(AndroidRuntime|ReactNativeJS|ReactNative)\s*\(\s*\d+\):\s*(.*)', line) + if not match: + continue + tag, message = match.groups() + if re.search(r'(?:FATAL EXCEPTION|Caused by:|\b[\w.$]*(?:Exception|Error)\b|' + r'Invariant Violation|Unable to load script|Could not)', message): + if categories[tag] not in selected: + selected.append(categories[tag]) + return selected + + +def metro_startup_errors(log): + selected = [] + for raw in log.splitlines(): + line = re.sub(r'\x1b\[[0-9;]*[A-Za-z]', '', raw).strip() + if re.match(r'^(?:error\b|(?:Syntax|Type|Reference)Error:|Unable to resolve module)', line, re.IGNORECASE): + category = 'METRO_ERROR' + for marker, known in [('Unable to resolve module', 'METRO_MODULE_RESOLUTION_ERROR'), + ('SyntaxError:', 'METRO_SYNTAX_ERROR'), + ('TypeError:', 'METRO_TYPE_ERROR'), + ('ReferenceError:', 'METRO_REFERENCE_ERROR')]: + if marker.lower() in line.lower(): + category = known + break + if category not in selected: + selected.append(category) + return selected + + +def metro_startup_progress(log): + # Metro's TerminalReporter uses the same bare BUNDLE line for done and + # failed builds. Count observed markers without inferring completion. + lines = [re.sub(r'\x1b\[[0-9;]*[A-Za-z]', '', line).strip() for line in log.splitlines()] + bundles = sum(bool(re.match(r'^BUNDLE\s', line)) for line in lines) + prebundles = sum(bool(re.match(r'^PREBUNDLE\s', line)) for line in lines) + return {'bundle_marker_observed': bundles > 0, 'bundle_marker_count': bundles, + 'prebundle_marker_count': prebundles} + + +def ios_build_progress(log_path: Path) -> dict: + """Observe bounded failure output; unknown markers prove no absence or rate.""" + phases = { + b'CompileC': 'C_COMPILE', b'SwiftCompile': 'SWIFT_COMPILE', + b'SwiftEmitModule': 'SWIFT_MODULE', b'Ld': 'LINK', b'Libtool': 'ARCHIVE', + b'PhaseScriptExecution': 'SCRIPT', b'CompileAssetCatalog': 'RESOURCE', + b'CompileAssetCatalogVariant': 'RESOURCE', b'CompileStoryboard': 'RESOURCE', + b'CompileXIB': 'RESOURCE', b'CompileXCStrings': 'RESOURCE', + b'Copy': 'COPY', b'CopyPlistFile': 'COPY', b'CopyStringsFile': 'COPY', + b'CopySwiftLibs': 'COPY', b'CpResource': 'COPY', b'PBXCp': 'COPY', + b'CodeSign': 'SIGN', + } + result = {'schema': 1, 'status': 'UNAVAILABLE', 'log_bytes': None, 'tail_bytes': 0, + 'input_truncated': False, 'line_scan_truncated': False, 'lines_scanned': 0, + 'terminal_record_incomplete': False, + 'phase_counts': dict.fromkeys(phases.values(), 0), + 'last_phase': 'UNKNOWN', + 'scope': 'FAILURE_ONLY_BOUNDED_XCODE_TAIL_NOT_PROGRESS_RATE_CAUSE_OR_ACCEPTANCE'} + try: + log_path = Path(log_path) + if log_path.parent != PRIVATE: + return result + # Descriptor-relative no-follow opens reject links and keep a replaced + # PRIVATE path from redirecting the read. Nonblocking rejects FIFOs safely. + directory = os.open(PRIVATE, os.O_RDONLY | os.O_DIRECTORY | os.O_NOFOLLOW) + try: + if os.fstat(directory).st_uid != os.getuid(): + return result + descriptor = os.open(log_path.name, os.O_RDONLY | os.O_NOFOLLOW | os.O_NONBLOCK, + dir_fd=directory) + finally: + os.close(directory) + with os.fdopen(descriptor, 'rb', buffering=0) as source: + info = os.fstat(source.fileno()) + if not stat.S_ISREG(info.st_mode) or info.st_uid != os.getuid(): + return result + source.seek(max(0, info.st_size - 65536)) + payload = source.read(65536) + tail_bytes = len(payload) + truncated = info.st_size > 65536 + terminal_incomplete = bool(payload) and not payload.endswith(b'\n') + if truncated: + # The bounded read may start mid-record; discard that first record. + payload = payload.partition(b'\n')[2] + if terminal_incomplete: + payload = payload.rpartition(b'\n')[0] + lines = payload.splitlines() + line_truncated = len(lines) > 1024 + lines = lines[-1024:] + counts = result['phase_counts'].copy() + last = 'UNKNOWN' + for line in lines: + match = re.match(rb'^([A-Za-z][A-Za-z0-9]*)(?:[ \t]|$)', line) + phase = phases.get(match[1]) if match else None + if phase: + counts[phase] += 1 + last = phase + result.update(status='OBSERVED', log_bytes=info.st_size, tail_bytes=tail_bytes, + input_truncated=truncated, line_scan_truncated=line_truncated, + terminal_record_incomplete=terminal_incomplete, + lines_scanned=len(lines), phase_counts=counts, last_phase=last) + except Exception: + pass + return result + + +def private_log_prefix(path): + """Read launch-time output, discarding a partially truncated final record.""" + limit = 1024 * 1024 + with path.open('rb') as source: + payload = source.read(limit + 1) + truncated = len(payload) > limit + payload = payload[:limit] + if truncated: + payload = payload.rsplit(b'\n', 1)[0] + b'\n' if b'\n' in payload else b'' + return payload.decode(errors='replace'), truncated + + +def ios_startup_diagnostics(payload, executable, started_ms=None, ended_ms=None, input_partial=False): + # Pure fixed-vocabulary classifier; no source message, stack or path leaves + # the fresh owned simulator's private compact-log query. + spec = importlib.util.spec_from_file_location( + 'ios_startup_errors', Path(__file__).with_name('ios_startup_errors.py')) + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module.classify(payload, executable, started_ms, ended_ms, input_partial) + + +def startup_hierarchy_signals(directory): + """Classify fixed public startup selectors; never export hierarchy content.""" + # Maestro cli-2.11.0 writes standalone TreeNode snapshots in this layout. + flags = ['createPIN', 'confirmPIN', 'RNredbox', 'noScriptURL', + 'developmentServerError', 'unableLoadScript'] + result = {flag: False for flag in flags} + result.update({'snapshot_count': 0, 'parse_error_types': []}) + redbox_ids = {'redbox-error', 'redbox-dismiss', 'redbox-reload', 'rn_redbox_stack', + 'rn_redbox_dismiss_button', 'rn_redbox_reload_button', 'catalyst_redbox_title'} + fixed_messages = {'noScriptURL': 'No script URL provided.', + 'developmentServerError': 'Could not connect to development server.', + 'unableLoadScript': 'Unable to load script.'} + for index, path in enumerate(directory.glob('**/screen-hierarchy/step-*.json')): + try: + if index >= 8: + raise ValueError('Hierarchy snapshot count exceeded limit.') + result['snapshot_count'] += 1 + if path.is_symlink() or not path.resolve().is_relative_to(directory.resolve()): + raise ValueError('Hierarchy snapshot must remain in private output.') + with path.open('rb') as source: + payload = source.read(1024 * 1024 + 1) + if len(payload) > 1024 * 1024: + raise ValueError('Hierarchy snapshot exceeded byte limit.') + pending = [json.loads(payload)] + snapshot = {flag: False for flag in flags} + nodes = 0 + while pending: + node = pending.pop() + nodes += 1 + if nodes > 10000: + raise ValueError('Hierarchy snapshot exceeded node limit.') + if not isinstance(node, dict): + raise TypeError('Unexpected hierarchy node format.') + attributes = node.get('attributes') or {} + children = node.get('children') or [] + if not isinstance(attributes, dict) or not isinstance(children, list): + raise TypeError('Unexpected hierarchy attributes or children format.') + # iOS StaticText often exposes only accessibilityText. Android + # resource IDs include the package prefix before :id/. + identifier = attributes.get('resource-id', '') + if isinstance(identifier, str) and identifier.rsplit(':id/', 1)[-1] in redbox_ids: + snapshot['RNredbox'] = True + for key in ['accessibilityText', 'text', 'title', 'value']: + value = attributes.get(key) + if not isinstance(value, str): + continue + snapshot['createPIN'] |= value.strip() == 'Create a passcode' + snapshot['confirmPIN'] |= value.strip() == 'Confirm your passcode' + for flag, message in fixed_messages.items(): + snapshot[flag] |= message in value + if len(pending) + len(children) + nodes > 10000: + raise ValueError('Hierarchy snapshot exceeded node limit.') + pending.extend(children) + for flag in flags: + result[flag] |= snapshot[flag] + except Exception as error: + kind = type(error).__name__ + if kind not in result['parse_error_types']: + result['parse_error_types'].append(kind) + if index >= 8: + break + return result + + +def maestro_preflight_diagnostics(directory=None, command_label='Assert startup and disposable passcode transition', owned_device=None): + """Export fixed cli-2.11.0 driver markers, never subprocess error text.""" + markers = { + 'runner_install_started': '[Start] Install XCUITest runner on', + 'runner_install_launch_returned': '[Done] Install XCUITest runner on', + 'status_check_started': '[Start] Perform XCUITest driver status check on', + 'status_response_received': '[Done] Perform XCUITest driver status check on', + 'status_check_failed': '[Failed] Perform XCUITest driver status check on', + 'driver_startup_timeout': 'iOS driver not ready in time, consider increasing timeout by configuring MAESTRO_DRIVER_STARTUP_TIMEOUT env variable', + 'prebuilt_driver_selected': 'Installing pre built driver without xcodebuild', + 'xcodebuild_driver_selected': 'Installing driver with xcodebuild', + 'xctestrun_config_missing': 'xctestrun config does not exist', + 'ui_test_runner_missing': 'ui test runner does not exist', + 'xcode_test_execute_failed': '** TEST EXECUTE FAILED **', + 'xcode_test_failed': '** TEST FAILED **', + 'xcode_testing_failed': 'Testing failed:', + } + if PLATFORM == 'IOS': + # Maestro 2.11 performs clearState and permissions inside launchApp. + # These log markers establish observation only; in particular its + # "reinstalled and launched" text does not prove an app launch. + app = APP_IDS['IOS'] + markers.update({ + 'app_clear_state_started_text': 'Clearing app %s state' % app, + 'app_reinstall_started_text': 'Reinstalling and launching %s' % app, + 'app_reinstall_returned_text': 'App %s reinstalled and launched' % app, + 'app_permissions_applesimutils_started_text': '[Start] Setting permissions all=deny through applesimutils', + 'app_permissions_applesimutils_returned_text': '[Done] Setting permissions through applesimutils', + 'app_permissions_simctl_started_text': '[Start] Setting Permissions all=deny through simctl', + 'app_permissions_simctl_returned_text': '[Done] Setting Permissions all=deny through simctl', + }) + if type(owned_device) is str and re.fullmatch(r'[A-Fa-f0-9]{8}(?:-[A-Fa-f0-9]{4}){3}-[A-Fa-f0-9]{12}', owned_device): + markers['app_simctl_launch_invocation_text'] = ( + 'Running command line operation: [xcrun, simctl, launch, %s, %s] with {}' % (owned_device, app)) + # The install/status "Done" markers precede readiness evaluation; they do + # not prove that the runner installed successfully or became ready. + result = {name: False for name in markers} + result.update({'host_port_unavailable': False, 'requested_device_disconnected': False, + 'exception_types': [], 'logs_read': 0, 'truncated_logs': 0, + 'log_limit_reached': False, 'collection_error_types': []}) + known_types = ['IOSDriverTimeoutException', 'FileNotFoundException', 'CliError', + 'IllegalStateException', 'IOException', 'ConnectException', + 'SocketTimeoutException', 'TimeoutException', 'RuntimeException', + 'UnknownHostException', 'SSLHandshakeException'] + paths = [] + for index, entry in enumerate(read()['commands'], 1): + if entry['label'] == command_label: + paths.append(PRIVATE / ('command-%03d.log' % index)) + directory = directory if directory is not None else PRIVATE / 'maestro-output' + if not directory.resolve().is_relative_to(PRIVATE.resolve()): + raise ValueError('Maestro diagnostic directory must remain private.') + for pattern in ['**/maestro.log', '**/xctest_runner_*.log']: + for path in directory.glob(pattern): + paths.append(path) + if len(paths) > 6: + break + if len(paths) > 6: + break + for path in paths[:6]: + try: + if path.is_symlink() or not path.resolve().is_relative_to(PRIVATE.resolve()): + raise ValueError('Maestro log must remain in private output.') + payload, truncated = private_log_prefix(path) + result['logs_read'] += 1 + result['truncated_logs'] += int(truncated) + for name, marker in markers.items(): + result[name] |= marker in payload + for line in payload.splitlines(): + result['host_port_unavailable'] |= ('Requested driver host port ' in line + and ' is not available' in line) + result['requested_device_disconnected'] |= (' was requested, but it is not connected.' in line) + for kind in known_types: + if re.search(r'(? 6 + return result + + +def record_startup_failure(report, device, startup_started_ms=None): + """Collect bounded diagnostics before destroying this job's fresh app state.""" + details = {'scope': 'Fresh startup only; no account or persisted PIN created.', + 'collection_errors': []} + try: + startup_ended_ms = int(time.time() * 1000) + except Exception: + startup_ended_ms = None + + def collect(name, action): + try: + details[name] = action() + except Exception as error: + details['collection_errors'].append({'stage': name, 'error_type': type(error).__name__}) + + def private_text(path): + # No raw logs/reports are copied to the artifact directory. + with path.open('rb') as source: + source.seek(0, os.SEEK_END) + source.seek(max(0, source.tell() - 1024 * 1024)) + return source.read().decode(errors='replace') + + def maestro_details(): + result = {'failed_commands': [], 'testcase_failures': [], 'report': 'MISSING'} + for index, entry in enumerate(read()['commands'], 1): + if entry['label'] == 'Assert startup and disposable passcode transition': + result['failed_commands'] = maestro_failure_details(private_text(PRIVATE / ('command-%03d.log' % index))) + if report.exists(): + if report.stat().st_size > 1024 * 1024: + raise RuntimeError('Maestro report exceeded diagnostic size limit.') + suite = ET.parse(report).getroot() + result['report'] = 'PRESENT' + result['testcase_statuses'] = [case.get('status') if case.get('status') in + ['SUCCESS', 'ERROR', 'WARNING', 'SKIPPED'] else 'UNKNOWN' + for case in list(suite.iter('testcase'))[:10]] + for failure in list(suite.iter('failure'))[:10]: + # Pinned Maestro JUnit stores exception.message as failure XML + # text, separately from UI hierarchy; it has no type attribute. + first_line = ''.join(failure.itertext()).splitlines()[:1] + result['testcase_failures'].extend(maestro_failure_details('\n'.join(first_line))) + return result + + collect('maestro', maestro_details) + collect('maestro_preflight', lambda: maestro_preflight_diagnostics(owned_device=device)) + collect('hierarchy_signals', lambda: startup_hierarchy_signals(PRIVATE / 'maestro-output')) + collect('metro_errors', lambda: metro_startup_errors(private_text(PRIVATE / 'metro.log'))) + collect('metro_progress', lambda: metro_startup_progress(private_log_prefix(PRIVATE / 'metro.log')[0])) + if PLATFORM == 'IOS' and device: + def ios_command_observation(): + spec = importlib.util.spec_from_file_location('ios_startup_command_observation', + Path(__file__).with_name('ios_startup_command_observation.py')) + observer = importlib.util.module_from_spec(spec) + spec.loader.exec_module(observer) + return observer.collect(PRIVATE, device, startup_started_ms, startup_ended_ms) + collect('command_observation', ios_command_observation) + def ios_replacement_observation(): + spec = importlib.util.spec_from_file_location('ios_replacement_observation', + Path(__file__).with_name('ios_replacement_observation.py')) + observer = importlib.util.module_from_spec(spec) + spec.loader.exec_module(observer) + return observer.collect(SimpleNamespace(PRIVATE=PRIVATE, PLATFORM=PLATFORM, + read=read, artifact_identity=artifact_identity, + _bundle_inventory=_bundle_inventory, owned_ios_bundle_path=owned_ios_bundle_path), device) + collect('replacement_observation', ios_replacement_observation) + if PLATFORM == 'ANDROID' and device: + def android_details(): + args = ['adb', '-s', device, 'logcat', '-d', '-t', '300', '-v', 'brief', + 'AndroidRuntime:E', 'ReactNativeJS:E', 'ReactNative:E', '*:S'] + path = PRIVATE / 'android-startup-errors.log' + with path.open('wb') as output: + try: + result = subprocess.run(args, stdout=output, stderr=subprocess.STDOUT, + timeout=20, check=False) + code = result.returncode + except subprocess.TimeoutExpired: + code = 124 + return {'command': args, 'exit_code': code, + 'errors': android_startup_errors(private_text(path)) if code == 0 else []} + collect('android_errors', android_details) + def android_runtime_details(): + spec = importlib.util.spec_from_file_location( + 'android_startup_diagnosis', Path(__file__).with_name('android_startup_diagnosis.py')) + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module.collect(PRIVATE, device) + collect('android_runtime_diagnosis', android_runtime_details) + elif PLATFORM == 'IOS' and device: + def ios_details(): + executable = read()['built_artifact']['executable'] + if not re.fullmatch(r'[A-Za-z0-9_][A-Za-z0-9_.-]*', executable): + raise ValueError('Unsafe iOS executable selector.') + predicate = ('process == "%s" AND (messageType == "error" OR messageType == "fault" ' + 'OR eventMessage CONTAINS[c] "error" OR eventMessage CONTAINS[c] "exception" ' + 'OR eventMessage CONTAINS[c] "fatal")') % executable + # Retain INFO for existing bounded diagnostics. The single new + # direct-hook failure marker uses RN's OS_LOG_TYPE_ERROR sink; + # actual marker routing still needs exact-source native proof. + args = ['xcrun', 'simctl', 'spawn', device, 'log', 'show', '--info', '--style', 'compact', + '--last', '10m', '--predicate', predicate] + path = PRIVATE / 'ios-startup-errors.log' + # Keep log/xcrun notices private and separate from compact records. + with path.open('wb') as output, (PRIVATE / 'ios-startup-errors-stderr.log').open('wb') as stderr: + try: + result = subprocess.run(args, stdout=output, stderr=stderr, + timeout=25, check=False) + code = result.returncode + except subprocess.TimeoutExpired: + code = 124 + result = {'command': args, 'exit_code': code, 'errors': [], + 'query_status': 'SUCCESS' if code == 0 else 'TIMEOUT' if code == 124 else 'FAILED', + 'parse_status': 'UNKNOWN', 'cause': 'UNDETERMINED', 'input_truncated': None, + 'startup_failure_receipt': {'schema': 1, 'status': 'UNKNOWN', 'records': [], + 'cause': 'UNDETERMINED', 'reasons': ['QUERY_UNAVAILABLE'], + 'scope': 'OPTIONAL_ORIGINAL_STARTUP_MARKERS_NOT_CAUSE_OR_ACCEPTANCE'}} + if code == 0: + try: + payload, truncated = private_log_prefix(path) + result.update(ios_startup_diagnostics(payload, executable, startup_started_ms, + startup_ended_ms, input_partial=truncated)) + result['input_truncated'] |= truncated + if truncated: + result['parse_status'] = 'PARTIAL' if result['matching_process_records'] else 'UNKNOWN' + except (ValueError, TypeError) as error: + result['parse_error'] = type(error).__name__ + return result + collect('ios_errors', ios_details) + data = read() + data['startup_diagnostics'] = details + save(data) + print('Bounded startup diagnostics: ' + json.dumps(details), flush=True) + + +def command(args, label, timeout=120, cwd=ROOT, capture=False, error_details=True): + """Record exact nonsecret argv and status; keep all process output private.""" + PRIVATE.mkdir(parents=True, exist_ok=True) + data = read() + child_env = None + maestro = data.get('maestro', {}) + if PLATFORM == 'IOS' and args and args[0] == maestro.get('binary'): + if maestro.get('version') != MAESTRO_VERSION or maestro.get('sha256') != MAESTRO_SHA256: + raise ValueError('Expected the verified pinned Maestro.') + # cli-2.11.0's own macos-15-intel CI uses 240000ms, while its SDK + # defaults to 120000ms. Budget only the extra driver setup time; + # flow waits, product metrics and Android remain unchanged. + child_env = os.environ.copy() + child_env['MAESTRO_DRIVER_STARTUP_TIMEOUT'] = '240000' + driver_setup = {'configured_timeout_ms': 240000, 'sdk_default_timeout_ms': 120000, + 'outer_setup_margin_seconds': 120, 'base_process_timeout_seconds': timeout, + 'scope': 'IOS_DRIVER_SETUP_ONLY_NOT_UI_WAIT_OR_PERFORMANCE'} + timeout += 120 + entry = {'label': label, 'command': args, 'timeout_seconds': timeout} + if child_env is not None: + entry['maestro_driver_setup'] = driver_setup + print('IOS_MAESTRO_DRIVER_SETUP_BUDGET_MS=240000;OUTER_SETUP_MARGIN_SECONDS=120', flush=True) + data['commands'].append(entry) + save(data) + started = time.monotonic() + output_file = PRIVATE / ('command-%03d.log' % len(data['commands'])) + print(label, flush=True) + try: + with output_file.open('wb') as output: + proc = subprocess.Popen(args, cwd=cwd, stdout=output, stderr=subprocess.STDOUT, + start_new_session=True, env=child_env) + try: + code = proc.wait(timeout=timeout) + except subprocess.TimeoutExpired: + os.killpg(proc.pid, signal.SIGKILL) + proc.wait() + code = 124 + except OSError: + code = 127 + entry['exit_code'] = code + entry['duration_seconds'] = round(time.monotonic() - started, 1) + if code: + # Fixed diagnostics expose install blockers without publishing arbitrary logs. + log = output_file.read_text(errors='replace') if output_file.exists() else '' + known = { + 'lockfile needs to be updated': 'Manifest and yarn.lock are inconsistent; repair the lockfile separately.', + 'could not find compatible versions': 'The native dependency lock is incompatible with the manifest.', + 'no space left on device': 'The hosted runner ran out of disk space.', + 'command not found': 'A required native tool is unavailable.', + 'requires a higher minimum deployment target': 'A pod requires a higher iOS deployment target.', + } + entry['diagnostic'] = next((message for needle, message in known.items() + if needle in log.lower()), 'Inspect this command locally; raw output remains private to the runner.') + # Keep the schema-1 field name for consumers; values are now fixed + # categories, never redacted excerpts or arbitrary exception messages. + entry['redacted_tool_errors'] = failure_details(log) if error_details else [] + for line in entry['redacted_tool_errors']: + print(line, flush=True) + save(data) + raise RuntimeError('%s failed (exit %s). %s' % (label, code, entry['diagnostic'])) + save(data) + return output_file.read_text(errors='replace').strip() if capture else None + + +def dependency_command(args, label, stage, timeout=900, cwd=ROOT): + """Observe the existing private output before it disappears with the runner.""" + if stage not in ('RUBY_BUNDLE', 'RUBY_CONTEXT', 'PODS_DEPLOYMENT'): + raise ValueError('Unsupported dependency observation stage.') + spec = importlib.util.spec_from_file_location('pod_diagnostics', + Path(__file__).with_name('pod_diagnostics.py')) + observer = importlib.util.module_from_spec(spec) + spec.loader.exec_module(observer) + before = observer.lock_state(ROOT) + pods = observer.locked_pods(ROOT) + index = len(read()['commands']) + 1 + original = None + try: + command(args, label, timeout=timeout, cwd=cwd) + except Exception as error: + original = error + raise + finally: + try: + observation = observer.observe(PRIVATE / ('command-%03d.log' % index), + pods, before, observer.lock_state(ROOT)) + data = read() + data.setdefault('dependency_install_observations', {})[stage] = observation + save(data) + print('DEPENDENCY_INSTALL_OBSERVATION ' + json.dumps( + {'stage': stage, 'observation': observation}, sort_keys=True), flush=True) + if not observation['locks_unchanged'] and original is None: + raise RuntimeError('Locked dependency input changed during preparation.') + except Exception: + if original is None: + raise + + +def record(): + source = subprocess.check_output(['git', 'rev-parse', 'HEAD'], cwd=ROOT, text=True).strip() + if source != os.environ.get('KEEPER_SOURCE_SHA'): + raise RuntimeError('Checkout does not match the exact requested PR head.') + save({ + 'schema': 1, 'platform': PLATFORM, 'source_sha': source, + 'diagnostic_only': os.environ.get('KEEPER_DIAGNOSTIC') == 'true', + 'workflow_ref': os.environ.get('GITHUB_WORKFLOW_REF'), + 'workflow_sha': os.environ.get('GITHUB_WORKFLOW_SHA'), + 'run_id': os.environ.get('GITHUB_RUN_ID'), 'run_attempt': os.environ.get('GITHUB_RUN_ATTEMPT'), + 'runner': {key: os.environ.get(key) for key in + ['RUNNER_OS', 'RUNNER_ARCH', 'ImageOS', 'ImageVersion']}, + 'os': platform.platform(), 'commands': [], 'build': 'NOT RUN', 'smoke': 'NOT RUN', 'native_probe': 'NOT RUN', 'r5_features': 'NOT RUN', + 'hardware': 'NOT RUN', 'signing': 'NOT RUN', 'app_id': APP_IDS[PLATFORM], + 'smoke_scope': 'Fresh install; render Create a passcode; type four disposable digits; assert Confirm your passcode; stop before persisting credentials.', + 'startup_timing_scope': 'Bounded cold development bundle/startup allowance; no release performance acceptance.', + 'artifact_policy': 'Metadata only. No raw logs, screenshots, app binaries, simulator data or credentials.', + }) + versions = { + 'node': command(['node', '--version'], 'Record Node version', capture=True), + 'java': command(['java', '-version'], 'Record Java version', capture=True), + } + if PLATFORM == 'IOS': + versions['xcode'] = command(['xcodebuild', '-version'], 'Record Xcode version', capture=True) + versions['sdk'] = command(['xcrun', '--sdk', 'iphonesimulator', '--show-sdk-version'], + 'Record iOS Simulator SDK', capture=True) + else: + versions['expected_sdk_build'] = 'compileSdk=36; buildTools=35.0.0; NDK=27.1.12297006' + versions['expected_emulator_target'] = 'Android API 35; google_apis; x86_64; pixel_6' + data = read() + data['toolchain'] = versions + save(data) + disk_preflight('initial') + + +def fixtures(): + # Empty values would fall back to real development services in config.ts. + overlays = ['.env.ci', 'firebase.json'] + (ROOT / '.env.ci').write_text('''ENVIRONMENT=DEVELOPMENT +RELAY=http://127.0.0.1:8765/ +SIGNING_SERVER_TESTNET=https://127.0.0.1:9/ +SIGNING_SERVER_MAINNET=https://127.0.0.1:9/ +CHANNEL_URL=https://127.0.0.1:9/ +SENTRY_DNS=https://public@127.0.0.1:9/1 +ENC_KEY_STORAGE_IDENTIFIER=KEEPER-HOSTED-DISPOSABLE +''') + (ROOT / 'firebase.json').write_text(json.dumps({'react-native': { + 'app_data_collection_default_enabled': False, + 'analytics_auto_collection_enabled': False, + 'analytics_collection_deactivated': True, + 'messaging_auto_init_enabled': False, + }}) + '\n') + if PLATFORM == 'IOS': + import plistlib + for path, bundle in [('ios/GoogleService-Info.plist', 'io.hexawallet.keeper'), + ('ios/GoogleServices/dev/GoogleService-Info.plist', APP_IDS['IOS'])]: + overlays.append(path) + with (ROOT / path).open('wb') as output: + plistlib.dump({'API_KEY': 'AIzaSy' + '0' * 33, 'GCM_SENDER_ID': '123456789012', + 'PLIST_VERSION': '1', 'BUNDLE_ID': bundle, + 'PROJECT_ID': 'keeper-disposable-ci', + 'GOOGLE_APP_ID': '1:123456789012:ios:0000000000000000000000', + 'IS_GCM_ENABLED': False, 'IS_ANALYTICS_ENABLED': False, + 'IS_ADS_ENABLED': False, 'IS_SIGNIN_ENABLED': False}, output) + else: + overlays.append('android/app/google-services.json') + (ROOT / 'android/app/google-services.json').write_text(json.dumps({ + 'project_info': {'project_number': '123456789012', 'project_id': 'keeper-disposable-ci'}, + 'client': [{'client_info': {'mobilesdk_app_id': '1:123456789012:android:0000000000000000', + 'android_client_info': {'package_name': APP_IDS['ANDROID']}}, + 'api_key': [{'current_key': 'AIzaSy' + '0' * 33}]}], + 'configuration_version': '1', + }) + '\n') + data = read() + data['configuration'] = 'Synthetic Firebase project; DEVELOPMENT/testnet; loopback-only configured services; no production secrets.' + data['configuration_overlays'] = [{'path': path, 'sha256': hashlib.sha256((ROOT / path).read_bytes()).hexdigest()} + for path in overlays] + save(data) + + +def install_maestro(): + archive = PRIVATE / 'maestro.zip' + url = 'https://github.com/mobile-dev-inc/Maestro/releases/download/%s/maestro.zip' % MAESTRO_VERSION + command(['curl', '--fail', '--location', '--silent', '--show-error', '--max-time', '120', + '--output', str(archive), url], 'Download pinned Maestro', timeout=130) + if hashlib.sha256(archive.read_bytes()).hexdigest() != MAESTRO_SHA256: + raise RuntimeError('Maestro archive checksum mismatch.') + # unzip preserves executable modes for the bundled native iOS driver; + # Python ZipFile extraction would silently strip those modes. + command(['unzip', '-q', str(archive), '-d', str(PRIVATE / 'maestro')], + 'Extract verified Maestro including native executable modes', timeout=120) + candidates = list((PRIVATE / 'maestro').rglob('bin/maestro')) + if len(candidates) != 1: + raise RuntimeError('Unexpected Maestro archive layout.') + data = read() + data['maestro'] = {'version': MAESTRO_VERSION, 'sha256': MAESTRO_SHA256, 'binary': str(candidates[0])} + save(data) + + +def android_sdk_metadata(sdk, component): + properties = (sdk / component / 'source.properties').read_text() + values = dict((key.strip(), value.strip()) for line in properties.splitlines() + if '=' in line for key, value in [line.split('=', 1)]) + if not values.get('Pkg.Revision'): + raise RuntimeError('Android SDK component %s has no Pkg.Revision metadata.' % component) + return {key: value for key, value in values.items() + if key in ['Pkg.Revision', 'AndroidVersion.ApiLevel']} + + +def prepare(): + dependencies_only = os.environ.get('KEEPER_DEPENDENCIES_ONLY') == 'true' + if dependencies_only and (PLATFORM != 'IOS' or os.environ.get('KEEPER_DIAGNOSTIC') != 'true'): + raise RuntimeError('Dependency-only execution requires an iOS diagnostic.') + if dependencies_only and os.environ.get('KEEPER_POD_LOCK_REPAIR') == '1': + raise RuntimeError('Dependency-only execution cannot repair locks.') + if dependencies_only: + data = read() + data['dependency_preparation'] = {'scope': 'DEPENDENCIES_ONLY_NOT_NATIVE_ACCEPTANCE', + 'status': 'RUNNING', 'native_credit': False} + save(data) + disk_preflight('before_dependencies') + command(['npm', 'install', '--global', 'yarn@1.22.22', '--ignore-scripts'], 'Install pinned Yarn', timeout=180) + command(['yarn', 'install', '--frozen-lockfile', '--ignore-scripts', '--non-interactive'], + 'Install immutable JavaScript dependencies', timeout=900) + command(['node_modules/.bin/patch-package', '--error-on-fail'], 'Apply repository dependency patches', timeout=180) + original_manifest = (ROOT / 'package.json').read_bytes() + command(['node', '.github/native/nodeify.cjs'], + 'Apply committed Node compatibility whitelist without installing dependencies', timeout=180) + installed_manifest = (ROOT / 'package.json').read_bytes() + original = json.loads(original_manifest) + installed = json.loads(installed_manifest) + for key in ['browser', 'react-native', 'browserify']: + original.pop(key, None) + installed.pop(key, None) + if original != installed: + raise RuntimeError('Node compatibility setup changed fields beyond module mappings.') + data = read() + data['nodeify_overlay'] = { + 'path': 'package.json', 'before_sha256': hashlib.sha256(original_manifest).hexdigest(), + 'after_sha256': hashlib.sha256(installed_manifest).hexdigest(), + 'diff_sha256': hashlib.sha256(subprocess.check_output(['git', 'diff', 'HEAD', '--', 'package.json'], cwd=ROOT)).hexdigest(), + 'allowed_fields': ['browser', 'react-native', 'browserify'], + } + save(data) + command(['git', 'diff', '--exit-code', 'HEAD', '--', 'yarn.lock', 'ios/Podfile.lock'], + 'Verify dependency locks remain immutable') + fixtures() + if not dependencies_only: + install_maestro() + if PLATFORM == 'IOS': + # Reuse the repository's frozen Ruby route, including transitive pins. + # Bare gem install/pod bypassed Gemfile.lock even with a pinned top-level gem. + os.environ.update({'BUNDLE_FROZEN': 'true', 'BUNDLE_PATH': str(PRIVATE / 'ruby-gems'), + 'BUNDLE_GEMFILE': str(ROOT / 'Gemfile')}) + dependency_command(['bundle', 'install', '--jobs', '4', '--retry', '1'], + 'Install frozen Ruby dependencies', 'RUBY_BUNDLE') + command(['bundle', 'check'], 'Verify frozen Ruby dependencies', timeout=120) + # This repository also has ios/Gemfile. Check the actual runtime from + # the pod command's cwd so Bundler cannot silently select that bundle. + dependency_command(['bundle', 'exec', 'ruby', str(ROOT / '.github/native/verify_ruby_context.rb'), + str(ROOT / 'Gemfile'), str(ROOT / 'Gemfile.lock')], + 'Verify actual root Ruby bundle context', 'RUBY_CONTEXT', timeout=120, cwd=ROOT / 'ios') + data = read() + data['frozen_ruby_context'] = {'status': 'PASS', 'gemfile': 'ROOT_GEMFILE', + 'lockfile': 'ROOT_GEMFILE_LOCK', 'cocoapods': '1.16.2', 'xcodeproj': '1.27.0'} + save(data) + if os.environ.get('KEEPER_POD_LOCK_REPAIR') == '1': + # Diagnostic bootstrap only. A candidate lock is reviewed/committed, + # then a separate strict deployment run must build and test the app. + command(['bundle', 'exec', 'pod', 'update', 'NitroIap', 'NitroModules', 'openiap', '--no-repo-update'], + 'Generate targeted CocoaPods lock candidate (not validation)', + timeout=900, cwd=ROOT / 'ios') + shutil.copyfile(ROOT / 'ios/Podfile.lock', EVIDENCE.parent / 'Podfile.lock.candidate') + raise RuntimeError('Pod lock candidate generated for review; no native validation performed.') + dependency_command(['bundle', 'exec', 'pod', 'install', '--deployment'], + 'Install immutable CocoaPods dependencies', 'PODS_DEPLOYMENT', cwd=ROOT / 'ios') + # Existing PBX build phases hardcode this path. Adapt the disposable host, + # preserving the app project and the source commit being tested. + if not dependencies_only: + command(['sudo', 'mkdir', '-p', '/opt/homebrew/bin'], 'Prepare existing Xcode Node path') + command(['sudo', 'ln', '-sf', shutil.which('node'), '/opt/homebrew/bin/node'], + 'Connect existing Xcode Node path to pinned Node') + else: + command(['sdkmanager', 'platforms;android-36', 'build-tools;35.0.0', 'ndk;27.1.12297006'], + 'Install pinned Android build SDK and NDK', timeout=600) + sdk = Path(os.environ.get('ANDROID_HOME', os.environ.get('ANDROID_SDK_ROOT', ''))) + observed = {} + for component in ['platforms/android-36', 'build-tools/35.0.0', 'ndk/27.1.12297006']: + observed[component] = android_sdk_metadata(sdk, component) + data = read() + data['toolchain']['observed_android_sdk'] = observed + save(data) + if (ROOT / 'package.json').read_bytes() != installed_manifest: + raise RuntimeError('Native installation unexpectedly rewrote the installed manifest.') + command(['git', 'diff', '--exit-code', 'HEAD', '--', '.github/native', '.github/workflows', + 'Gemfile', 'Gemfile.lock', 'ios/Gemfile', 'ios/Gemfile.lock', 'yarn.lock', 'ios/Podfile.lock'], + 'Verify native installation did not rewrite locks') + if dependencies_only: + data = read() + data['dependency_preparation']['status'] = 'PASS' + save(data) + + +def artifact_identity(): + if PLATFORM == 'IOS': + import plistlib + bundle = PRIVATE / 'DerivedData/Build/Products/Debug-iphonesimulator/hexa_keeper_dev.app' + with (bundle / 'Info.plist').open('rb') as source: + executable = plistlib.load(source)['CFBundleExecutable'] + binary = bundle / executable + tree = hashlib.sha256() + for file in sorted(path for path in bundle.rglob('*') if path.is_file()): + tree.update(str(file.relative_to(bundle)).encode() + b'\0') + tree.update(hashlib.sha256(file.read_bytes()).digest()) + return {'path': str(bundle), 'executable': executable, + 'executable_sha256': hashlib.sha256(binary.read_bytes()).hexdigest(), + 'bundle_tree_sha256': tree.hexdigest()} + apk = ROOT / 'android/app/build/outputs/apk/development/debug/app-development-debug.apk' + return {'path': str(apk.relative_to(ROOT)), 'sha256': hashlib.sha256(apk.read_bytes()).hexdigest()} + + +def _file_sha256(path): + digest = hashlib.sha256() + with path.open('rb') as source: + for block in iter(lambda: source.read(1024 * 1024), b''): + digest.update(block) + return digest.hexdigest() + + +def _bundle_inventory(bundle): + """Private typed tree; internal links must remain inside this bundle.""" + import plistlib + if bundle.is_symlink() or not bundle.is_dir(): + raise ValueError('Invalid app bundle.') + root = bundle.resolve(strict=True) + with (bundle / 'Info.plist').open('rb') as source: + info = plistlib.load(source) + executable = info['CFBundleExecutable'] + if (info['CFBundleIdentifier'] != APP_IDS['IOS'] or + not re.fullmatch(r'[A-Za-z0-9_][A-Za-z0-9_.-]*', executable)): + raise ValueError('Unexpected app identity.') + tree = {} + for entry in sorted(bundle.rglob('*')): + if not entry.resolve(strict=True).is_relative_to(root): + raise ValueError('Bundle entry escaped app.') + if entry.is_symlink(): + value = ['link', os.readlink(entry)] + elif entry.is_file(): + value = ['file', _file_sha256(entry)] + elif entry.is_dir(): + value = ['directory'] + else: + raise ValueError('Unsupported bundle entry.') + tree[entry.relative_to(bundle).as_posix()] = value + if tree.get(executable, [None])[0] != 'file': + raise ValueError('Missing regular app executable.') + return tree, executable + + +def owned_ios_bundle_path(installed, device, built): + """The same exact owned container policy for acceptance and observations.""" + uuid = r'[A-Fa-f0-9]{8}(?:-[A-Fa-f0-9]{4}){3}-[A-Fa-f0-9]{12}' + if not re.fullmatch(uuid, device): + raise ValueError('Expected owned simulator UUID.') + installed = Path(installed) + base = Path.home() / 'Library/Developer/CoreSimulator/Devices' / device / 'data/Containers/Bundle/Application' + if (not installed.is_absolute() or installed.name != built.name or + not re.fullmatch(uuid, installed.parent.name) or + installed.parent.parent.resolve() != base.resolve() or + installed.resolve(strict=True).parent.parent != base.resolve()): + raise ValueError('Unexpected installed app container.') + return installed + + +def verify_installed_artifact(device): + """Attest owned installed bytes, never installation argv or app assertions.""" + data = read() + data['installed_artifact_verified'] = False + data['installed_artifact_scope'] = 'NOT VERIFIED: installed bytes require full comparison.' + data.pop('installed_artifact', None) + save(data) + stage = 'ARTIFACT_METADATA' + try: + expected = data['built_artifact'] + if PLATFORM == 'IOS': + uuid = r'[A-Fa-f0-9]{8}(?:-[A-Fa-f0-9]{4}){3}-[A-Fa-f0-9]{12}' + if not re.fullmatch(uuid, device): + raise ValueError('Expected owned simulator UUID.') + built = PRIVATE / 'DerivedData/Build/Products/Debug-iphonesimulator/hexa_keeper_dev.app' + original, executable = _bundle_inventory(built) + installed = Path(command(['xcrun', 'simctl', 'get_app_container', device, APP_IDS['IOS'], 'app'], + 'Locate owned installed app bundle', capture=True, error_details=False)) + installed = owned_ios_bundle_path(installed, device, built) + actual, installed_executable = _bundle_inventory(installed) + if original != actual or executable != installed_executable: + raise ValueError('Installed bundle differs; no installer normalization is accepted.') + result = {'method': 'FULL_SIMULATOR_BUNDLE_TREE_AND_EXECUTABLE_SHA256', + 'inventory_sha256': hashlib.sha256(json.dumps(original, sort_keys=True).encode()).hexdigest(), + 'executable_sha256': actual[executable][1]} + elif PLATFORM == 'ANDROID': + stage = 'APK_DEVICE_POLICY' + if not re.fullmatch(r'emulator-[0-9]+', device): + raise ValueError('Expected owned emulator serial.') + stage = 'APK_PACKAGE_QUERY' + listing = command(['adb', '-s', device, 'shell', 'pm', 'path', APP_IDS['ANDROID']], + 'Locate exact installed development package', capture=True, error_details=False) + pattern = (r'package:(/data/app/(?:~~[A-Za-z0-9_=-]+/)?' + re.escape(APP_IDS['ANDROID']) + + r'-[A-Za-z0-9_=-]+/base\.apk)') + stage = 'APK_PATH_POLICY' + match = re.fullmatch(pattern, listing) + if not match: + raise ValueError('Expected one exact base APK; split or unexpected paths rejected.') + built = ROOT / 'android/app/build/outputs/apk/development/debug/app-development-debug.apk' + stage = 'APK_BUILT_FILE' + if built.is_symlink() or not built.is_file(): + raise ValueError('Invalid built APK.') + PRIVATE.mkdir(parents=True, exist_ok=True) + with tempfile.TemporaryDirectory(prefix='installed-artifact-', dir=PRIVATE) as temporary: + pulled = Path(temporary) / 'base.apk' + # Package paths and pulled bytes stay private, including tool output. + stage = 'APK_PULL' + copied = subprocess.run(['adb', '-s', device, 'pull', match[1], str(pulled)], + stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, + timeout=120, check=False) + stage = 'APK_PULL_RESULT' + if copied.returncode or pulled.is_symlink() or not pulled.is_file(): + raise ValueError('Installed APK bytes differ or are unavailable.') + stage = 'APK_SIZE_COMPARISON' + if pulled.stat().st_size != built.stat().st_size: + raise ValueError('Installed APK bytes differ or are unavailable.') + stage = 'APK_HASH_COMPARISON' + if _file_sha256(pulled) != expected['sha256']: + raise ValueError('Installed APK bytes differ or are unavailable.') + result = {'method': 'SINGLE_INSTALLED_BASE_APK_SHA256', 'sha256': expected['sha256']} + else: + raise ValueError('Unsupported artifact platform.') + stage = 'BUILT_ARTIFACT_STABILITY' + if artifact_identity() != expected: + raise ValueError('Built artifact changed during installed comparison.') + except Exception: + # Fixed labels distinguish a rejected path from pull/byte failures. + # Never print pm output, installed paths, exception text or APK bytes. + print('Installed artifact verification failed at ' + stage, flush=True) + raise RuntimeError('Installed artifact bytes were not verified; no equivalence claimed.') from None + data = read() + data.update(installed_artifact_verified=True, installed_artifact=result, + installed_artifact_scope='Full installed artifact byte comparison; no installer transformations accepted.') + save(data) + return result + + +def wait_for_simulator_app_services(device): + """Boot completion alone did not establish app-service responsiveness.""" + if PLATFORM != 'IOS': + raise ValueError('Simulator app-service readiness is iOS-only.') + spec = importlib.util.spec_from_file_location('simulator_readiness', Path(__file__).with_name('simulator_readiness.py')) + readiness = importlib.util.module_from_spec(spec) + spec.loader.exec_module(readiness) + return readiness.wait_for_app_services(SimpleNamespace(PRIVATE=PRIVATE, read=read, save=save), + device, timeout=120) + + +def driver(): + """Fail early on iOS driver setup without installing or crediting the app.""" + if PLATFORM != 'IOS': + raise ValueError('The disposable driver preflight is iOS-only.') + state = PRIVATE / 'maestro-preflight-state' + os.environ['XDG_STATE_HOME'] = str(state) + details = {'scope': 'DRIVER PREFLIGHT/setup only; no app build, install or UI credit.', + 'result': 'FAIL', 'cleanup_errors': []} + device = None + original_error = None + label = 'Check disposable iOS driver readiness' + try: + created = command(['xcrun', 'simctl', 'create', 'Keeper-driver-preflight-ci', + 'com.apple.CoreSimulator.SimDeviceType.iPhone-16', + 'com.apple.CoreSimulator.SimRuntime.iOS-18-5'], + 'Create owned driver-preflight simulator', timeout=120, capture=True) + if not re.fullmatch(r'[A-Fa-f0-9]{8}(?:-[A-Fa-f0-9]{4}){3}-[A-Fa-f0-9]{12}', created): + raise ValueError('simctl create did not return an owned simulator UUID.') + device = created + command(['xcrun', 'simctl', 'boot', device], 'Boot owned driver-preflight simulator', timeout=120) + command(['xcrun', 'simctl', 'bootstatus', device, '-b'], + 'Wait for driver-preflight simulator boot', timeout=240) + wait_for_simulator_app_services(device) + command([read()['maestro']['binary'], '--platform', 'ios', '--device', device, 'hierarchy'], + label, timeout=300, error_details=False) + details['result'] = 'PASS' + except Exception as error: + original_error = error + details['error_type'] = type(error).__name__ + if device: + # Reuse the setup-only collector while the owned device and private + # XCTest logs still exist. Observation never replaces this failure. + try: + spec = importlib.util.spec_from_file_location('driver_failure_diagnosis', + Path(__file__).with_name('driver_diagnosis.py')) + observer = importlib.util.module_from_spec(spec) + spec.loader.exec_module(observer) + observer.native = SimpleNamespace(PRIVATE=PRIVATE) + diagnosis = observer.collect(device, read()) + except Exception: + diagnosis = {'scope': 'Before owned simulator cleanup; driver setup only.', + 'collection_errors': [{'stage': 'pre_cleanup', 'error_type': 'OTHER_ERROR'}]} + try: + data = read() + data['driver_diagnosis'] = diagnosis + save(data) + except Exception: + pass # Preserve the primary driver failure even if its receipt fails. + raise + finally: + if device: + for action in ['shutdown', 'delete']: + try: + subprocess.run(['xcrun', 'simctl', action, device], stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, timeout=30, check=True) + except Exception as error: + details['cleanup_errors'].append({'action': action, 'error_type': type(error).__name__}) + try: + details['diagnostics'] = maestro_preflight_diagnostics(state / 'maestro/tests', label) + except Exception as error: + details['diagnostics'] = {'collection_error_types': [type(error).__name__]} + if details['cleanup_errors']: + details['result'] = 'FAIL' + try: + data = read() + data['driver_preflight'] = details + save(data) + except Exception: + if original_error is None: + raise RuntimeError('Driver-preflight receipt failed.') from None + if details['cleanup_errors'] and original_error is None: + raise RuntimeError('Owned driver-preflight simulator cleanup failed.') + + +def disk_preflight(stage): + free = shutil.disk_usage(ROOT).free + minimum = 12 * 1024 ** 3 + data = read() + data.setdefault('disk_preflight', []).append({ + 'stage': stage, 'free_bytes': free, 'minimum_free_bytes': minimum, + 'result': 'PASS' if free >= minimum else 'FAIL', + }) + save(data) + if free < minimum: + raise RuntimeError('Insufficient disposable-runner disk reserve; no cache or user data was removed.') + + +def sign_simulator_app(): + spec = importlib.util.spec_from_file_location('simulator_signing', Path(__file__).with_name('simulator_signing.py')) + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + module.sign(SimpleNamespace(PLATFORM=PLATFORM, PRIVATE=PRIVATE, read=read, save=save, command=command)) + + +def build(): + disk_preflight('before_native_build') + if PLATFORM == 'IOS': + architecture = platform.machine() + if architecture not in ['arm64', 'x86_64']: + raise RuntimeError('Unsupported hosted iOS simulator architecture: %s' % architecture) + data = read() + data.setdefault('toolchain', {})['ios_simulator_architecture'] = architecture + save(data) + try: + command(['xcodebuild', '-workspace', 'ios/hexa_keeper.xcworkspace', '-scheme', 'hexa_keeper_dev', + '-configuration', 'Debug', '-sdk', 'iphonesimulator', '-destination', 'generic/platform=iOS Simulator', + '-derivedDataPath', str(PRIVATE / 'DerivedData'), 'CODE_SIGNING_ALLOWED=NO', + 'CODE_SIGNING_REQUIRED=NO', 'ARCHS=' + architecture, 'ONLY_ACTIVE_ARCH=YES', 'build'], + 'Build unsigned development simulator app', timeout=3600 if architecture == 'x86_64' else 2400) + except Exception: + try: + data = read() + index = len(data['commands']) + if index and data['commands'][-1].get('label') == 'Build unsigned development simulator app': + data['ios_build_progress'] = ios_build_progress(PRIVATE / ('command-%03d.log' % index)) + save(data) + except Exception: + pass + raise + # Keychain rejects unsigned simulator executables with + # errSecMissingEntitlement. This uses no certificate or release key; + # record built/installed identities only after the ad-hoc signature. + sign_simulator_app() + else: + observed = command(['./gradlew', '--version'], 'Record actual Gradle and JVM version', + capture=True, cwd=ROOT / 'android') + data = read() + data['toolchain']['gradle'] = observed + save(data) + command(['./gradlew', ':app:assembleDevelopmentDebug', '--no-daemon', '--max-workers=2', + '-PreactNativeArchitectures=x86_64'], 'Build debug-key-only development APK', + timeout=2100, cwd=ROOT / 'android') + data = read() + data['built_artifact'] = artifact_identity() + data['build'] = 'PASS' + save(data) + + +def warm_metro_bundle(suite='startup'): + """Bound cold development bundling before starting either disposable app.""" + bundle_platform = {'IOS': 'ios', 'ANDROID': 'android'}.get(PLATFORM) + if bundle_platform is None: + raise ValueError('Unsupported native bundle platform.') + url = 'http://127.0.0.1:8081/index.bundle?platform=%s&dev=true&minify=false' % bundle_platform + body = PRIVATE / ('%s-development-warmup.bundle' % bundle_platform) + details = {'scope': 'Development bundle setup only; not native or UI evidence.', + 'platform': bundle_platform, 'url': url, 'timeout_seconds': 240, + 'result': 'FAIL', 'http_status': None} + started = time.monotonic() + try: + status = command(['curl', '--fail', '--silent', '--show-error', '--max-time', '240', + '--output', str(body), '--write-out', '%{http_code}', url], + 'Warm %s development bundle (setup only)' % PLATFORM, timeout=250, capture=True) + if re.fullmatch(r'\d{3}', status): + details['http_status'] = int(status) + if status != '200' or not body.is_file() or body.stat().st_size == 0: + raise RuntimeError('Development bundle warmup did not return a nonempty HTTP 200 body.') + digest = hashlib.sha256() + with body.open('rb') as source: + for chunk in iter(lambda: source.read(1024 * 1024), b''): + digest.update(chunk) + details.update({'result': 'PASS', 'bytes': body.stat().st_size, 'sha256': digest.hexdigest()}) + finally: + details['duration_seconds'] = round(time.monotonic() - started, 1) + data = read() + data['metro_warmup' if suite == 'startup' else suite + '_bundle'] = details + save(data) + + +R5_SCENARIOS = [ + 'production-setup-offline-retry-native-counts', + 'production-deletion-retained-native-references', + 'production-deletion-native-rollback', + 'production-worker-stale-session-rejection', + 'native-worker-process-relaunch-counts', +] +R5_LIMITS = [ + 'controlled-transport-no-backend-acceptance', + 'synthetic-wallet-generation-no-crypto-acceptance', + 'injected-dispatch-selection-no-login-acceptance', + 'no-product-ui-navigation-pin-or-biometric-acceptance', + 'probe-and-worker-timings-not-product-startup-ux', +] +R5_SOURCES = { + 'src/utils/utilities.ts': ['getAccountFromSigner', 'getKeyUID'], + 'src/store/sagas/storage.ts': ['setupKeeperAppWorker'], + 'src/store/sagas/wallets.ts': ['currentWalletAccountId', 'assertWalletAccount', 'deleteSigningDeviceWorker'], + 'src/store/sagas/bhr.ts': ['currentBackupAppId', 'deleteAppImageEntityWorker'], + 'src/services/backup/transport.ts': ['observeBackupAccount', 'currentBackupAccountSession', 'clearVerifiedRemoteRevision'], +} + + +def validate_r5_feature_evidence(raw): + if not isinstance(raw, str) or len(raw.encode()) > 65536: + raise ValueError('Feature evidence absent or oversized.') + def invalid_constant(value): + raise ValueError('Non-finite feature measurement.') + payload = json.loads(raw, parse_constant=invalid_constant) + if (type(payload.get('schema')) is not int or payload.get('schema') != 1 or payload.get('scope') != 'production-worker-native-storage' + or payload.get('status') != 'PASS' or payload.get('stage') != 'pass' + or payload.get('scenarios') != [{'id': item, 'status': 'PASS'} for item in R5_SCENARIOS] + or payload.get('limits') != R5_LIMITS): + raise ValueError('Feature scenario evidence is incomplete or outside reviewed scope.') + measurements = payload.get('measurements') + if not isinstance(measurements, list) or len(measurements) != 5: + raise ValueError('Five raw helper measurements are required.') + for iteration, row in enumerate(measurements, 1): + if not isinstance(row, dict) or set(row) != {'iteration', 'probe_ready_ms', 'setup_first_wallet_ms'} or type(row['iteration']) is not int or row['iteration'] != iteration: + raise ValueError('Invalid feature measurement shape.') + for field in ['probe_ready_ms', 'setup_first_wallet_ms']: + value = row[field] + if type(value) not in [int, float] or not math.isfinite(value) or not 0 <= value <= 600000: + raise ValueError('Invalid bounded feature measurement.') + sources = payload.get('source_functions') + expected = [] + for path, names in R5_SOURCES.items(): + committed = subprocess.check_output(['git', 'show', 'HEAD:' + path], cwd=ROOT) + if (ROOT / path).read_bytes() != committed: + raise ValueError('Feature production source differs from committed candidate.') + expected.append({'path': path, 'sha256': hashlib.sha256(committed).hexdigest(), 'names': names}) + if sources != expected: + raise ValueError('Feature workers do not match exact-candidate production source.') + # Export only fixed scope/status, public source digests and bounded numbers. + # Never forward arbitrary payload fields or native failure messages. + return {'schema': 1, 'scope': 'production-worker-native-storage', 'status': 'PASS', + 'scenarios': payload['scenarios'], 'measurements': measurements, + 'limits': R5_LIMITS, 'source_functions': sources} + + +def collect_r5_feature_evidence(device): + filename = 'keeper-r5-native-evidence.json' + if PLATFORM == 'IOS': + container = Path(command(['xcrun', 'simctl', 'get_app_container', device, APP_IDS[PLATFORM], 'data'], + 'Locate disposable feature evidence', capture=True)) + path = container / 'Documents' / filename + if path.stat().st_size > 65536: + raise ValueError('Oversized native feature evidence.') + raw = path.read_text() + else: + raw = command(['adb', '-s', device, 'exec-out', 'run-as', APP_IDS[PLATFORM], + 'head', '-c', '65537', 'files/' + filename], + 'Read bounded disposable feature evidence', capture=True, error_details=False) + return validate_r5_feature_evidence(raw) + + +def smoke(suite='startup'): + if suite not in ['startup', 'probe', 'features']: + raise ValueError('Unknown disposable native suite.') + is_probe = suite == 'probe' + is_features = suite == 'features' + result_key = 'r5_features' if is_features else 'native_probe' if is_probe else 'smoke' + if (is_probe or is_features) and read()['smoke'] != 'PASS': + raise RuntimeError('The real app startup smoke must pass before module probes.') + if read()['build'] != 'PASS': + raise RuntimeError('A successful native build is required before smoke testing.') + if artifact_identity() != read()['built_artifact']: + raise RuntimeError('Built artifact changed before the disposable startup smoke.') + if is_features: + command(['git', 'diff', '--exit-code', 'HEAD', '--', *R5_SOURCES], + 'Verify committed R5 worker source before extraction', error_details=False) + command(['node', '.github/native/r5_prepare.cjs'], 'Prepare exact-source R5 production worker probe', timeout=60, error_details=False) + report = PRIVATE / ('features-junit.xml' if is_features else 'probe-junit.xml' if is_probe else 'smoke-junit.xml') + report.unlink(missing_ok=True) + metro_output = (PRIVATE / 'metro.log').open('wb') + metro_args = ['node', 'node_modules/react-native/cli.js', 'start', '--port', '8081'] + startup_observer = None + startup_capture = None + startup_context = SimpleNamespace(PRIVATE=PRIVATE, PLATFORM=PLATFORM) + metro_env = os.environ.copy() + try: + if is_probe or is_features: + metro_args += ['--config', str(ROOT / ('.github/native/r5_metro.cjs' if is_features else '.github/native/release_metro.cjs'))] + else: + # Observation retains the normal entrypoint/resolver/transformer and + # response body on both platforms. Only fixed facts leave the window. + spec = importlib.util.spec_from_file_location('startup_diagnosis', Path(__file__).with_name('startup_diagnosis.py')) + startup_observer = importlib.util.module_from_spec(spec) + spec.loader.exec_module(startup_observer) + observer_path = PRIVATE / 'startup-observer.jsonl' + observer_path.unlink(missing_ok=True) + metro_env['KEEPER_STARTUP_OBSERVER_PATH'] = str(observer_path) + # Standard CLI log forwarding only; no resolver, bundle or app changes. + metro_args += ['--client-logs', '--config', str(ROOT / '.github/native/startup_observer.cjs'), + '--custom-log-reporter-path', str(ROOT / '.github/native/startup_reporter.cjs')] + metro = subprocess.Popen(metro_args, + cwd=ROOT, stdout=metro_output, stderr=subprocess.STDOUT, start_new_session=True, + env=metro_env) + except Exception: + try: + metro_output.close() + except Exception: + pass + raise + device = None + startup_error = None + try: + for _ in range(60): + if metro.poll() is not None: + raise RuntimeError('Metro exited before the disposable smoke.') + try: + with urllib.request.urlopen('http://127.0.0.1:8081/status', timeout=2) as response: + if b'packager-status:running' in response.read(): + break + except OSError: + time.sleep(1) + else: + raise RuntimeError('Metro did not become ready within 60 attempts.') + warm_metro_bundle(suite) + if PLATFORM == 'IOS': + created = command(['xcrun', 'simctl', 'create', 'Keeper-disposable-ci', + 'com.apple.CoreSimulator.SimDeviceType.iPhone-16', + 'com.apple.CoreSimulator.SimRuntime.iOS-18-5'], + 'Create fresh iPhone 16 iOS 18.5 simulator', capture=True) + if not re.fullmatch(r'[A-Fa-f0-9]{8}(?:-[A-Fa-f0-9]{4}){3}-[A-Fa-f0-9]{12}', created): + raise ValueError('Expected owned simulator UUID before installation.') + device = created + command(['xcrun', 'simctl', 'boot', device], 'Boot disposable simulator') + command(['xcrun', 'simctl', 'bootstatus', device, '-b'], 'Wait for simulator boot', timeout=240) + wait_for_simulator_app_services(device) + install_args = ['xcrun', 'simctl', 'install', device, + str(PRIVATE / 'DerivedData/Build/Products/Debug-iphonesimulator/hexa_keeper_dev.app')] + if not (is_probe or is_features): + spec = importlib.util.spec_from_file_location('install_diagnosis', Path(__file__).with_name('install_diagnosis.py')) + installer = importlib.util.module_from_spec(spec) + spec.loader.exec_module(installer) + installer.observe(SimpleNamespace(PRIVATE=PRIVATE, ROOT=ROOT, read=read, save=save, command=command), + device, install_args, 'Install unsigned development app', timeout=120) + else: + command(install_args, 'Install unsigned development app', timeout=120) + descriptor = 'iPhone 16; iOS 18.5' + else: + sdk = Path(os.environ.get('ANDROID_HOME', os.environ.get('ANDROID_SDK_ROOT', ''))) + # Query the installed SDK package without loading emulator GUI libraries + # on the headless host. The emulator runner already booted this image. + emulator_version = android_sdk_metadata(sdk, 'emulator') + adb_version = command(['adb', 'version'], 'Record actual adb version', capture=True) + data = read() + data['toolchain']['emulator'] = {'source': 'emulator/source.properties', **emulator_version} + data['toolchain']['adb'] = adb_version + save(data) + device = command(['adb', 'get-serialno'], 'Identify fresh emulator', capture=True) + if not re.fullmatch(r'emulator-[0-9]+', device): + raise ValueError('Expected owned emulator before installation.') + command(['adb', '-s', device, 'reverse', 'tcp:8081', 'tcp:8081'], 'Connect emulator to Metro') + command(['adb', '-s', device, 'install', '-r', + 'android/app/build/outputs/apk/development/debug/app-development-debug.apk'], + 'Install development debug APK', timeout=120) + descriptor = command(['adb', '-s', device, 'shell', 'getprop', 'ro.build.fingerprint'], + 'Record Android emulator image', capture=True) + data = read() + data['device'] = {'id': device, 'image': descriptor, 'fresh_disposable_state': True} + data['install_succeeded'] = True + save(data) + verify_installed_artifact(device) + data = read() + if startup_observer: + try: + startup_capture = startup_observer.start(startup_context, device) + except Exception: + data['startup_observation_capture'] = 'UNAVAILABLE' + save(data) + command([data['maestro']['binary'], '--device', device, 'test', '--env', 'APP_ID=' + APP_IDS[PLATFORM], + '--flatten-debug-output', '--debug-output', str(PRIVATE / 'maestro-output'), + '--format', 'junit', '--output', str(report), '--test-output-dir', str(PRIVATE / 'maestro-output'), + '.github/native/r5_features.yaml' if is_features else '.github/native/release_probe.yaml' if is_probe else '.github/native/startup.yaml'], + 'Assert R5 production workers over native storage' if is_features else 'Assert native persistence across process launches' if is_probe else + 'Assert startup and disposable passcode transition', timeout=660 if is_features else 300 if is_probe else 420, + error_details=not (is_probe or is_features)) + cases = successful_junit(report) + if artifact_identity() != read()['built_artifact']: + raise RuntimeError('Built artifact changed during native execution.') + verify_installed_artifact(device) + data = read() + data[result_key] = 'PASS' + data[result_key + '_test_cases'] = cases + if is_features: + data['r5_feature_evidence'] = collect_r5_feature_evidence(device) + if is_probe: + data['probe_scope'] = ('CI-only Metro entrypoint; actual encrypted Realm/current schemas and production MMKV; ' + 'three process launches, rollback, wrong-key rejection, targeted primitive deletion, ' + 'separate database files and cleanup. NOT app key deletion, account isolation, ' + 'setup/recovery or release performance acceptance.') + save(data) + except Exception as original_error: + startup_error = original_error + try: + if not (is_probe or is_features): + record_startup_failure(report, device, getattr(startup_capture, 'started_ms', None)) + else: + data = read() + data[result_key] = 'FAIL' + data['probe_diagnostics'] = 'Native module or worker probe failed; raw output retained only on ephemeral runner.' + save(data) + except Exception as error: + # Diagnostics never replace the original failed native/UI result. + try: + data = read() + data['startup_diagnostics'] = {'collection_errors': [ + {'stage': 'collector', 'error_type': type(error).__name__}]} + save(data) + except Exception: + pass + raise + finally: + # Receipt I/O is never allowed to skip owned-resource disposal or hide + # the original native failure. Retain only fixed cleanup labels. + cleanup_errors = [] + updates = {} + + def cleanup(label, action): + try: + action() + except Exception: + cleanup_errors.append(label) + + if startup_observer: + try: + updates['startup_observation'] = startup_observer.collect(startup_context, device, startup_capture) if device else {'scope': 'App launch NOT RUN; no owned device.'} + except Exception: + updates['startup_observation'] = {'collection_errors': ['collector']} + cleanup('startup_capture', lambda: startup_observer.stop(startup_capture)) + if 'startup_capture' in cleanup_errors: + updates['startup_observation_cleanup'] = 'FAIL' + + if startup_error is not None and PLATFORM == 'IOS' and device and not (is_probe or is_features): + # Original failure diagnostics are already frozen. This one later + # launch cannot repair smoke or unlock subsequent acceptance phases. + try: + spec = importlib.util.spec_from_file_location('ios_direct_launch_observation', + Path(__file__).with_name('ios_direct_launch_observation.py')) + observer = importlib.util.module_from_spec(spec) + spec.loader.exec_module(observer) + updates['ios_direct_launch_observation'] = observer.collect( + SimpleNamespace(PLATFORM=PLATFORM, PRIVATE=PRIVATE, read=read, + original_smoke_failed=True), device) + except Exception: + updates['ios_direct_launch_observation'] = { + 'state': 'UNAVAILABLE', 'native_acceptance': False} + + def stop_metro(): + if metro.poll() is None: + os.killpg(metro.pid, signal.SIGTERM) + try: + metro.wait(timeout=10) + except subprocess.TimeoutExpired: + os.killpg(metro.pid, signal.SIGKILL) + metro.wait(timeout=10) + + cleanup('metro_process', stop_metro) + cleanup('metro_output', metro_output.close) + if device and PLATFORM == 'IOS': + for action in ['shutdown', 'delete']: + cleanup('device_' + action, lambda action=action: subprocess.run( + ['xcrun', 'simctl', action, device], stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, timeout=30, check=True)) + elif device: + cleanup('device_uninstall', lambda: subprocess.run( + ['adb', '-s', device, 'uninstall', APP_IDS[PLATFORM]], stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, timeout=30, check=True)) + if startup_observer: + # Purge only this observer's raw files; other phases still need the + # shared private build/tool directories on the ephemeral runner. + observer_files = ['startup-logcat.log', 'startup-observer.jsonl', 'startup-foreground.log'] + if PLATFORM == 'IOS': + observer_files += ['ios-startup-errors.log', 'ios-startup-errors-stderr.log'] + for name in observer_files: + cleanup('private_' + name.split('.')[0], lambda name=name: (PRIVATE / name).unlink(missing_ok=True)) + if startup_error is not None or cleanup_errors: + updates[result_key] = 'FAIL' + if cleanup_errors: + updates['startup_cleanup_errors'] = cleanup_errors.copy() + try: + data = read() + data.update(updates) + save(data) + except Exception: + cleanup_errors.append('metadata') + # A transient write failure may allow one best-effort FAIL receipt. + # A permanently unavailable receipt still causes a failing phase. + try: + data = read() + data.update(updates) + data.update({result_key: 'FAIL', 'startup_cleanup_errors': cleanup_errors.copy()}) + save(data) + except Exception: + pass + if cleanup_errors and startup_error is None: + raise RuntimeError('Startup cleanup or receipt failed; subsequent probes cannot run.') from None + + +def successful_junit(report): + if not report.is_file() or report.stat().st_size > 1024 * 1024: + raise RuntimeError('Missing or oversized native JUnit report.') + root = ET.parse(report).getroot() + for suite in root.iter(): + if suite.tag in ['testsuite', 'testsuites']: + for attribute in ['failures', 'errors', 'skipped', 'disabled']: + if int(suite.get(attribute, '0')) != 0: + raise RuntimeError('Native JUnit declares unsuccessful cases.') + cases = list(root.iter('testcase')) + if not cases or any(list(root.iter(tag)) for tag in ['failure', 'error', 'skipped']) or any( + case.get('status', 'SUCCESS').upper() != 'SUCCESS' for case in cases): + raise RuntimeError('Maestro did not produce a complete successful native report.') + return len(cases) + + +def probe(): + smoke('probe') + + +def features(): + smoke('features') + + +def main(): + if PLATFORM not in APP_IDS or len(sys.argv) != 2 or sys.argv[1] not in ['record', 'prepare', 'driver', 'build', 'smoke', 'probe', 'features']: + raise RuntimeError('Expected a hosted platform and record|prepare|driver|build|smoke|probe|features.') + os.environ.update({'ENVFILE': '.env.ci', 'RCT_NO_LAUNCH_PACKAGER': '1', + 'RCT_NEW_ARCH_ENABLED': '1', 'MAESTRO_CLI_NO_ANALYTICS': '1'}) + run_phase(sys.argv[1]) + + +def run_phase(phase): + try: + globals()[phase]() + except Exception: + try: + if EVIDENCE.exists(): + evidence = read() + evidence['failed_phase'] = phase + if phase == 'prepare' and 'dependency_preparation' in evidence: + evidence['dependency_preparation']['status'] = 'FAIL' + if phase in ['build', 'smoke', 'probe', 'features']: + evidence['r5_features' if phase == 'features' else 'native_probe' if phase == 'probe' else phase] = 'FAIL' + save(evidence) + except Exception: + pass + raise + + +if __name__ == '__main__': + try: + main() + except Exception: + # Unexpected exceptions can contain private paths or process output too. + print('Hosted native validation failed; inspect phase status and fixed diagnostic categories.', file=sys.stderr) + sys.exit(1) diff --git a/.github/native/install_diagnosis.py b/.github/native/install_diagnosis.py new file mode 100644 index 0000000000..ea1df541f6 --- /dev/null +++ b/.github/native/install_diagnosis.py @@ -0,0 +1,249 @@ +"""Observe one unchanged simulator install; export only fixed, bounded facts. + +The caller supplies its live harness context. This module never imports another +hosted_native instance, changes the install deadline, retries, opens Simulator, +or changes simulator/app services. Raw process/log output stays private. +""" + +from concurrent.futures import ThreadPoolExecutor, TimeoutError as FutureTimeoutError +import hashlib +import math +import os +from pathlib import Path +import re +import signal +import shutil +import subprocess +import time + + +SNAPSHOT_SECONDS = 60 +LIMIT = 1024 * 1024 +APP_ID = 'io.hexawallet.hexakeeper.dev' +DOMAINS = { + 'CORE_SIMULATOR': 'com.apple.CoreSimulator.SimError', + 'MOBILE_INSTALLATION': 'MIInstallerErrorDomain', + 'INSTALL_COORDINATION': 'IXErrorDomain', + 'USER_PRESENTABLE_INSTALL': 'IXUserPresentableErrorDomain', + 'LAUNCH_SERVICES': 'LSApplicationWorkspaceErrorDomain', + 'COCOA': 'NSCocoaErrorDomain', + 'POSIX': 'NSPOSIXErrorDomain', +} +CATEGORIES = { + 'INSTALL_FAILED': r'failed to install|could not install|installation failed', + 'BUNDLE_VALIDATION_FAILED': r'bundle.*(?:invalid|validation failed)|invalid.*(?:bundle|executable)', + 'SIGNATURE_VERIFICATION_FAILED': r'applicationverificationfailed|invalid code signature|signature.*(?:invalid|verification failed)', + 'SERVICE_CONNECTION_FAILED': r'connection.*(?:invalidated|interrupted|refused|timed out)|failed to connect|service.*not available', + 'CONTAINER_SERVICE_FAILED': r'(?:container|containermanager).*\b(?:error|failed|unavailable)\b', + 'PERMISSION_DENIED': r'permission denied|operation not permitted|not authorized', + 'DISK_SPACE_EXHAUSTED': r'no space left on device|not enough (?:free )?(?:disk )?space', + 'RESOURCE_BUSY': r'resource busy|device busy|another installation.*in progress', + 'DEVICE_NOT_BOOTED': r'device.*(?:not booted|shutdown|unavailable)|invalid device state', + 'INSTALL_COORDINATION_REFERENCE': r'(?:install coordination|IXCoordinator|IXSClient|IXClient)', +} +STACK_MARKERS = { + 'CORE_SIMULATOR': r'CoreSimulator|SimDevice|SimServiceContext', + 'MOBILE_INSTALLATION': r'MobileInstallation|MIClient|installApplication|installBundle', + 'INSTALL_COORDINATION': r'InstallCoordination|IXCoordinator|IXSClient|IXClient', + 'CONTAINER_SERVICE': r'MobileContainerManager|MCMClient|containermanager', + 'LAUNCH_SERVICES': r'LaunchServices|LSApplication|_LSServer', + 'SIGNATURE_VERIFICATION': r'MISValidate|SecStaticCode|AMFI', + 'XPC_SYNC_WAIT': r'xpc_connection_send_message_with_reply_sync', + 'MACH_WAIT': r'mach_msg|mach_msg2_trap', + 'DISPATCH_WAIT': r'dispatch_sync|dispatch_semaphore_wait|dispatch_group_wait', + 'LOCK_WAIT': r'__psynch_mutexwait|__ulock_wait|pthread_mutex_lock', +} + + +def _error(error): + name = type(error).__name__ + return name if name in {'RuntimeError', 'OSError', 'ValueError', 'TypeError', + 'FileNotFoundError', 'PermissionError', 'TimeoutExpired'} else 'OTHER_ERROR' + + +def _classify(payload): + categories = [name for name, pattern in CATEGORIES.items() + if re.search(pattern, payload, re.IGNORECASE)] + codes = {} + for name, domain in DOMAINS.items(): + matches = re.findall(r'Domain\s*=\s*' + re.escape(domain) + r'\s+Code\s*=\s*(-?\d+)\b', + payload, re.IGNORECASE) + values = list(dict.fromkeys(int(value) for value in matches if len(value) <= 6 + and -65535 <= int(value) <= 65535))[:4] + if values: + codes[name] = values + return {'categories': categories, 'error_domain_codes': codes} + + +def _bounded(context, path): + if path.is_symlink() or not path.resolve().is_relative_to(Path(context.PRIVATE).resolve()): + raise ValueError('Expected private diagnostic file.') + size_before = path.stat().st_size + with path.open('rb') as source: + payload = source.read(LIMIT) + size_after = path.stat().st_size + text = payload.decode(errors='replace') + return text, {'bytes': size_after, 'prefix_bytes': len(payload), + 'truncated': size_after > len(payload), 'grew_during_read': size_after != size_before, + 'prefix_sha256': hashlib.sha256(payload).hexdigest(), **_classify(text)} + + +def _capture(context, args, label, prefix, timeout): + path = Path(context.PRIVATE) / (prefix + '-' + label + '.log') + facts = {'label': label} + started = time.monotonic() + try: + if path.is_symlink(): + raise ValueError('Expected private diagnostic output.') + with path.open('wb') as output: + proc = subprocess.Popen(args, stdout=output, stderr=subprocess.STDOUT, start_new_session=True) + try: + facts['exit_code'] = proc.wait(timeout=timeout) + except subprocess.TimeoutExpired: + os.killpg(proc.pid, signal.SIGKILL) + proc.wait() + facts['exit_code'] = 124 + payload, result = _bounded(context, path) + facts.update(result) + return payload, facts + except Exception as error: + facts['error_type'] = _error(error) + return '', facts + finally: + facts['duration_seconds'] = round(time.monotonic() - started, 2) + + +def _baseline(context, device, prefix): + facts = {'commands': [], 'resource_errors': []} + try: + facts['disk_free_bytes'] = shutil.disk_usage(context.PRIVATE).free + except Exception as error: + facts['resource_errors'].append({'stage': 'disk', 'error_type': _error(error)}) + try: + values = os.getloadavg() + if len(values) != 3 or not all(math.isfinite(value) for value in values): + raise ValueError('Expected finite host load observations.') + facts['host_load_1_5_15m'] = [round(value, 2) for value in values] + except Exception as error: + facts['resource_errors'].append({'stage': 'host_load', 'error_type': _error(error)}) + for args, label in [ + (['xcrun', 'simctl', 'listapps', device], 'baseline-listapps'), + (['xcrun', 'simctl', 'get_app_container', device, 'com.apple.Preferences', 'app'], + 'baseline-preferences-container'), + ]: + _, result = _capture(context, args, label, prefix, 5) + facts['commands'].append(result) + return facts + + +def _collect(context, device, prefix, command_log): + facts = {'commands': [], 'collection_errors': []} + + def command(args, label, timeout): + payload, result = _capture(context, args, label, prefix, timeout) + facts['commands'].append(result) + return payload, result + + ps, result = command(['ps', '-axo', 'pid=,stat=,args='], 'processes', 3) + pattern = re.compile(r'^\s*(\d+)\s+(\S+)\s+(.*)$') + owned = [] + for line in ps.splitlines(): + match = pattern.match(line) + if match and re.search(r'(?:^|[/\s])simctl\s+install\s+' + re.escape(device) + r'(?=\s|$)', match[3]): + owned.append((int(match[1]), match[2])) + known = result.get('exit_code') == 0 + facts['owned_install_process_count'] = len(owned) if known else None + facts['owned_install_running'] = any('Z' not in state for _, state in owned) if known else None + if len(owned) == 1 and 'Z' not in owned[0][1]: + sample_path = Path(context.PRIVATE) / (prefix + '-install.sample') + command(['sample', str(owned[0][0]), '3', '1', '-file', str(sample_path)], 'install-sample', 8) + try: + payload, result = _bounded(context, sample_path) + result['stack_categories'] = [name for name, marker in STACK_MARKERS.items() + if re.search(marker, payload, re.IGNORECASE)] + facts['install_sample'] = result + except Exception as error: + facts['collection_errors'].append({'stage': 'install_sample', 'error_type': _error(error)}) + launchctl, result = command(['xcrun', 'simctl', 'spawn', device, 'launchctl', 'list'], 'simulator-services', 4) + facts['services'] = {} + for name, identifier in [('installd', 'com.apple.mobile.installd'), ('appinstalld', 'com.apple.appinstalld'), + ('lsd', 'com.apple.lsd')]: + rows = [line.split() for line in launchctl.splitlines() if line.split()[-1:] == [identifier]] + facts['services'][name] = { + 'entries': len(rows) if result.get('exit_code') == 0 else None, + 'running': any(row[0].isdigit() and int(row[0]) > 0 for row in rows) + if result.get('exit_code') == 0 else None, + } + host_predicate = '(process == "CoreSimulatorService" OR process == "simctl") AND eventMessage CONTAINS "' + device + '"' + command(['log', 'show', '--style', 'compact', '--last', '3m', '--predicate', host_predicate], 'owned-host-events', 5) + command(['xcrun', 'simctl', 'spawn', device, 'log', 'show', '--style', 'compact', '--last', '3m', + '--predicate', 'process == "installd" OR process == "appinstalld" OR process == "lsd"'], + 'simulator-install-events', 8) + try: + _, facts['install_command_output'] = _bounded(context, command_log) + except Exception as error: + facts['collection_errors'].append({'stage': 'install_command_output', 'error_type': _error(error)}) + return facts + + +def observe(context, device, args, label, timeout=120): + """Call context.command once with the original install argv and deadline. + + Missing observation persistence blocks this diagnostic path after an install + succeeds. It cannot replace an original install exception or grant UI credit. + """ + if (not re.fullmatch(r'[A-Fa-f0-9]{8}(?:-[A-Fa-f0-9]{4}){3}-[A-Fa-f0-9]{12}', device) + or list(args[:4]) != ['xcrun', 'simctl', 'install', device] or len(args) != 5): + raise ValueError('Expected the owned simulator app install command.') + index = len(context.read()['commands']) + 1 + prefix = 'install-diagnosis-%03d' % index + command_log = Path(context.PRIVATE) / ('command-%03d.log' % index) + facts = {'scope': 'Read-only install observation; no UI or installed-byte acceptance.', + 'command_index': index, 'collection_errors': []} + original_error = None + try: + try: + facts['baseline'] = _baseline(context, device, prefix) + except Exception as error: + facts['collection_errors'].append({'stage': 'baseline', 'error_type': _error(error)}) + with ThreadPoolExecutor(max_workers=1) as executor: + future = executor.submit(context.command, args, label, timeout=timeout) + try: + result = future.result(timeout=SNAPSHOT_SECONDS) + except FutureTimeoutError: + pass + except Exception as error: + original_error = error + else: + facts['command_returned'] = True + facts['snapshot_phase'] = 'DURING_INSTALL' if not future.done() else 'AFTER_INSTALL_COMMAND' + facts['install_running_before_collection'] = not future.done() + try: + facts['snapshot'] = _collect(context, device, prefix, command_log) + except Exception as error: + facts['collection_errors'].append({'stage': 'snapshot', 'error_type': _error(error)}) + facts['install_running_after_collection'] = not future.done() + try: + result = future.result() + except Exception as error: + original_error = error + facts['command_error_type'] = _error(error) + raise + else: + facts['command_returned'] = True + return result + finally: + try: + _, lookup = _capture(context, ['xcrun', 'simctl', 'get_app_container', device, APP_ID, 'app'], + 'post-install-container', prefix, 5) + code = lookup.get('exit_code') + lookup['state'] = 'SUCCESS' if code == 0 else 'TIMEOUT' if code == 124 else 'ERROR' if code is None else 'FAIL' + facts['post_install_lookup'] = lookup + data = context.read() + data.setdefault('install_diagnoses', []).append(facts) + context.save(data) + except Exception: + if original_error is None: + raise RuntimeError('Bounded install diagnostics could not be saved.') from None + # Never replace the original install failure with a collector error. + print('Bounded install diagnostics could not be saved.', flush=True) diff --git a/.github/native/installed_parent_software.cjs b/.github/native/installed_parent_software.cjs new file mode 100644 index 0000000000..838c304f3d --- /dev/null +++ b/.github/native/installed_parent_software.cjs @@ -0,0 +1,72 @@ +'use strict'; +// Compose independently authenticated read-only windows. No profile installation or release authority. +const { isDeepStrictEqual: equal } = require('node:util'); +const window = require('./installed_parent_window.cjs'); +const privateSoftware = require('./private_actions_software.cjs'); +const issued = new WeakSet(), publicBindings = new WeakMap(); +const demand = (yes, code) => { if (!yes) throw new Error(code); }; +const PARENTS = require('./release_acceptance.cjs').REQUIRED.slice(2); +function frozen(value) { + if (value && typeof value === 'object') { Object.values(value).forEach(frozen); Object.freeze(value); } + return value; +} +async function collectInstalledParentSoftware(privateGithub, github, context, request, publicProof, options = {}) { + const canonical = require('./canonical_native.cjs'); + demand(canonical.isInstalledPublicEvidence(publicProof), 'INSTALLED_PARENT_PUBLIC_CAPABILITY_REQUIRED'); + // Only the committed finite profile is admissible. An API response, option or artifact cannot add one. + const admitted = await window.authenticateInstalledParentWindow(github, context, request, { pinReader: options.pinReader }); + demand(window.isInstalledParentWindow(admitted), 'INSTALLED_PARENT_WINDOW_CAPABILITY_REQUIRED'); + demand(privateGithub && privateGithub !== github && typeof privateGithub.rest?.actions?.downloadArtifact === 'function', + 'INSTALLED_PARENT_PRIVATE_READ_IDENTITY_REQUIRED'); + demand(publicProof.source_sha === admitted.app.sha && publicProof.source_tree === admitted.app.tree && + publicProof.workflow_sha === admitted.public.workflowSha && publicProof.workflow_path === admitted.public.workflowPath && + publicProof.run_id === admitted.public.runId && publicProof.run_attempt === admitted.public.runAttempt, + 'INSTALLED_PARENT_PUBLIC_WINDOW_MISMATCH'); + const privateProof = await privateSoftware.collectPrivateActionsSoftware(privateGithub, + admitted.expectedPerPlatform, { clock: options.clock, fetchImpl: options.fetchImpl }); + demand(privateSoftware.isPrivateActionsSoftware(privateProof), 'INSTALLED_PARENT_PRIVATE_CAPABILITY_REQUIRED'); + demand(equal(publicProof.worker_sources, privateProof.worker_sources), 'INSTALLED_PARENT_WORKER_SOURCE_MISMATCH'); + const platforms = {}; + for (const platform of ['IOS', 'ANDROID']) { + const raw = privateProof.platforms[platform], software = raw.software; + demand(software.software_complete === true && equal(software.identity.app, admitted.app) && + equal(software.identity.backend, admitted.backend) && + equal(software.identity.control, admitted.expectedPerPlatform[platform].control) && + equal(software.identity.execution, admitted.expectedPerPlatform[platform].execution) && + Array.isArray(software.cases) && software.cases.length === PARENTS.length && PARENTS.every(id => + software.cases.filter(row => row.id === id && row.status === 'PASS').length === 1), + 'INSTALLED_PARENT_SOFTWARE_INCOMPLETE'); + platforms[platform] = { cases: software.cases.map(({ id, status }) => ({ id, status })), + private_artifact_id: raw.artifact_id, private_archive_sha256: raw.archive_sha256, + private_job_id: raw.job_id, private_check_run_id: raw.check_run_id, + original_member_sha256: { ...software.raw_sha256 } }; + } + // Recheck both authorities after all downloads/validation. This is repeated readback, + // not an atomic GitHub snapshot; the final API phase rereads the public source and pins. + await window.revalidateInstalledParentWindow(github, context, request, admitted, { pinReader: options.pinReader }); + await privateSoftware.revalidatePrivateActionsSoftware(privateGithub, privateProof, { clock: options.clock }); + await canonical.revalidateInstalledPublicEvidence(github, context, request, publicProof, + { pinReader: options.pinReader, clock: options.clock }); + await window.revalidateInstalledParentWindow(github, context, request, admitted, { pinReader: options.pinReader }); + const result = frozen({ schema: 1, kind: 'installed-parent-original-software', app: admitted.app, + window_id: admitted.window_id, manifest_sha256: admitted.manifest_sha256, platforms, + installed_trust: 'UNVERIFIED', native_acceptance: false, release_ready: false, + authorization_eligible: false, execution_allowed: false, release_authorized: false }); + issued.add(result); publicBindings.set(result, publicProof); return result; +} +function projectInstalledParentCases(publicProof, privateProof) { + const canonical = require('./canonical_native.cjs'); + demand(canonical.isInstalledPublicEvidence(publicProof) && issued.has(privateProof) && + publicBindings.get(privateProof) === publicProof && + publicProof.source_sha === privateProof.app.sha && publicProof.source_tree === privateProof.app.tree, + 'INSTALLED_PARENT_COMPOSITION_CAPABILITIES_REQUIRED'); + return Object.fromEntries(['IOS', 'ANDROID'].map(platform => { + const original = publicProof.platforms[platform], parents = privateProof.platforms[platform].cases; + demand(original.cases.length === PARENTS.length + 2 && original.cases.slice(0, 2).every(row => row.status === 'PASS'), + 'INSTALLED_PARENT_PUBLIC_SOFTWARE_INCOMPLETE'); + return [platform, { ...original, cases: original.cases.map(row => PARENTS.includes(row.id) ? + { ...parents.find(parent => parent.id === row.id) } : { ...row }) }]; + })); +} +module.exports = { collectInstalledParentSoftware, projectInstalledParentCases, + isInstalledParentSoftware: value => issued.has(value) }; diff --git a/.github/native/installed_parent_window.cjs b/.github/native/installed_parent_window.cjs new file mode 100644 index 0000000000..00ddc8a2b6 --- /dev/null +++ b/.github/native/installed_parent_window.cjs @@ -0,0 +1,112 @@ +'use strict'; +// Read-only authority admission. This does not install a profile or write a pin. +const crypto = require('node:crypto'); +const { isDeepStrictEqual: equal } = require('node:util'); +const { parseStrictJson } = require('./strict_json.cjs'); +const WINDOW_PIN = 'KEEPER_CANONICAL_PRIVATE_WINDOW_JSON'; +// Offline source candidate for one reviewed stable producer baseline. This is +// not live installation. No environment, artifact, caller option or live variable +// can add another profile or change the committed producer/ref/raw contract. +const INSTALLED_PROFILES = Object.freeze({ + "r5-f0-intel-20261025": { + "repository": "KeeperCommunity/bitcoin-keeper-relay", + "repositoryId": 496119449, + "workflowPath": ".github/workflows/r5-native-integration.yml", + "ref": "refs/heads/codex/r5-cloud-native-execution-20261025", + "driverBase": "f0a695a6bba068ba93962e90ab52e8901ef4a852", + "backendSha": "29597b27eeeb299205bf7d0555db0a90c38cc4be", + "rawProfile": "D448_RAW13" + } +}); +const issued = new WeakSet(), originals = new WeakMap(); +const SHA = /^[a-f0-9]{40}$/, HEX = /^[a-f0-9]{64}$/; +const sha = value => typeof value === 'string' && SHA.test(value); +const hex = value => typeof value === 'string' && HEX.test(value); +const APP = 'KeeperCommunity/bitcoin-keeper', PRIVATE = 'KeeperCommunity/bitcoin-keeper-relay'; +const BACKEND = '29597b27eeeb299205bf7d0555db0a90c38cc4be'; +const WORKFLOW = '.github/workflows/r5-native-integration.yml'; +const REF = 'refs/heads/codex/r5-cloud-native-execution-20261025'; +const DRIVER = 'f0a695a6bba068ba93962e90ab52e8901ef4a852'; +const demand = (yes, code = 'INSTALLED_PARENT_WINDOW_INVALID') => { if (!yes) throw new Error(code); }; +const object = v => v !== null && typeof v === 'object' && !Array.isArray(v); +const keys = (v, names) => object(v) && Object.keys(v).sort().join('|') === [...names].sort().join('|'); +const id = v => Number.isSafeInteger(v) && v > 0; +const digest = bytes => crypto.createHash('sha256').update(bytes).digest('hex'); +function frozen(value) { + if (object(value) || Array.isArray(value)) { Object.values(value).forEach(frozen); Object.freeze(value); } + return value; +} +function validateManifest(value, request, tree) { + demand(keys(value, ['schema', 'profile', 'window_id', 'app', 'backend', 'public', 'private']) && value.schema === 1 && + hex(value.window_id) && typeof value.profile === 'string' && Object.hasOwn(INSTALLED_PROFILES, value.profile), + 'INSTALLED_PARENT_PROFILE_REQUIRED'); + const profile = INSTALLED_PROFILES[value.profile]; + demand(keys(profile, ['repository', 'repositoryId', 'workflowPath', 'ref', 'driverBase', 'backendSha', 'rawProfile']) && + profile.repository === PRIVATE && profile.repositoryId === 496119449 && profile.workflowPath === WORKFLOW && + profile.ref === REF && profile.driverBase === DRIVER && profile.backendSha === BACKEND && profile.rawProfile === 'D448_RAW13'); + demand(keys(value.app, ['repository', 'sha', 'tree']) && value.app.repository === APP && + sha(value.app.sha) && value.app.sha === request.sourceSha && sha(value.app.tree) && value.app.tree === tree); + demand(keys(value.backend, ['repository', 'sha']) && value.backend.repository === PRIVATE && value.backend.sha === BACKEND); + demand(keys(value.public, ['workflowPath', 'workflowSha', 'ref', 'runId', 'runAttempt']) && + value.public.workflowPath === '.github/workflows/keeper-canonical-native.yml' && + sha(value.public.workflowSha) && value.public.workflowSha === request.workflowSha && value.public.ref === 'refs/heads/sprint' && + value.public.runId === request.runId && value.public.runAttempt === request.attempt); + demand(keys(value.private, ['IOS', 'ANDROID'])); + for (const platform of ['IOS', 'ANDROID']) { + const d = value.private[platform]; + demand(keys(d, ['platform', 'app', 'control', 'backend', 'execution']) && d.platform === platform && + equal(d.app, value.app) && equal(d.backend, value.backend)); + demand(keys(d.control, ['repository', 'sha', 'tree']) && d.control.repository === PRIVATE && + sha(d.control.sha) && sha(d.control.tree)); + const e = d.execution; + demand(keys(e, ['repository', 'workflowPath', 'workflowSha', 'ref', 'runId', 'runAttempt']) && + e.repository === PRIVATE && e.workflowPath === WORKFLOW && sha(e.workflowSha) && e.workflowSha === d.control.sha && + e.ref === REF && id(e.runId) && id(e.runAttempt)); + } + demand(value.private.IOS.control.sha !== value.private.ANDROID.control.sha && + value.private.IOS.execution.runId !== value.private.ANDROID.execution.runId); + return value; +} +async function readWindow(github, context, request, pinReader) { + demand(Object.keys(INSTALLED_PROFILES).length > 0, 'INSTALLED_PARENT_PROFILE_REQUIRED'); + // Reuse the existing installed public driver checks and dedicated read-only + // Variables identity; the ordinary Actions client cannot supply a fallback. + const driver = require('./canonical_native.cjs'); + driver.requestFromEnvironment(context, { KEEPER_SOURCE_SHA: request.sourceSha, + GITHUB_WORKFLOW_SHA: request.workflowSha, GITHUB_WORKFLOW_REF: request.workflowRef, + KEEPER_CANONICAL_NATIVE_DRIVER_SHA: request.pin, GITHUB_RUN_ID: String(request.runId), + GITHUB_RUN_ATTEMPT: String(request.attempt) }); + await driver.origin(github, context, request, { pinReader }); + const [pin, commit] = await Promise.all([ + pinReader.rest.actions.getRepoVariable({ ...context.repo, name: WINDOW_PIN, + headers: { 'cache-control': 'no-cache' } }).then(r => r.data) + .catch(() => { throw new Error('INSTALLED_PARENT_WINDOW_UNREADABLE'); }), + github.rest.git.getCommit({ ...context.repo, commit_sha: request.sourceSha }).then(r => r.data), + ]); + demand(pin?.name === WINDOW_PIN && typeof pin.value === 'string' && Buffer.byteLength(pin.value) > 0 && + Buffer.byteLength(pin.value) <= 16384, 'INSTALLED_PARENT_WINDOW_UNREADABLE'); + demand(sha(commit?.sha) && commit.sha === request.sourceSha && sha(commit.tree?.sha)); + const bytes = Buffer.from(pin.value, 'utf8'); + let value; try { value = parseStrictJson(bytes, 16384); } catch { throw new Error('INSTALLED_PARENT_WINDOW_INVALID'); } + validateManifest(value, request, commit.tree.sha); + await driver.origin(github, context, request, { pinReader }); + return { value, sha256: digest(bytes) }; +} +async function authenticateInstalledParentWindow(github, context, request, { pinReader } = {}) { + try { + const { value, sha256 } = await readWindow(github, context, request, pinReader); + const capability = frozen({ schema: 1, kind: 'installed-reviewed-private-window', profile: value.profile, + window_id: value.window_id, manifest_sha256: sha256, app: value.app, backend: value.backend, + public: value.public, expectedPerPlatform: value.private, + release_ready: false, authorization_eligible: false, release_authorized: false, execution_allowed: false }); + originals.set(capability, sha256); issued.add(capability); return capability; + } catch (error) { throw new Error(/^INSTALLED_PARENT_[A-Z0-9_]+$/.test(error?.message || '') ? error.message : 'INSTALLED_PARENT_WINDOW_INVALID'); } +} +async function revalidateInstalledParentWindow(github, context, request, capability, { pinReader } = {}) { + demand(issued.has(capability), 'INSTALLED_PARENT_WINDOW_CAPABILITY_REQUIRED'); + const latest = await authenticateInstalledParentWindow(github, context, request, { pinReader }); + demand(latest.manifest_sha256 === originals.get(capability), 'INSTALLED_PARENT_WINDOW_CHANGED'); + return true; +} +module.exports = { WINDOW_PIN, authenticateInstalledParentWindow, revalidateInstalledParentWindow, + isInstalledParentWindow: value => issued.has(value) }; diff --git a/.github/native/ios_direct_launch_observation.py b/.github/native/ios_direct_launch_observation.py new file mode 100644 index 0000000000..0c8158152e --- /dev/null +++ b/.github/native/ios_direct_launch_observation.py @@ -0,0 +1,123 @@ +"""One failure-only simulator launch; never retry a test or grant acceptance.""" +import importlib.util +import os +from pathlib import Path +import re +import signal +import subprocess +import tempfile + +APP = 'io.hexawallet.hexakeeper.dev' +LIMIT = 65536 +DOMAINS = frozenset(('FBSOpenApplicationServiceErrorDomain', 'FBSOpenApplicationErrorDomain', + 'RBSRequestErrorDomain', 'RBSErrorDomain', 'NSPOSIXErrorDomain', 'NSCocoaErrorDomain', + 'NSOSStatusErrorDomain', 'NSMachErrorDomain', 'OSLaunchdErrorDomain', + 'BSServiceConnectionErrorDomain', 'FBProcessExit')) +PAIR = re.compile(r'(?:domain=|Error Domain=)([A-Za-z][A-Za-z0-9.]{0,95})' + r'(?:, code=| Code=)(-?(?:0|[1-9][0-9]{0,9}))\b') +SCOPE = 'FAILURE_ONLY_LATER_LAUNCH_NOT_ORIGINAL_CAUSE_UI_OR_NATIVE_ACCEPTANCE' + + +def codes(payload): + """Only allowlisted domain names and bounded integers leave private stderr.""" + result = {'pairs': [], 'input_partial': len(payload) > LIMIT, 'unknown_domain_observed': False} + text = payload[:LIMIT].decode('utf-8', errors='replace') + result['input_partial'] |= '\ufffd' in text + for match in PAIR.finditer(text): + domain, number = match.groups() + if domain not in DOMAINS: + result['unknown_domain_observed'] = True + continue + number = int(number) + if not -(2 ** 31) <= number < 2 ** 31: + result['input_partial'] = True + continue + pair = {'domain': domain, 'code': number} + if pair not in result['pairs']: + if len(result['pairs']) == 8: + result['input_partial'] = True + break + result['pairs'].append(pair) + return result + + +def process(private, device): + result = {'state': 'UNKNOWN', 'query': 'UNAVAILABLE', 'owned_entry_count': None} + try: + with tempfile.TemporaryFile(dir=private) as output: + outcome = subprocess.run(['xcrun', 'simctl', 'spawn', device, 'launchctl', 'list'], + stdout=output, stderr=subprocess.DEVNULL, timeout=5, check=False) + if outcome.returncode != 0: + return {**result, 'query': 'NONZERO_EXIT'} + output.seek(0) + spec = importlib.util.spec_from_file_location('direct_launch_process_parser', + Path(__file__).with_name('ios_startup_command_observation.py')) + parser = importlib.util.module_from_spec(spec) + spec.loader.exec_module(parser) + projected = parser.process_entries(output.read(1024 * 1024 + 1)) + return {**projected, 'query': 'EXIT_ZERO'} + except subprocess.TimeoutExpired: + return {**result, 'query': 'TIMEOUT'} + except Exception: + return result + + +def collect(native, device): + """Run after original failure observations, before normal owned cleanup. + + No app-state reset, termination request, signing, timeout extension or test + retry occurs. The original smoke remains FAIL, even if this later launch + is accepted. An empty exclusive marker prevents a second diagnostic launch. + """ + result = {'schema': 1, 'state': 'NOT_ELIGIBLE', 'command': 'NOT_RUN', 'timeout_seconds': 20, + 'launch_response': 'UNKNOWN', 'error_codes': {'pairs': [], 'input_partial': False, + 'unknown_domain_observed': False}, 'before_process': None, 'after_process': None, + 'cause': 'UNKNOWN', 'native_acceptance': False, 'scope': SCOPE} + data = native.read() + rows = data.get('startup_diagnostics', {}).get('command_observation', {}).get('commands', {}).get('rows', []) + if (native.PLATFORM != 'IOS' or type(device) is not str or not re.fullmatch( + r'[A-Fa-f0-9]{8}(?:-[A-Fa-f0-9]{4}){3}-[A-Fa-f0-9]{12}', device) or + data.get('build') != 'PASS' or data.get('smoke') not in ('NOT RUN', 'FAIL') or + getattr(native, 'original_smoke_failed', False) is not True or + data.get('install_succeeded') is not True or data.get('installed_artifact_verified') is not True or + data.get('device', {}).get('id') != device or + data.get('device', {}).get('fresh_disposable_state') is not True or + type(rows) is not list or not any(type(row) is dict and row.get('kind') == 'LAUNCH_APP' + and row.get('status') == 'FAILED' for row in rows)): + return result + try: + descriptor = os.open(native.PRIVATE / 'ios-direct-launch.once', + os.O_WRONLY | os.O_CREAT | os.O_EXCL | os.O_NOFOLLOW, 0o600) + os.close(descriptor) + except FileExistsError: + return {**result, 'state': 'ALREADY_ATTEMPTED'} + result['state'] = 'OBSERVED' + result['before_process'] = process(native.PRIVATE, device) + try: + with tempfile.TemporaryFile(dir=native.PRIVATE) as output, tempfile.TemporaryFile(dir=native.PRIVATE) as error: + child = subprocess.Popen(['xcrun', 'simctl', 'launch', device, APP], + stdout=output, stderr=error, start_new_session=True) + try: + exit_code = child.wait(timeout=20) + result['command'] = 'EXIT_ZERO' if exit_code == 0 else 'NONZERO_EXIT' + except subprocess.TimeoutExpired: + result['command'] = 'TIMEOUT' + os.killpg(child.pid, signal.SIGKILL) + child.wait(timeout=5) + finally: + # Any exceptional wait path also disposes the owned command. + if child.poll() is None: + os.killpg(child.pid, signal.SIGKILL) + child.wait(timeout=5) + output.seek(0) + response = output.read(LIMIT + 1) + if result['command'] == 'EXIT_ZERO' and re.fullmatch( + re.escape(APP).encode() + rb': [1-9][0-9]{0,9}\n?', response): + result['launch_response'] = 'ACCEPTED_AT_LATER_TIME' + error.seek(0) + result['error_codes'] = codes(error.read(LIMIT + 1)) + except Exception: + result['command'] = 'UNAVAILABLE' + finally: + result['after_process'] = process(native.PRIVATE, device) + return result diff --git a/.github/native/ios_replacement_observation.py b/.github/native/ios_replacement_observation.py new file mode 100644 index 0000000000..88a751d3e6 --- /dev/null +++ b/.github/native/ios_replacement_observation.py @@ -0,0 +1,146 @@ +"""Read-only post-failure bundle observations; never repair or acceptance. + +Maestro clearState replaces the first installed instance. Inspect the remaining +owned instance after failure without a launch, retry or metadata promotion. +""" +import importlib.util +from pathlib import Path +import plistlib +import re +import subprocess +import tempfile + +APP = 'io.hexawallet.hexakeeper.dev' +LIMIT = 1024 * 1024 +UUID = r'[A-Fa-f0-9]{8}(?:-[A-Fa-f0-9]{4}){3}-[A-Fa-f0-9]{12}' + + +def query(private, args): + """Keep output in an unnamed private file; retain no command or raw text.""" + try: + with tempfile.TemporaryFile(dir=private) as output: + process = subprocess.run(args, stdout=output, stderr=subprocess.STDOUT, + timeout=20, check=False) + output.seek(0) + payload = output.read(LIMIT + 1) + if len(payload) > LIMIT: + return 'OVERSIZED', None + if process.returncode != 0: + return 'NONZERO_EXIT', None + return 'EXIT_ZERO', payload + except subprocess.TimeoutExpired: + return 'TIMEOUT', None + except OSError: + return 'UNAVAILABLE', None + + +def entitlement_state(payload): + """Compare types and complete effective values to the fixed signing policy.""" + try: + if type(payload) is not bytes or len(payload) > LIMIT: + return 'UNKNOWN' + begin, end = payload.find(b'') + if (begin < 0 or end < begin or payload.count(b'') != 1): + return 'UNKNOWN' + effective = plistlib.loads(payload[begin:end + len(b'')]) + spec = importlib.util.spec_from_file_location('observed_simulator_policy', + Path(__file__).with_name('simulator_signing.py')) + policy = importlib.util.module_from_spec(spec) + spec.loader.exec_module(policy) + if type(effective) is not dict: + return 'UNKNOWN' + return ('MATCH' if plistlib.dumps(effective, sort_keys=True) == + plistlib.dumps(policy.ENTITLEMENTS, sort_keys=True) else 'MISMATCH') + except Exception: + return 'UNKNOWN' + + +def unknown(stage='NOT_RUN'): + return {'schema': 1, 'state': 'UNKNOWN', 'stage': stage, + 'bytes': 'UNKNOWN', 'signature': 'UNKNOWN', 'identity': 'UNKNOWN', + 'entitlements': 'UNKNOWN', 'built_internal_links': None, + 'installed_internal_links': None, 'queries': {}, + 'scope': 'POST_FAILURE_CURRENT_INSTANCE_NOT_LAUNCH_CAUSE_OR_ACCEPTANCE'} + + +def owned_observation_bundle_path(native, value, device, built): + """Admit the pinned producer's name without changing acceptance policy. + + Maestro 2.11 LocalSimulatorUtils.kt:260 copies clearState's replacement to + "$bundleId-${System.currentTimeMillis()}.app". Only the validation + descriptor's basename differs; all existing owned-device/container/realpath + checks are reused. The actual built path/inventory is never substituted. + """ + installed = Path(value) + descriptor = built + if installed.name != built.name: + if (getattr(native, 'MAESTRO_VERSION', None) != 'cli-2.11.0' or + not re.fullmatch(re.escape(APP) + r'-[1-9][0-9]{12}\.app', installed.name)): + raise ValueError('Unsupported replacement bundle name.') + descriptor = built.with_name(installed.name) + return native.owned_ios_bundle_path(value, device, descriptor) + + +def collect(native, device): + result = unknown() + stage = 'DEVICE_POLICY' + try: + if (native.PLATFORM != 'IOS' or type(device) is not str or + not re.fullmatch(UUID, device)): + return unknown(stage) + stage = 'BUILT_ARTIFACT_STABILITY' + expected = native.read()['built_artifact'] + if native.artifact_identity() != expected: + return unknown(stage) + built = native.PRIVATE / 'DerivedData/Build/Products/Debug-iphonesimulator/hexa_keeper_dev.app' + stage = 'BUILT_INVENTORY' + original, executable = native._bundle_inventory(built) + get_container = ['xcrun', 'simctl', 'get_app_container', device, APP, 'app'] + stage = 'INSTALLED_CONTAINER_QUERY' + status, payload = query(native.PRIVATE, get_container) + result['queries']['container'] = status + if status != 'EXIT_ZERO': + return {**result, 'stage': stage} + stage = 'INSTALLED_CONTAINER_POLICY' + installed = owned_observation_bundle_path(native, payload.decode('utf-8').strip(), device, built) + stage = 'INSTALLED_INVENTORY' + actual, actual_executable = native._bundle_inventory(installed) + result['bytes'] = 'MATCH' if (original, executable) == (actual, actual_executable) else 'MISMATCH' + result['built_internal_links'] = sum(value[0] == 'link' for value in original.values()) + result['installed_internal_links'] = sum(value[0] == 'link' for value in actual.values()) + stage = 'SIGNATURE_QUERIES' + status, _ = query(native.PRIVATE, ['codesign', '--verify', '--deep', '--strict', str(installed)]) + result['queries']['signature'] = status + result['signature'] = ('VERIFIED' if status == 'EXIT_ZERO' else + 'NOT_VERIFIED' if status == 'NONZERO_EXIT' else 'UNKNOWN') + status, payload = query(native.PRIVATE, ['codesign', '--display', '--verbose=4', str(installed)]) + result['queries']['identity'] = status + if status == 'EXIT_ZERO': + lines = payload.decode('utf-8').splitlines() + result['identity'] = ('ADHOC' if 'Signature=adhoc' in lines and + not any(line.startswith('Authority=') for line in lines) else 'OTHER') + status, payload = query(native.PRIVATE, ['codesign', '--display', '--entitlements', ':-', str(installed)]) + result['queries']['entitlements'] = status + if status == 'EXIT_ZERO': + result['entitlements'] = entitlement_state(payload) + # All conclusions refer to one unchanged current instance and built input. + stage = 'FINAL_CONTAINER_QUERY' + status, payload = query(native.PRIVATE, get_container) + result['queries']['final_container'] = status + if status != 'EXIT_ZERO': + return unknown(stage) + final = owned_observation_bundle_path(native, payload.decode('utf-8').strip(), device, built) + stage = 'INSTALLED_STABILITY' + if final != installed or native._bundle_inventory(final) != (actual, actual_executable): + return unknown(stage) + stage = 'BUILT_ARTIFACT_STABILITY' + if (native._bundle_inventory(built) != (original, executable) or + native.artifact_identity() != expected): + return unknown(stage) + result.update(state='OBSERVED', stage='COMPLETE') + return result + except Exception: + # Unsupported/missing bundles and failed collectors supply no pass and + # never replace the original startup exception or export its message. + return unknown(stage) diff --git a/.github/native/ios_startup_command_observation.py b/.github/native/ios_startup_command_observation.py new file mode 100644 index 0000000000..064d7db08d --- /dev/null +++ b/.github/native/ios_startup_command_observation.py @@ -0,0 +1,293 @@ +"""Failure-only fixed Maestro command and simulator service observations. + +No launch, app-state mutation, raw error, identifier, PID or hierarchy is +exported. These observations cannot establish launch cause or native acceptance. +""" +from datetime import datetime +import importlib.util +import json +from pathlib import Path +import re +import subprocess +import tempfile + +APP = 'io.hexawallet.hexakeeper.dev' +FLOW = 'Disposable startup and passcode input' +LIMIT = 1024 * 1024 +KINDS = { + 'applyConfigurationCommand': 'CONFIGURATION', + 'launchAppCommand': 'LAUNCH_APP', + 'clearStateCommand': 'CLEAR_STATE', + 'stopAppCommand': 'STOP_APP', + 'assertConditionCommand': 'ASSERT_CONDITION', + 'tapOnElement': 'TAP_ELEMENT', +} +STATUSES = ('PENDING', 'RUNNING', 'COMPLETED', 'FAILED', 'WARNED', 'SKIPPED') +ERROR_MARKERS = { + 'APP_LAUNCH_TEXT': r'Unable to launch app|Failed to launch app|Cannot launch app', + 'APP_NOT_INSTALLED_TEXT': r'app(?:lication)? is not installed|Application not installed', + 'DEVICE_DISCONNECTED_TEXT': r'was requested, but it is not connected|No devices found|No connected devices', + 'DRIVER_CONNECTION_TEXT': r'Unable to connect|Failed to connect|Connection refused|ConnectException|DeviceConnectionException', + 'DRIVER_TIMEOUT_TEXT': r'IOSDriverTimeoutException|iOS driver not ready in time', + 'FBS_OPEN_APPLICATION_TEXT': r'\bFBSOpenApplicationServiceErrorDomain\b|\bFBSOpenApplicationErrorDomain\b', + 'SIMCTL_OPERATION_TEXT': r'An error was encountered processing the command', + 'ASSERTION_TEXT': r'Assertion is false:|Element not found:', +} +ERROR_DOMAINS = frozenset(('FBSOpenApplicationServiceErrorDomain', 'FBSOpenApplicationErrorDomain', + 'RBSRequestErrorDomain', 'RBSErrorDomain', 'NSPOSIXErrorDomain', 'NSCocoaErrorDomain', + 'NSOSStatusErrorDomain', 'NSMachErrorDomain')) +LAUNCH_SERVICES = frozenset(('SpringBoard', 'runningboardd', 'launchd', 'lsd')) +LAUNCH_MARKERS = { + 'CAPABILITY_TEXT': r'\b(?:UIRequiredDeviceCapabilities|capabilit(?:y|ies))\b', + 'ARCHITECTURE_TEXT': r'\b(?:architecture|armv7|arm64|x86_64|EBADARCH)\b', + 'SIGNATURE_TEXT': r'\b(?:signature|codesign|AMFI|entitlement|entitlements)\b', + 'NOT_FOUND_TEXT': r'\b(?:not found|NotFound|unknown to FrontBoard|no such file)\b', + 'PERMISSION_TEXT': r'\b(?:operation not permitted|permission denied|EPERM|EACCES)\b', + 'SPAWN_FAILURE_TEXT': r'\b(?:failed to (?:launch|spawn)|spawn failed|launch failed)\b', + 'DYLD_TEXT': r'\b(?:dyld|Library not loaded|Symbol not found)\b', +} + + +def launch_error_codes(error): + """Project exact simctl header pairs, without descriptions or interpretation.""" + result = {'state': 'UNKNOWN', 'pairs': [], 'scope': 'RECORDED_CODES_NOT_INTERPRETED_CAUSE'} + pairs = [] + if (type(error) is not dict or not set(error) <= {'message', 'debugMessage'} or + any(type(value) is not str or len(value) > 65536 for value in error.values())): + return result + pattern = (r'(?:An error was encountered processing the command|Underlying error) ' + r'\(domain=([A-Za-z][A-Za-z0-9.]{0,95}), code=(-?(?:0|[1-9][0-9]{0,9}))\):') + for text in error.values(): + for line in text.splitlines(): + if 'domain=' not in line and 'Error Domain=' not in line: + continue + match = re.fullmatch(pattern, line.strip(' \t')) + if not match or match[1] not in ERROR_DOMAINS: + return result + code = int(match[2]) + if not -(2 ** 31) <= code < 2 ** 31: + return result + pair = {'domain': match[1], 'code': code} + if pair not in pairs: + pairs.append(pair) + if len(pairs) > 8: + return result + if pairs: + result.update(state='RECORDED', pairs=pairs) + return result + + +def unknown(): + return {'schema': 1, 'state': 'UNKNOWN', 'rows': [], + 'scope': 'RECORDED_COMMAND_STATUS_NOT_COMPLETION_OR_LAUNCH_CAUSE'} + + +def commands(directory): + result = unknown() + try: + spec = importlib.util.spec_from_file_location('startup_private_json', + Path(__file__).with_name('maestro_failure_snapshot.py')) + reader = importlib.util.module_from_spec(spec) + spec.loader.exec_module(reader) + path = reader.commands_file(directory) + if path.parent != directory / FLOW: + return result + source = reader.read_json(path, directory) + if type(source) is not list or not 1 <= len(source) <= 64: + return result + rows, previous = [], -1 + fields = {'status', 'timestamp', 'duration', 'error', 'sequenceNumber', + 'depth', 'evaluatedCommand', 'artifacts'} + for item in source: + if type(item) is not dict or set(item) != {'command', 'metadata'}: + return result + command, metadata = item['command'], item['metadata'] + if (type(command) is not dict or len(command) > 1 or + type(metadata) is not dict or not set(metadata) <= fields or + metadata.get('status') not in STATUSES): + return result + sequence = metadata.get('sequenceNumber') + if type(sequence) is not int or not previous < sequence < 64: + return result + previous = sequence + if type(metadata.get('depth', 0)) is not int or not 0 <= metadata.get('depth', 0) <= 8: + return result + for field in ('timestamp', 'duration'): + if field in metadata and (type(metadata[field]) is not int or not 0 <= metadata[field] <= 10 ** 14): + return result + error = metadata.get('error', {}) + if (type(error) is not dict or not set(error) <= {'message', 'debugMessage'} or + any(type(value) is not str for value in error.values())): + return result + text = '\n'.join(error.values()) + key = next(iter(command), None) + kind = KINDS.get(key, 'OTHER') + if key in ('launchAppCommand', 'clearStateCommand', 'stopAppCommand'): + config = command[key] + if type(config) is not dict or config.get('appId') not in (APP, '${APP_ID}'): + kind = 'OTHER' + if 'evaluatedCommand' in metadata: + evaluated = metadata['evaluatedCommand'] + if (type(evaluated) is not dict or set(evaluated) != {key} or + type(evaluated[key]) is not dict or evaluated[key].get('appId') != APP): + return result + row = {'sequence': sequence, 'kind': kind, + 'status': metadata['status'], + 'error_markers': [label for label, pattern in ERROR_MARKERS.items() + if re.search(pattern, text, re.IGNORECASE)]} + if kind == 'LAUNCH_APP' and metadata['status'] == 'FAILED': + row['launch_error_codes'] = launch_error_codes(error) + rows.append(row) + result.update(state='RECORDED', rows=rows) + except (OSError, ValueError, TypeError, RecursionError): + pass + return result + + +def process_entries(payload): + """Select only the exact owned app's UIKitApplication service label.""" + result = {'state': 'UNKNOWN', 'scope': 'POST_FAILURE_SERVICE_ENTRY_NOT_LAUNCH_HISTORY', + 'owned_entry_count': None} + if type(payload) is not bytes or len(payload) > LIMIT: + return result + try: + lines = payload.decode('utf-8').splitlines() + except UnicodeDecodeError: + return result + if not lines or not re.fullmatch(r'PID\s+Status\s+Label', lines[0]): + return result + found = [] + label = r'UIKitApplication:' + re.escape(APP) + r'(?:\[[A-Za-z0-9_.:-]{1,128}\]){0,3}' + for line in lines[1:]: + row = re.fullmatch(r'(-|[1-9][0-9]{0,9})\s+(-?[0-9]{1,10})\s+(\S+)', line) + if not row: + return result + if re.fullmatch(label, row[3]): + found.append(row[1] != '-') + if len(found) > 1: + return result + result.update(owned_entry_count=len(found), state=( + 'OWNED_RUNNING_ENTRY_OBSERVED' if found == [True] else + 'OWNED_NONRUNNING_ENTRY_OBSERVED' if found == [False] else + 'NO_OWNED_ENTRY_OBSERVED')) + return result + + +def process(private, device): + result = {'state': 'UNKNOWN', 'scope': 'POST_FAILURE_SERVICE_ENTRY_NOT_LAUNCH_HISTORY', + 'owned_entry_count': None, 'query': 'NOT_RUN'} + if type(device) is not str or not re.fullmatch(r'[A-Fa-f0-9]{8}(?:-[A-Fa-f0-9]{4}){3}-[A-Fa-f0-9]{12}', device): + return result + try: + # An unnamed private file is never in an upload glob and closes even + # on timeout. Query the simulator service manager; do not launch an app. + with tempfile.TemporaryFile(dir=private) as output: + outcome = subprocess.run(['xcrun', 'simctl', 'spawn', device, 'launchctl', 'list'], + stdout=output, stderr=subprocess.DEVNULL, timeout=15, check=False) + if outcome.returncode != 0: + return {**result, 'query': 'NONZERO_EXIT'} + output.seek(0) + return {**process_entries(output.read(LIMIT + 1)), 'query': 'EXIT_ZERO'} + except subprocess.TimeoutExpired: + return {**result, 'query': 'TIMEOUT'} + except OSError: + return {**result, 'query': 'UNAVAILABLE'} + + +def launch_services_unknown(): + return {'state': 'UNKNOWN', 'query': 'NOT_RUN', 'matching_records': 0, 'rows': [], + 'input_partial': False, 'scope': 'OWNED_TIME_BOUND_SYSTEM_LOG_MARKERS_NOT_CAUSE_OR_ACCEPTANCE'} + + +def launch_window(started_ms, ended_ms): + return (type(started_ms) is int and type(ended_ms) is int and + 0 <= started_ms < ended_ms <= 10 ** 14 and ended_ms - started_ms <= 900000) + + +def launch_service_records(payload, started_ms, ended_ms): + result = launch_services_unknown() + if not launch_window(started_ms, ended_ms) or type(payload) is not bytes: + return result + if len(payload) > LIMIT: + return {**result, 'input_partial': True} + try: + def unique(pairs): + value = {} + for key, item in pairs: + if key in value: + raise ValueError('Duplicate log key.') + value[key] = item + return value + records = json.loads(payload.decode('utf-8'), object_pairs_hook=unique, + parse_constant=lambda value: (_ for _ in ()).throw(ValueError('Nonfinite log value.'))) + if type(records) is not list: + return result + result['input_partial'] = len(records) > 512 + for record in records[:512]: + if type(record) is not dict: + result['input_partial'] = True + continue + path, message, timestamp = (record.get(key) for key in + ('processImagePath', 'eventMessage', 'timestamp')) + if (type(path) is not str or not path.startswith('/') or + Path(path).name not in LAUNCH_SERVICES or type(message) is not str or + len(message) > 16384 or type(timestamp) is not str or + not re.fullmatch(r'\d{4}-\d{2}-\d{2}[ T]\d{2}:\d{2}:\d{2}(?:\.\d{1,9})?(?:Z|[+-]\d{2}:?\d{2})', timestamp)): + continue + try: + instant = datetime.fromisoformat(timestamp.replace('Z', '+00:00')).timestamp() * 1000 + except (ValueError, OverflowError): + continue + if (not started_ms <= instant <= ended_ms or not re.search( + r'(?\d{4}-\d{2}-\d{2}[ T]\d{2}:\d{2}:\d{2}(?:\.\d{1,9})?' + r'(?:[+-]\d{2}:?\d{2}|Z)?)\s+(?P[A-Za-z]+)\s+' + r'(?P[A-Za-z0-9_.-]{1,128})\[(?P[1-9]\d{0,9}):' + r'(?:0x)?[A-Fa-f0-9]{1,16}\]\s+(?P.*)') +_SEVERITIES = {'a', 'd', 'db', 'df', 'i', 'info', 'default', 'debug', + 'e', 'er', 'error', 'f', 'ft', 'fault'} +_ERROR_SEVERITIES = {'e', 'er', 'error', 'f', 'ft', 'fault'} +_ERROR_MARKER = re.compile( + r'\b(?:error|exception|fatal)\b|\b[\w.$]*(?:Exception|Error)\b', re.IGNORECASE) + + +def classify(payload, executable, started_ms=None, ended_ms=None, input_partial=False): + """Return fixed positive observations only; an empty result proves no pass. + + Invalid selectors/types raise fixed errors. Oversized input is classified + only through its complete bounded prefix; long or malformed records are + skipped. A supplied prefix cannot reveal truncation done by an earlier read. + """ + if (not isinstance(executable, str) or + not re.fullmatch(r'[A-Za-z0-9_][A-Za-z0-9_.-]{0,127}', executable)): + raise ValueError('Unsafe iOS executable selector.') + if not isinstance(payload, str): + raise TypeError('Expected compact log text.') + receipt, receipt_failed, receipt_frame_partial = None, False, False + try: + spec = importlib.util.spec_from_file_location( + 'startup_failure_receipts', Path(__file__).with_name('startup_failure_receipts.py')) + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + receipt = module.Collector(executable, started_ms, ended_ms) + except Exception: + receipt_failed = True + + def note_invalid_marker(line): + nonlocal receipt_failed + if receipt is not None and 'KEEPER_STARTUP_FAILURE_V1' in line: + try: + receipt.invalid_record(True) + except Exception: + receipt_failed = True + + # Slice characters before encoding so even an arbitrarily large argument + # creates at most a bounded four-byte-per-character temporary buffer. + encoded = payload[:MAX_INPUT_BYTES + 1].encode('utf-8', errors='replace') + truncated = len(payload) > MAX_INPUT_BYTES or len(encoded) > MAX_INPUT_BYTES + prefix = encoded[:MAX_INPUT_BYTES].decode('utf-8', errors='replace') + if truncated: + prefix = prefix.rsplit('\n', 1)[0] + '\n' if '\n' in prefix else '' + result = { + 'format': 'compact', 'matching_process_records': 0, 'errors': [], + 'error_records': 0, 'category_record_counts': {name: 0 for name in CATEGORIES}, + 'unclassified_error_records': 0, 'malformed_record_lines': 0, + 'oversized_record_lines': 0, 'input_truncated': truncated, + 'replacement_character_observed': '\ufffd' in prefix, + 'scan_limit_reached': False, 'record_limit_reached': False, + 'parse_status': 'UNKNOWN', 'cause': 'UNDETERMINED', + 'scope': 'PREFIXED_OWN_PROCESS_MARKERS_ONLY_NOT_CAUSE_OR_ACCEPTANCE', + } + for index, raw in enumerate(io.StringIO(prefix)): + if index >= MAX_SCAN_LINES: + result['scan_limit_reached'] = True + break + if result['matching_process_records'] >= MAX_OWN_RECORDS: + result['record_limit_reached'] = True + break + line = raw.removesuffix('\n').removesuffix('\r') + if len(line) > MAX_RECORD_BYTES or len(line.encode('utf-8')) > MAX_RECORD_BYTES: + result['oversized_record_lines'] += 1 + note_invalid_marker(line) + continue + match = _RECORD.fullmatch(line) + if not match: + # Headers, notices and unprefixed multiline continuations are not + # attributed. Timestamp-shaped malformed records remain visible. + result['malformed_record_lines'] += int(bool(re.match(r'^\d{4}-', line))) + note_invalid_marker(line) + continue + try: + datetime.fromisoformat(match['timestamp'].replace('Z', '+00:00')) + valid = (match['severity'].lower() in _SEVERITIES and + int(match['pid']) <= 2147483647 and + not any(ord(char) < 32 and char != '\t' for char in line)) + except ValueError: + valid = False + if not valid: + result['malformed_record_lines'] += 1 + note_invalid_marker(line) + continue + if receipt is not None: + try: + if any(ord(char) > 127 for char in line[:match.start('message')]): + receipt_frame_partial = True + note_invalid_marker(line) + elif ('KEEPER_STARTUP_FAILURE_V1' in match['message'] and + match['severity'].lower() not in _ERROR_SEVERITIES): + receipt.invalid_record(True) + else: + receipt.observe(match['timestamp'], match['process'], match['pid'], match['message']) + except Exception: + receipt_failed = True + if match['process'] != executable: + continue + result['matching_process_records'] += 1 + categories = [name for name, pattern in _CATEGORY_PATTERNS.items() + if pattern.search(match['message'])] + error = (bool(categories) or match['severity'].lower() in _ERROR_SEVERITIES or + bool(_ERROR_MARKER.search(match['message']))) + if error: + result['error_records'] += 1 + for category in categories: + result['category_record_counts'][category] += 1 + result['unclassified_error_records'] += int(not categories) + if result['error_records']: + result['errors'] = ['IOS_STARTUP_ERROR'] + partial = any(result[name] for name in ( + 'input_truncated', 'scan_limit_reached', 'record_limit_reached', + 'malformed_record_lines', 'oversized_record_lines', + 'replacement_character_observed')) + if result['matching_process_records']: + result['parse_status'] = 'PARTIAL' if partial else 'PARSED' + result['startup_failure_receipt'] = { + 'schema': 1, 'status': 'UNKNOWN', 'records': [], 'cause': 'UNDETERMINED', + 'reasons': ['COLLECTOR_UNAVAILABLE'], + 'scope': 'OPTIONAL_ORIGINAL_STARTUP_MARKERS_NOT_CAUSE_OR_ACCEPTANCE'} + if receipt is not None and not receipt_failed: + try: + result['startup_failure_receipt'] = receipt.result( + input_partial=partial or receipt_frame_partial or input_partial is not False) + except Exception: + pass + return result diff --git a/.github/native/maestro_failure_snapshot.py b/.github/native/maestro_failure_snapshot.py new file mode 100644 index 0000000000..08d22a0095 --- /dev/null +++ b/.github/native/maestro_failure_snapshot.py @@ -0,0 +1,266 @@ +"""Project one Maestro 2.11 failed-assertion outcome; never tap-time evidence. + +Format inspected at upstream c8c361be1cc89b2e742f185466a302a22778be6b: +TestOutputWriter, FlowDebugOutput, ArtifactsGenerator, ArtifactCollector, +TreeNode and AndroidDriver. commands.json carries per-execution sequenceNumber +and artifacts-folder-relative SCREEN_HIERARCHY paths. Passing taps have no +hierarchy; neither commands nor the outcome hierarchy records resolved taps. +""" +import json +import math +import os +from pathlib import Path, PurePosixPath +import re +import stat + +FILE_LIMIT = 1024 * 1024 +ENTRY_LIMIT = 512 +STEP_LIMIT = 2048 +NODE_LIMIT = 10000 +COORD_LIMIT = 100000 +FLAGS = ('clickable', 'enabled', 'focused', 'selected') +SCOPE = 'POST_FAILED_ASSERT_OUTCOME_NOT_TAP_TIME' +# TestRunner/TestSuiteInteractor use the checked-in YAML config.name for the +# direct per-flow folder. These commands use no shard/retry CLI option. +FLOWS = {'product_first_wallet.yaml': 'Full app startup first wallet and cold persistence', + 'product_retry.yaml': 'Full app controlled setup disconnect Retry and cold persistence'} +STATUSES = ('PENDING', 'RUNNING', 'COMPLETED', 'FAILED', 'WARNED', 'SKIPPED') + + +def unknown(reason='UNAVAILABLE'): + return {'schema': 1, 'binding_state': 'UNKNOWN', 'reason': reason, + 'scope': SCOPE, 'failed_assertion': None, 'node_count': None, + 'skip': {'state': 'UNKNOWN', 'exact_resource_id_count': None, + 'bounds': None, **dict.fromkeys(FLAGS), + 'conflicts': dict.fromkeys(FLAGS)}, + 'tap_coordinates': 'NOT_RECORDED', 'visible_to_user': 'NOT_RECORDED'} + + +def reject(): + raise ValueError() + + +def pairs(items): + result = {} + for key, value in items: + if key in result: + reject() + result[key] = value + return result + + +def read_json(path, root): + if root.is_symlink() or not root.is_dir() or not path.resolve().is_relative_to(root.resolve()): + reject() + for item in (path, *path.parents): + if item.is_symlink(): + reject() + if item == root: + break + before = path.stat() + if not stat.S_ISREG(before.st_mode) or before.st_size > FILE_LIMIT: + reject() + with os.fdopen(os.open(path, os.O_RDONLY | os.O_NOFOLLOW | os.O_NONBLOCK), 'rb') as stream: + actual = os.fstat(stream.fileno()) + if (actual.st_dev, actual.st_ino, actual.st_size) != (before.st_dev, before.st_ino, before.st_size): + reject() + payload = stream.read(FILE_LIMIT + 1) + after = os.fstat(stream.fileno()) + if (actual.st_size, actual.st_mtime_ns) != (after.st_size, after.st_mtime_ns): + reject() + if len(payload) > FILE_LIMIT: + reject() + data = json.loads(payload.decode('utf-8'), object_pairs_hook=pairs, parse_constant=lambda _: reject()) + # Bound even ignored command payloads, including embedded scripts/errors. + pending, count = [(data, 0)], 0 + while pending: + item, depth = pending.pop() + count += 1 + if depth > 128 or count > 50000: + reject() + if type(item) is dict: + if len(item) > 256 or any(type(key) is not str for key in item): + reject() + pending.extend((value, depth + 1) for value in item.values()) + elif type(item) is list: + if len(item) > NODE_LIMIT: + reject() + pending.extend((value, depth + 1) for value in item) + elif type(item) is str and len(item) > 65536: + reject() + elif type(item) in (int, float) and (not math.isfinite(item) or abs(item) > 2 ** 53 - 1): + reject() + return data + + +def commands_file(root): + if root.is_symlink() or not root.is_dir(): + reject() + pending, found, count = [(root, 0)], [], 0 + while pending: + directory, depth = pending.pop() + if depth > 6: + reject() + with os.scandir(directory) as entries: + for entry in entries: + count += 1 + if count > ENTRY_LIMIT or entry.is_symlink(): + reject() + if entry.is_dir(follow_symlinks=False): + pending.append((Path(entry.path), depth + 1)) + elif entry.name == 'commands.json': + found.append(Path(entry.path)) + if len(found) > 1: + reject() + if len(found) != 1: + reject() + return found[0] + + +def target(command): + if type(command) is not dict or set(command) != {'assertConditionCommand'}: + return False + assertion = command['assertConditionCommand'] + if (type(assertion) is not dict or + not set(assertion) <= {'condition', 'timeout', 'label', 'optional'} or + assertion.get('optional', False) is not False or assertion.get('label') is not None): + return False + timeout = assertion.get('timeout') + if timeout is not None and (type(timeout) is not str or not re.fullmatch(r'[0-9]{1,10}', timeout)): + return False + condition = assertion.get('condition') + if type(condition) is not dict or set(condition) != {'visible'}: + return False + visible = condition['visible'] + return (type(visible) is dict and set(visible) <= {'idRegex', 'optional'} and + visible.get('idRegex') == 'view_startNewTile' and visible.get('optional', False) is False) + + +def failed_step(rows): + if type(rows) is not list or not 1 <= len(rows) <= STEP_LIMIT: + reject() + candidates, previous, artifact_paths = [], -1, set() + metadata_fields = {'status', 'timestamp', 'duration', 'error', 'sequenceNumber', + 'depth', 'evaluatedCommand', 'artifacts'} + for row in rows: + if type(row) is not dict or set(row) != {'command', 'metadata'}: + reject() + metadata, command = row['metadata'], row['command'] + if (type(command) is not dict or len(command) > 1 or type(metadata) is not dict or + not set(metadata) <= metadata_fields or metadata.get('status') not in STATUSES): + reject() + sequence = metadata.get('sequenceNumber') + if type(sequence) is not int or not previous < sequence < STEP_LIMIT: + reject() + previous = sequence + if type(metadata.get('depth', 0)) is not int or not 0 <= metadata.get('depth', 0) <= 128: + reject() + for field in ('timestamp', 'duration'): + if field in metadata and (type(metadata[field]) is not int or not 0 <= metadata[field] <= 10 ** 14): + reject() + if 'error' in metadata and (type(metadata['error']) is not dict or + not set(metadata['error']) <= {'message', 'debugMessage'} or + any(type(value) is not str for value in metadata['error'].values())): + reject() + if 'artifacts' in metadata and type(metadata['artifacts']) is not list: + reject() + artifacts = metadata.get('artifacts', []) + if len(artifacts) > 32: + reject() + for artifact in artifacts: + if (type(artifact) is not dict or set(artifact) != {'type', 'path'} or + type(artifact['type']) is not str or type(artifact['path']) is not str): + reject() + if artifact['path'] in artifact_paths: + reject() + artifact_paths.add(artifact['path']) + if metadata['status'] == 'FAILED' and target(command): + if 'evaluatedCommand' in metadata and not target(metadata['evaluatedCommand']): + reject() + candidates.append((sequence, artifacts)) + if len(candidates) != 1: + reject() + sequence, artifacts = candidates[0] + snapshots = [item['path'] for item in artifacts if item['type'] == 'SCREEN_HIERARCHY'] + if len(snapshots) != 1: + reject() + path = snapshots[0] + parts = PurePosixPath(path).parts + match = re.fullmatch(r'screen-hierarchy/step-([0-9]{3,6})(?:-[A-Za-z0-9_.-]{1,40})?\.json', path) + if (not match or len(parts) != 2 or any(part in ('.', '..') for part in parts) or + int(match[1]) != sequence + 1): + reject() + return sequence, path + + +def snapshot(tree): + pending, count, matches = [(tree, 0)], 0, [] + while pending: + node, depth = pending.pop() + count += 1 + if (count > NODE_LIMIT or depth > 128 or type(node) is not dict or + not set(node) <= {'attributes', 'children', 'checked', *FLAGS}): + reject() + attrs, children = node.get('attributes', {}), node.get('children', []) + if (type(attrs) is not dict or type(children) is not list or len(attrs) > 256 or + any(type(key) is not str or type(value) is not str for key, value in attrs.items()) or + any(node.get(flag) is not None and type(node[flag]) is not bool for flag in (*FLAGS, 'checked'))): + reject() + if attrs.get('resource-id') in ('btn_skip', 'io.hexawallet.keeper.development:id/btn_skip'): + matches.append(node) + pending.extend((child, depth + 1) for child in children) + return count, matches + + +def project_skip(nodes): + result = unknown()['skip'] + result['exact_resource_id_count'] = len(nodes) + if not nodes: + result['state'] = 'ABSENT' + return result + if len(nodes) != 1: + return result + node, attrs = nodes[0], nodes[0].get('attributes', {}) + number = r'(0|-?[1-9][0-9]{0,5})' + bounds = re.fullmatch(r'\[' + number + ',' + number + r'\]\[' + number + ',' + number + r'\]', attrs.get('bounds', '')) + if not bounds: + return result + left, top, right, bottom = map(int, bounds.groups()) + if (any(abs(value) > COORD_LIMIT for value in (left, top, right, bottom)) or + right <= left or bottom <= top): + return result + for flag in FLAGS: + if flag in attrs and attrs[flag] not in ('true', 'false'): + return result + result.update(state='UNIQUE_VALID_BOUNDS', bounds={'left': left, 'top': top, 'right': right, 'bottom': bottom}) + for flag in FLAGS: + direct = node.get(flag) + attribute = attrs.get(flag) + attribute = attribute == 'true' if attribute is not None else None + conflict = direct is not None and attribute is not None and direct != attribute + result['conflicts'][flag] = conflict + result[flag] = None if conflict else direct if direct is not None else attribute + return result + + +def collect(directory, flow, failure): + result = unknown('UNBOUND_FAILED_ASSERTION') + if (flow not in FLOWS or type(failure) is not dict or + failure.get('state') != 'ERROR_RECORDED' or + failure.get('failure_kind') != 'ASSERT_VISIBLE' or + failure.get('failed_selector') != 'start_new_tile'): + return result + try: + root = Path(directory).absolute() + commands = commands_file(root) + if commands.parent != root / FLOWS[flow]: + reject() + sequence, relative = failed_step(read_json(commands, root)) + tree = read_json(commands.parent / relative, root) + count, nodes = snapshot(tree) + result.update(binding_state='BOUND', reason='FAILED_ASSERTION_OUTCOME', node_count=count, + failed_assertion={'kind': 'ASSERT_VISIBLE', 'selector': 'start_new_tile', + 'sequence_number': sequence}, skip=project_skip(nodes)) + except Exception: + pass + return result diff --git a/.github/native/maestro_skip_tap.py b/.github/native/maestro_skip_tap.py new file mode 100644 index 0000000000..ab5a5be23b --- /dev/null +++ b/.github/native/maestro_skip_tap.py @@ -0,0 +1,243 @@ +"""Pinned Maestro Skip intent only; never app delivery or arbitrary coordinates. + +Producer: Maestro c8c361be1cc89b2e742f185466a302a22778be6b, +Maestro.kt:227/408/448, ScopedLogCapture.kt FILE_LOG_PATTERN, and +ArtifactsGenerator.kt:74. The exact compiled aliases below were observed by +executing cli-2.11.0's released JVM classes with a synthetic Driver. Kotlin's +ViewHierarchy value class mangles these JVM method names. The existing per-flow +log stays private. +""" +import importlib.util +import os +from pathlib import Path +import re +import stat + +SCOPE = 'HOST_DRIVER_TAP_INTENT_NOT_APP_DELIVERY' +BINDING = 'LOG_SEMANTIC_ORDER_NOT_EPOCH_CORRELATED' +LIMIT = 1024 * 1024 +SKIP_REGEX = r'^Skip[ \u00A0]*$' +NUMBER = r'(0|-?[1-9][0-9]{0,4})' +METHODS = { + 'tap': 'tap', + 'hierarchyBasedTap': 'hierarchyBasedTap', + 'screenshotBasedTap': 'screenshotBasedTap', + 'tap-HgV9Ck0': 'tap', + 'hierarchyBasedTap-ogj28Uc': 'hierarchyBasedTap', + 'screenshotBasedTap-ogj28Uc': 'screenshotBasedTap', +} +HEADER = re.compile(r'^(?:[01][0-9]|2[0-3]):[0-5][0-9]:[0-5][0-9]\.[0-9]{3} ' + r'\[ *INFO\] maestro\.Maestro\.(' + + '|'.join(re.escape(method) for method in METHODS) + r'): (.*)$') +START = 'Tapping on element: ' +ELEMENT = 'UiElement(treeNode=TreeNode(attributes={' +SELECTOR_FIELDS = frozenset(('textRegex', 'idRegex', 'size', 'below', 'above', 'leftOf', 'rightOf', + 'containsChild', 'containsDescendants', 'traits', 'index', 'enabled', 'optional', 'selected', + 'checked', 'focused', 'childOf', 'css')) +TAP_FIELDS = frozenset(('selector', 'retryIfNoChange', 'waitUntilVisible', 'longPress', 'repeat', + 'waitToSettleTimeoutMs', 'relativePoint', 'label', 'optional')) + + +def unknown(reason='UNBOUND_FAILED_ASSERTION'): + return {'schema': 1, 'state': 'UNKNOWN', 'reason': reason, 'scope': SCOPE, + 'binding': BINDING, 'command_sequence': None, 'target_identity': None, + 'selected_bounds': None, 'driver_point': None, 'point_relation': None, + 'driver_path': None, 'app_delivery': 'UNKNOWN'} + + +def reject(): + raise ValueError() + + +def source_helper(): + spec = importlib.util.spec_from_file_location('skip_snapshot_policy', + Path(__file__).with_name('maestro_failure_snapshot.py')) + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +def selector(value, field, expected, parent=False): + if type(value) is not dict or not set(value) <= SELECTOR_FIELDS or value.get(field) != expected: + return False + active = {key: item for key, item in value.items() if item is not None and + not (key == 'optional' and item is False)} + return active == ({field: expected, 'childOf': value['childOf']} if parent else {field: expected}) + + +def skip_command(command): + if type(command) is not dict or set(command) != {'tapOnElement'}: + return False + tap = command['tapOnElement'] + if type(tap) is not dict or not set(tap) <= TAP_FIELDS: + return False + for key, value in tap.items(): + if key != 'selector' and value is not None and not ( + key in ('retryIfNoChange', 'waitUntilVisible', 'longPress', 'optional') and value is False): + return False + target = tap.get('selector') + if type(target) is not dict or not selector(target, 'textRegex', SKIP_REGEX, parent=True): + return False + return selector(target.get('childOf'), 'idRegex', 'btn_skip') + + +def command_binding(rows, failed_sequence): + taps, chosen = [], [] + for row in rows: + command, metadata = row['command'], row['metadata'] + if (metadata['status'] not in ('SKIPPED', 'PENDING') and + any(key in command for key in ('tapOnPoint', 'tapOnPointV2Command'))): + reject() + if 'tapOnElement' not in command: + continue + status = metadata['status'] + if status in ('SKIPPED', 'PENDING'): + continue + if status != 'COMPLETED' or metadata['sequenceNumber'] >= failed_sequence: + reject() + tap = command['tapOnElement'] + # The selected-element logger runs once for these ordinary element taps. + # Relative/repeated/visibility-reentry forms cannot be ordinal-bound here. + if type(tap) is not dict or any(tap.get(key) is not None and not ( + key in ('longPress', 'waitUntilVisible', 'retryIfNoChange') and tap[key] is False) + for key in ('relativePoint', 'repeat', 'longPress', 'waitUntilVisible', 'retryIfNoChange')): + reject() + evaluated = metadata.get('evaluatedCommand') + if skip_command(command): + if not skip_command(evaluated): + reject() + chosen.append((len(taps), metadata['sequenceNumber'])) + taps.append(row) + if len(chosen) != 1: + reject() + return len(taps), chosen[0] + + +def read_log(path, root): + if path.is_symlink() or not path.resolve().is_relative_to(root.resolve()): + reject() + for item in (path, *path.parents): + if item.is_symlink(): + reject() + if item == root: + break + before = path.stat() + if not stat.S_ISREG(before.st_mode) or before.st_size > LIMIT: + reject() + identity = lambda value: (value.st_dev, value.st_ino, value.st_size, value.st_mtime_ns) + with os.fdopen(os.open(path, os.O_RDONLY | os.O_NOFOLLOW | os.O_NONBLOCK), 'rb') as stream: + opened = os.fstat(stream.fileno()) + if identity(opened) != identity(before): + reject() + payload = stream.read(LIMIT + 1) + if identity(os.fstat(stream.fileno())) != identity(opened): + reject() + if (len(payload) > LIMIT or identity(path.stat()) != identity(before) or path.is_symlink()): + reject() + text = payload.decode('utf-8') + if text and not text.endswith('\n'): + reject() + # The producer terminates records with %n (LF on the hosted Unix runner). + # Unicode separators inside a UI string must never become log records. + if any(value in text for value in ('\u2028', '\u2029', '\ufffd')): + reject() + lines = [line[:-1] if line.endswith('\r') else line for line in text.split('\n')[:-1]] + if len(lines) > 20000 or any(len(line) > 65536 for line in lines): + reject() + return lines + + +def selected(message): + if not message.startswith(START + ELEMENT): + reject() + body = message[len(START + ELEMENT):] + if '}, children=' not in body: + reject() + attributes = {} + for part in body.split('}, children=', 1)[0].split(', '): + key, separator, value = part.partition('=') + if not separator or not re.fullmatch(r'[A-Za-z][A-Za-z0-9_.:-]{0,63}', key) or key in attributes: + reject() + attributes[key] = value + label = any(re.fullmatch(r'Skip[ \u00a0]*', attributes.get(key, '')) for key in + ('text', 'hintText', 'accessibilityText')) + if not label: + reject() + if attributes.get('resource-id') not in (None, '', 'btn_skip', 'io.hexawallet.keeper.development:id/btn_skip'): + reject() + identity = ('SKIP_BUTTON_AND_LABEL' if attributes.get('resource-id') in + ('btn_skip', 'io.hexawallet.keeper.development:id/btn_skip') else 'SKIP_LABEL') + match = re.search(r', bounds=Bounds\(x=' + NUMBER + r', y=' + NUMBER + + r', width=([1-9][0-9]{0,4}), height=([1-9][0-9]{0,4})\)\) *$', message) + if not match: + reject() + left, top, width, height = map(int, match.groups()) + if any(abs(value) > 100000 for value in (left, top, left + width, top + height)): + reject() + return identity, {'left': left, 'top': top, 'right': left + width, 'bottom': top + height} + + +def project(lines, count, ordinal): + starts, point_records = [], [] + for index, line in enumerate(lines): + parsed = HEADER.fullmatch(line) + if not parsed: + continue + method, message = parsed.groups() + method = METHODS[method] + if any(ord(value) < 32 or 127 <= ord(value) <= 159 for value in line): + reject() + if method == 'tap' and message.startswith('Tapping on element:'): + starts.append((index, message)) + pattern = ('Tapping at ' if method == 'hierarchyBasedTap' else + 'Try tapping at ' if method == 'screenshotBasedTap' else None) + if pattern: + point = re.fullmatch(re.escape(pattern) + r'\(' + NUMBER + ', ' + NUMBER + + r'\) using hierarchy based logic for wait *', message) + if point: + point_records.append((index, method, tuple(map(int, point.groups())))) + if len(starts) != count or not 0 <= ordinal < len(starts): + reject() + index, message = starts[ordinal] + identity, bounds = selected(message) + end = starts[ordinal + 1][0] if ordinal + 1 < len(starts) else len(lines) + points = [item for item in point_records if index < item[0] < end] + if len(points) != 1: + reject() + _, method, (x, y) = points[0] + if any(abs(value) > 100000 for value in (x, y)): + reject() + relation = ('INSIDE_INITIAL_SELECTED_BOUNDS' if bounds['left'] <= x < bounds['right'] and + bounds['top'] <= y < bounds['bottom'] else 'OUTSIDE_INITIAL_SELECTED_BOUNDS') + return identity, bounds, {'x': x, 'y': y}, relation, ( + 'HIERARCHY_BASED' if method == 'hierarchyBasedTap' else 'SCREENSHOT_BASED') + + +def collect(directory, flow, failure): + result = unknown() + if (type(failure) is not dict or failure.get('state') != 'ERROR_RECORDED' or + failure.get('failure_kind') != 'ASSERT_VISIBLE' or failure.get('failed_selector') != 'start_new_tile'): + return result + stage = 'SOURCE_POLICY' + try: + policy = source_helper() + if flow not in policy.FLOWS: + return result + root = Path(directory).absolute() + stage = 'COMMAND_BINDING' + commands = policy.commands_file(root) + if commands.parent != root / policy.FLOWS[flow]: + reject() + rows = policy.read_json(commands, root) + failed_sequence, _ = policy.failed_step(rows) + count, (ordinal, sequence) = command_binding(rows, failed_sequence) + stage = 'PRIVATE_LOG' + lines = read_log(commands.parent / 'logs/maestro.log', root) + stage = 'TAP_INTENT_BINDING' + identity, bounds, point, relation, driver = project(lines, count, ordinal) + result.update(state='OBSERVED', reason='UNIQUE_BOUND_SKIP_DRIVER_INTENT', command_sequence=sequence, + target_identity=identity, selected_bounds=bounds, driver_point=point, + point_relation=relation, driver_path=driver) + except Exception: + return unknown(stage) + return result diff --git a/.github/native/nodeify.cjs b/.github/native/nodeify.cjs new file mode 100644 index 0000000000..d24127bf11 --- /dev/null +++ b/.github/native/nodeify.cjs @@ -0,0 +1,33 @@ +#!/usr/bin/env node +'use strict'; + +const path = require('node:path'); + +// Match setup.sh's existing polyfills without invoking its dependency installer. +// The unfiltered --hack command selects every core shim, including packages this +// app does not depend on, and makes valid imports such as constants unresolvable. +const SHIMS = Object.freeze([ + 'buffer', 'events', 'process', 'stream', 'inherits', 'path', 'assert', 'crypto', +]); + +function runNodeify(root = path.resolve(__dirname, '../..')) { + if (process.cwd() !== root) { + throw new Error('Run native Node compatibility preparation from the repository root.'); + } + const cli = require.resolve('rn-nodeify/cmd.js', { paths: [root] }); + const coreList = require.resolve(path.join(path.dirname(cli), 'coreList.js')); + require(coreList); + require.cache[coreList].exports = [...SHIMS]; + const originalArguments = process.argv; + try { + // No --install flag: the pinned upstream CLI can apply its existing module + // and file compatibility fixes, but cannot enter installShims(). + process.argv = [process.execPath, cli, '--hack']; + require(cli); + } finally { + process.argv = originalArguments; + } +} + +module.exports = { runNodeify, SHIMS }; +if (require.main === module) runNodeify(); diff --git a/.github/native/onboarding_failure_receipts.py b/.github/native/onboarding_failure_receipts.py new file mode 100644 index 0000000000..4242b40023 --- /dev/null +++ b/.github/native/onboarding_failure_receipts.py @@ -0,0 +1,152 @@ +"""Pure, bounded projection of reserved development-only Android receipts.""" +from datetime import date +import re + +LIMIT = 512 * 1024 +MAX_RECORDS = 10000 +MAX_MARKERS = 64 +PREFIX = 'KEEPER_ONBOARDING_BOUNDARY|' +STAGES = ('ONBOARDING_EFFECT', 'SKIP_PRESS_IN', 'SKIP_PRESS_OUT', + 'SKIP_PRESS', 'SKIP_RESET_RETURNED', + 'SKIP_RESET_THROWN', 'NEW_KEEPER_EFFECT') +_FRAME = re.compile( + r'(?:0[1-9]|1[0-2])-(?:0[1-9]|[12][0-9]|3[01]) ' + r'(?:[01][0-9]|2[0-3]):[0-5][0-9]:[0-5][0-9]\.[0-9]{3}' + r' +([1-9][0-9]{0,9}) +([1-9][0-9]{0,9}) +E +ReactNativeJS: (.*)') +_FORM = re.compile(re.escape(PREFIX) + r'(v[0-9]{1,4})\|([A-Z][A-Z0-9_]{0,63})') + + +def _frame(line): + if not isinstance(line, str) or len(line) > LIMIT or '\n' in line: + return None + match = _FRAME.fullmatch(line) + if not match or any(int(match[index]) > 2147483647 for index in (1, 2)): + return None + try: + date(2000, int(line[:2]), int(line[3:5])) # Year is absent; permit February 29. + except ValueError: + return None + return match + + +def has_diagnostic_marker_prefix(line): + """Reserve diagnostic frames for separate parsing, including rejected suffixes. + + This predicate establishes neither a valid receipt nor process attribution. + It must not exclude ordinary errors that merely mention the marker later. + """ + match = _frame(line) + return bool(match and match[3].startswith(PREFIX)) + + +def _clean(line): + # Only LF is a record delimiter; CR is accepted solely as part of CRLF by + # parse(). Reject replacement, C0/C1 controls and Unicode line separators. + return not any(ord(char) < 32 or 127 <= ord(char) <= 159 or + char in '\ufffd\u2028\u2029' for char in line) + + +def _receipt(line): + match = _frame(line) + if not match or not _clean(line): + return None + form = _FORM.fullmatch(match[3]) + if form and form[1] == 'v1' and form[2] in STAGES: + return int(match[1]), form[2] + return None + + +def is_diagnostic_marker_record(line): + """Recognize an exact valid enum record, independently of process ownership.""" + return _receipt(line) is not None + + +def parse(payload, owned_pids, *, usable=True, full_input=True): + """Return positive receipts only; missing stages never establish absence. + + Inputs are an already-collected threadtime E/F payload and the existing + current-process inventory. No guest-clock freshness, route, touch delivery, + causal relationship or application success is inferred from these records. + """ + result = { + 'schema': 1, 'scope': 'DEV_MARKERS_NOT_ROUTE_TAP_DELIVERY_OR_CAUSE_PROOF', + 'state': 'UNKNOWN', 'cause': 'UNDETERMINED', + 'freshness': 'GUEST_CLOCK_UNVERIFIED', + 'coverage': 'POSITIVE_ONLY_MISSING_STAGES_UNKNOWN', + 'producer_per_stage_limit': 4, 'producer_total_limit': 28, + 'consumer_marker_limit': MAX_MARKERS, 'input_byte_limit': LIMIT, + 'scan_record_limit': MAX_RECORDS, + 'input_usable': usable is True, 'full_input': full_input is True, + 'ownership_usable': False, 'stages': {stage: None for stage in STAGES}, + 'ordered_stages': [], + 'scanned_records': 0, 'marker_records': 0, 'owned_marker_records': 0, + 'foreign_pid_records': 0, 'duplicate_records': 0, + 'malformed_marker_records': 0, 'unsupported_marker_records': 0, + 'invalid_text_records': 0, 'partial_records': 0, + 'input_limit_exceeded': False, 'record_limit_reached': False, + 'marker_limit_reached': False, + } + if not isinstance(owned_pids, (set, frozenset)) or any( + type(pid) is not int or not 0 < pid <= 2147483647 for pid in owned_pids): + owned_pids = set() + result['ownership_usable'] = bool(owned_pids) + if usable is not True or not isinstance(payload, str): + return result + # Reject over-budget input rather than accepting a potentially split prefix. + if len(payload) > LIMIT: + result['input_limit_exceeded'] = True + return result + try: + if len(payload.encode('utf-8')) > LIMIT: + result['input_limit_exceeded'] = True + return result + except UnicodeEncodeError: + result['invalid_text_records'] = 1 + return result + lines = payload.split('\n') + if lines[-1]: + result['partial_records'] = 1 + lines.pop() # Never accept the terminal non-LF fragment as a complete record. + for line in lines: + if result['scanned_records'] >= MAX_RECORDS: + result['record_limit_reached'] = True + break + result['scanned_records'] += 1 + if line.endswith('\r'): + line = line[:-1] + if PREFIX not in line: + continue + if result['marker_records'] >= MAX_MARKERS: + result['marker_limit_reached'] = True + break + result['marker_records'] += 1 + if not _clean(line): + result['invalid_text_records'] += 1 + result['malformed_marker_records'] += 1 + continue + match = _frame(line) + if not match or not match[3].startswith(PREFIX): + result['malformed_marker_records'] += 1 + continue + form = _FORM.fullmatch(match[3]) + if not form: + result['malformed_marker_records'] += 1 + continue + if form[1] != 'v1' or form[2] not in STAGES: + result['unsupported_marker_records'] += 1 + continue + if int(match[1]) not in owned_pids: + result['foreign_pid_records'] += 1 + continue + result['owned_marker_records'] += 1 + stage = form[2] + result['ordered_stages'].append(stage) + if result['stages'][stage] is True: + result['duplicate_records'] += 1 + result['stages'][stage] = True + if result['owned_marker_records']: + partial = (full_input is not True or result['partial_records'] or + result['record_limit_reached'] or result['marker_limit_reached'] or + result['malformed_marker_records'] or result['unsupported_marker_records']) + result['state'] = 'PARTIAL' if partial else 'OBSERVED' + return result diff --git a/.github/native/parent_software_evidence.cjs b/.github/native/parent_software_evidence.cjs new file mode 100644 index 0000000000..e11a6df457 --- /dev/null +++ b/.github/native/parent_software_evidence.cjs @@ -0,0 +1,208 @@ +'use strict'; +// Explicit D profile. Pure software facts; never Actions origin or release authority. +const crypto = require('node:crypto'); +const { parseStrictJson } = require('./strict_json.cjs'); +const { validatePublicSoftware } = require('./public_software_receipts.cjs'); +const { privateIntegrationMembers, PRIVATE_INTEGRATION_CORE: core } = require('./private_integration_receipts.cjs'); +const PHASES = ['probe', 'features', 'product'], SUITES = ['protected', 'recovery', 'account']; +const PORTS = [8081, 8765, 8766, 8767, 8768, 8769]; +const SCOPES = { probe: 'NATIVE_STORAGE_PRIMITIVES_ONLY', features: 'EXTRACTED_PRODUCTION_WORKERS_NATIVE_STORAGE_ONLY', + product: 'FIVE_NORMAL_APP_RAW_PAIRS_RETRY_AND_SCOPED_COMPANIONS' }; +const PROBE_SCOPE = 'CI-only Metro entrypoint; actual encrypted Realm/current schemas and production MMKV; ' + + 'three process launches, rollback, wrong-key rejection, targeted primitive deletion, separate database files and cleanup. ' + + 'NOT app key deletion, account isolation, setup/recovery or release performance acceptance.'; +const PARENTS = Object.freeze({ + 'key-deletion-live-reference-and-privacy': 'unused key/PIN, visible/hidden Vault references and logical archive; scoped protected Delete/readback and local recovery preservation', + 'setup-offline-retry-relaunch-no-duplicates': 'controlled disconnect before commit, Retry and cold persistent counts; no WAN-offline claim', + 'app-account-session-isolation': 'native A/B/A account/Realm/history selection and current-protocol read namespaces plus stale-session workers; B backend mutation remains untested', + 'normal-page-runtime-and-recovery-regression': 'normal UI, controlled credential/history companions and fresh public-vector recovery; no authentic historical installation claim', + 'cold-start-and-first-wallet-performance': 'five original full-product startup/first-wallet pairs and five raw timing fields; development observations without a release performance threshold', +}); +const demand = condition => { if (!condition) throw new Error('PARENT_SOFTWARE_RECEIPT_INVALID'); }; +const object = value => value !== null && typeof value === 'object' && !Array.isArray(value); +const keys = (value, fields) => object(value) && Object.keys(value).sort().join('|') === [...fields].sort().join('|'); +const hex = value => typeof value === 'string' && /^[a-f0-9]{64}$/.test(value); +const number = value => typeof value === 'number' && Number.isFinite(value); +const digest = bytes => crypto.createHash('sha256').update(bytes).digest('hex'); +function same(a, b) { + if (a === b) return true; + if (Array.isArray(a)) return Array.isArray(b) && a.length === b.length && a.every((row, i) => same(row, b[i])); + return object(a) && object(b) && keys(a, Object.keys(b)) && Object.keys(a).every(key => same(a[key], b[key])); +} +function fixed(value, fields) { demand(object(value) && Object.entries(fields).every(([key, expected]) => same(value[key], expected))); } +function noFailure(value) { + demand(object(value) && !Object.keys(value).some(key => /^failure(?:_|$)/.test(key)) && + (!Object.hasOwn(value, 'cleanup') || value.cleanup === 'PASS') && + (!Object.hasOwn(value, 'cleanup_failure_categories') || same(value.cleanup_failure_categories, []))); +} +function parentSoftwareMembers(platform) { + return Object.freeze([...privateIntegrationMembers(platform), ...PHASES.map(phase => + `control/artifacts/native-integration/${platform.toLowerCase()}/public-${phase}.json`)]); +} +function publicContext(expected) { + return { platform: expected.platform, sourceSha: expected.app.sha, workflowSha: expected.execution.workflowSha, + workflowRef: `${expected.execution.repository}/${expected.execution.workflowPath}@${expected.execution.ref}`, + runId: expected.execution.runId, runAttempt: expected.execution.runAttempt, workerSources: expected.workerSources }; +} +function publicMetadata(value, expected, built, proof) { + core.foundation(value, expected, built, proof); + demand(!Object.hasOwn(value, 'startup_cleanup_errors') || same(value.startup_cleanup_errors, [])); + demand(!Object.hasOwn(value, 'startup_observation_cleanup')); + if (value.r5_feature_evidence) noFailure(value.r5_feature_evidence); + if (value.product_ui) { + const ui = value.product_ui; + [ui, ...(Array.isArray(ui.samples) ? ui.samples : []), ui.controlled_disconnect_retry, ui.key_management_ui] + .filter(row => row !== undefined).forEach(noFailure); + } +} +function device(value, expected, phase, androidOwner) { + if (expected.platform === 'IOS') { + demand(keys(value, ['kind', 'inventory_before_sha256', 'inventory_after_sha256', 'new_devices_remaining'])); + fixed(value, { kind: 'SIMULATOR', new_devices_remaining: 0 }); + demand(hex(value.inventory_before_sha256) && value.inventory_after_sha256 === value.inventory_before_sha256); + } else { + demand(keys(value, ['kind', 'device_sha256', 'app_absent', 'reverse_ports_absent', 'removed_owned_reverse_ports'])); + fixed(value, { kind: 'EMULATOR', app_absent: true, reverse_ports_absent: PORTS, device_sha256: androidOwner }); + const removed = value.removed_owned_reverse_ports, allowed = phase === 'product' ? [8081, 8765, 8766, 8767, 8769] : [8081]; + demand(hex(value.device_sha256) && Array.isArray(removed) && same(removed, [...new Set(removed)].sort((a, b) => a - b)) && + removed.every(port => allowed.includes(port))); + } +} +function budget(value, wrappers) { + demand(keys(value, ['window_seconds', 'window_start_monotonic', 'remaining_seconds', 'events'])); + demand(value.window_seconds === 14400 && number(value.window_start_monotonic) && value.window_start_monotonic >= 0 && + number(value.remaining_seconds) && value.remaining_seconds >= 0 && value.remaining_seconds <= 14400 && Array.isArray(value.events) && value.events.length <= 2048); + let previous = 14400; + value.events.forEach((row, i) => { + demand(keys(row, ['index', 'kind', 'declared_timeout_seconds', 'remaining_seconds', 'elapsed_seconds', 'cleanup_reserve_seconds', 'admitted'])); + demand(row.index === i + 1 && number(row.declared_timeout_seconds) && row.declared_timeout_seconds > 0 && row.declared_timeout_seconds <= 7200 && + number(row.remaining_seconds) && row.remaining_seconds <= previous && number(row.elapsed_seconds) && row.elapsed_seconds >= 0 && + Math.abs(row.remaining_seconds + row.elapsed_seconds - 14400) <= 0.000001 && row.cleanup_reserve_seconds === 1200 && + row.admitted === true && row.remaining_seconds >= row.declared_timeout_seconds + 1200); + previous = row.remaining_seconds; + }); + let cursor = 0, previousEnd = 14400; + for (const phase of PHASES) { + const entry = value.events[cursor++], row = wrappers[phase].budget; + demand(entry?.kind === 'public_phase_entry' && entry.declared_timeout_seconds === 600 && entry.remaining_seconds <= previousEnd && + keys(row, ['window_seconds', 'window_start_monotonic', 'remaining_start_seconds', 'remaining_seconds', 'commands'])); + fixed(row, { window_seconds: 14400, window_start_monotonic: value.window_start_monotonic }); + demand(number(row.remaining_start_seconds) && row.remaining_start_seconds >= 0 && row.remaining_start_seconds <= entry.remaining_seconds && + number(row.remaining_seconds) && row.remaining_seconds >= 0 && row.remaining_seconds <= row.remaining_start_seconds && + Array.isArray(row.commands) && row.commands.length > 0 && row.commands.every(command => command.kind === 'normal_command') && + same(row.commands, value.events.slice(cursor, cursor + row.commands.length)) && + row.commands[0].remaining_seconds <= row.remaining_start_seconds && row.remaining_seconds <= row.commands.at(-1).remaining_seconds); + cursor += row.commands.length; previousEnd = row.remaining_seconds; + } + for (const suite of SUITES) { + const entry = value.events[cursor++]; + demand(entry?.kind === 'private_suite_entry' && entry.declared_timeout_seconds === (suite === 'account' ? 3600 : 3000) && + entry.remaining_seconds <= previousEnd); + previousEnd = entry.remaining_seconds; + } + demand(cursor === value.events.length && value.remaining_seconds <= previousEnd); +} +function validateParentSoftwareEvidence(rawFiles, suppliedExpected) { + try { + const expected = core.context(suppliedExpected), members = parentSoftwareMembers(expected.platform), values = {}, hashes = {}; + demand(Array.isArray(expected.workerSources) && expected.recoverySources.every(row => { + const overlap = expected.workerSources.find(worker => worker.path === row.path); + return !overlap || overlap.sha256 === row.sha256; + })); + demand(keys(rawFiles, members)); + for (const name of members) { + const bytes = rawFiles[name], cap = name.endsWith('/fixture.json') || name.endsWith('/mongo.json') ? 8192 : + /control\/artifacts\/native-(?:protected|recovery|account)\//.test(name) ? 32768 : 2_000_000; + demand(Buffer.isBuffer(bytes)); values[name] = parseStrictJson(bytes, cap); demand(object(values[name])); hashes[name] = digest(bytes); + } + const [aggregateName, startupName] = members, aggregate = values[aggregateName], startup = values[startupName]; + const prefix = `control/artifacts/native-integration/${expected.platform.toLowerCase()}`; + noFailure(aggregate); + demand(keys(aggregate, ['schema', 'status', 'app_sha', 'app_tree', 'control_sha', 'control_tree', 'backend_base', 'platform', + 'run_id', 'run_attempt', 'built_artifact', 'scope', 'source_admission', 'barriers', 'suites', 'physical_devices', + 'historical_upgrade', 'b_backend_mutation_isolation', 'biometrics', 'five_startup_measurements', 'trusted_acceptance', + 'release_readiness', 'startup_receipt_sha256', 'public_phases', 'window_budget', + ...(expected.platform === 'ANDROID' ? ['startup_resource_barrier'] : [])])); + fixed(aggregate, { schema: 1, status: 'PASS', app_sha: expected.app.sha, app_tree: expected.app.tree, + control_sha: expected.control.sha, control_tree: expected.control.tree, backend_base: expected.backend.sha, + platform: expected.platform, run_id: String(expected.execution.runId), run_attempt: String(expected.execution.runAttempt), + scope: 'PUBLIC_PHASES_AND_THREE_SERIAL_NORMAL_UI_FEATURE_SUITES_ONE_BUILD', physical_devices: 'NOT_TESTED', historical_upgrade: 'NOT_TESTED', + b_backend_mutation_isolation: 'NOT_TESTED', biometrics: 'NOT_TESTED', five_startup_measurements: 'REQUIRES_SEPARATE_RAW_MEASUREMENT_EVIDENCE', + trusted_acceptance: 'NOT_ESTABLISHED', release_readiness: 'NOT_ESTABLISHED', startup_receipt_sha256: hashes[startupName] }); + const built = aggregate.built_artifact; core.builtIdentity(built, expected.platform); + const proof = core.foundation(startup, expected, built); + core.sourceProofs(aggregate.source_admission, expected); + demand(aggregate.source_admission.protected.rawSources.control.files >= 101); + const final = values[members[9]], wrappers = {}; + let androidOwner; + if (expected.platform === 'ANDROID') { + demand(typeof startup.device?.id === 'string' && /^emulator-[0-9]+$/.test(startup.device.id)); + androidOwner = digest(Buffer.from(startup.device.id)); + device(aggregate.startup_resource_barrier, expected, 'startup', androidOwner); + } + demand(keys(aggregate.public_phases, PHASES)); + for (const phase of PHASES) { + const path = `${prefix}/public-${phase}.json`, wrapper = values[path], entry = aggregate.public_phases[phase]; + demand(keys(wrapper, ['schema', 'phase', 'scope', 'status', 'invocation', 'execution_started', 'failure_category', + 'app_sha', 'app_tree', 'backend_base', 'control_sha', 'control_tree', 'platform', 'run_id', 'run_attempt', + 'metadata', 'generated_source', 'owned_device_cleanup', 'budget'])); + fixed(wrapper, { schema: 1, phase, scope: SCOPES[phase], status: 'PASS', invocation: 'RETURNED', execution_started: true, + failure_category: null, app_sha: expected.app.sha, app_tree: expected.app.tree, backend_base: expected.backend.sha, + control_sha: expected.control.sha, control_tree: expected.control.tree, platform: expected.platform, + run_id: String(expected.execution.runId), run_attempt: String(expected.execution.runAttempt) }); + demand(keys(entry, ['status', 'scope', 'execution_started', 'invocation', 'receipt_path', 'receipt_sha256', 'post_phase_barrier'])); + fixed(entry, { status: 'PASS', scope: SCOPES[phase], execution_started: true, invocation: 'RETURNED', + receipt_path: path.slice('control/'.length), receipt_sha256: hashes[path], post_phase_barrier: 'PASS' }); + publicMetadata(wrapper.metadata, expected, built, proof); + device(wrapper.owned_device_cleanup, expected, phase, androidOwner); + const generated = wrapper.generated_source; + demand(keys(generated, ['path', 'created', 'absent_after', ...(phase === 'features' ? ['sha256', 'source_functions'] : [])])); + fixed(generated, { path: '.github/native/.generated/r5_workers.js', created: phase === 'features', absent_after: true }); + if (phase === 'features') demand(hex(generated.sha256) && same(generated.source_functions, expected.workerSources)); + wrappers[phase] = wrapper; + } + const probe = wrappers.probe.metadata, features = wrappers.features.metadata, product = wrappers.product.metadata; + fixed(probe, { native_probe: 'PASS', probe_scope: PROBE_SCOPE }); + demand(Number.isSafeInteger(probe.native_probe_test_cases) && probe.native_probe_test_cases > 0); + fixed(features, { native_probe: 'PASS', probe_scope: PROBE_SCOPE, native_probe_test_cases: probe.native_probe_test_cases, r5_features: 'PASS' }); + demand(Number.isSafeInteger(features.r5_features_test_cases) && features.r5_features_test_cases > 0); + validatePublicSoftware(product, publicContext(expected)); publicMetadata(final, expected, built, proof); + validatePublicSoftware(final, publicContext(expected)); + for (const row of [product, final]) { + fixed(row, { probe_scope: PROBE_SCOPE, native_probe_test_cases: probe.native_probe_test_cases, + r5_features_test_cases: features.r5_features_test_cases, r5_feature_evidence: features.r5_feature_evidence }); + } + for (const key of ['product_ui', 'product_credential_ui', 'product_version_history_ui', 'product_vault_ui']) demand(same(final[key], product[key])); + budget(aggregate.window_budget, wrappers); + const order = ['initial', ...PHASES.flatMap(phase => ['before_public_' + phase, 'after_public_' + phase]), + ...SUITES.flatMap(suite => ['before_' + suite, 'after_' + suite])], retained = {}; + demand(Array.isArray(aggregate.barriers) && aggregate.barriers.length === order.length); + aggregate.barriers.forEach((row, i) => { + if (i === 1) retained[startupName.slice('control/'.length)] = hashes[startupName]; + if (i >= 2 && i <= 6 && i % 2 === 0) { + const path = `${prefix}/public-${PHASES[i / 2 - 1]}.json`; retained[path.slice('control/'.length)] = hashes[path]; + } + if (i >= 8 && i % 2 === 0) for (const file of ['metadata.json', 'fixture.json']) { + const path = `control/artifacts/native-${SUITES[(i - 8) / 2]}/${expected.platform.toLowerCase()}/${file}`; + retained[path.slice('control/'.length)] = hashes[path]; + } + demand(keys(row, ['phase', 'source_proofs', 'built_artifact', 'resources', 'retained_receipt_sha256'])); + fixed(row, { phase: order[i], source_proofs: aggregate.source_admission, built_artifact: built, retained_receipt_sha256: retained, + resources: { overlays_absent: SUITES.map(suite => `.github/native/.private-${suite}`), + listeners: PORTS.map(port => ({ port, result: 'ECONNREFUSED' })) } }); + }); + core.privateSuites(values, hashes, expected, built, proof, aggregate.suites); + core.mongo(values['control/artifacts/native-protected/mongo.json'], expected); + return Object.freeze({ schema: 1, platform: expected.platform, software_complete: true, + cases: Object.freeze(Object.entries(PARENTS).map(([id, scope]) => Object.freeze({ id, status: 'PASS', scope }))), + identity: Object.freeze({ app: Object.freeze({ ...expected.app }), control: Object.freeze({ ...expected.control }), + backend: Object.freeze({ ...expected.backend }), execution: Object.freeze({ ...expected.execution }) }), + raw_sha256: Object.freeze(hashes), installed_artifact: Object.freeze({ ...proof }), + source_observations: Object.freeze({ app: Object.freeze({ ...aggregate.source_admission.protected.rawSources.app }), + control: Object.freeze({ ...aggregate.source_admission.protected.rawSources.control }) }), + generated_worker_sha256: wrappers.features.generated_source.sha256, + installed_trust: 'UNVERIFIED', release_ready: false, authorization_eligible: false, + execution_allowed: false, release_authorized: false }); + } catch { throw new Error('PARENT_SOFTWARE_RECEIPT_INVALID'); } +} +module.exports = { validateParentSoftwareEvidence, parentSoftwareMembers }; diff --git a/.github/native/pod_diagnostics.py b/.github/native/pod_diagnostics.py new file mode 100644 index 0000000000..99da8d2368 --- /dev/null +++ b/.github/native/pod_diagnostics.py @@ -0,0 +1,124 @@ +"""Finite CocoaPods failure facts; no message, URL, path or arbitrary value leaves.""" +import hashlib +import os +from pathlib import Path +import re +import stat + +LIMIT = 1024 * 1024 +LINE_LIMIT = 5000 +SCOPE = 'LOCKED_DEPENDENCY_INSTALL_ONLY_NOT_NATIVE_ACCEPTANCE_OR_CAUSE' +PATTERNS = ( + (r'\berror installing\b', 'POD_INSTALL_FAILURE'), + (r'could not find compatible versions|deployment mode|lockfile needs to be updated|sandbox is not in sync', + 'POD_LOCK_CONFLICT'), + (r'Gemfile.*(?:changed|change)|lockfile.*(?:frozen|deployment)|frozen mode|could not find gem', + 'RUBY_LOCK_OR_GEM_UNAVAILABLE'), + (r'\bBundler::(?:GemNotFound|VersionConflict|GemfileNotFound|GemfileLockNotFound|RubyVersionMismatch)\b', + 'RUBY_BUNDLE_RESOLUTION_FAILURE'), + (r'\bROOT_BUNDLE_CONTEXT_MISMATCH\b', 'RUBY_BUNDLE_CONTEXT_MISMATCH'), + (r'\bROOT_BUNDLE_VERSION_MISMATCH\b', 'RUBY_BUNDLE_VERSION_MISMATCH'), + (r'unable to find a specification|none of your spec sources contain', 'POD_SPEC_UNAVAILABLE'), + (r'checksum.*(?:mismatch|not match|invalid)|verification checksum was incorrect', 'POD_CHECKSUM_MISMATCH'), + (r'CDN:.*(?:couldn.t be downloaded|failed)|failed to download|error downloading', 'DEPENDENCY_DOWNLOAD_FAILURE'), + (r'could not resolve (?:host|proxy)|getaddrinfo|name or service not known', 'DEPENDENCY_DNS_FAILURE'), + (r'SSL certificate|certificate verify failed|SSL_connect|TLS handshake', 'DEPENDENCY_TLS_FAILURE'), + (r'failed to connect|connection (?:refused|reset)|network is unreachable', 'DEPENDENCY_CONNECTION_FAILURE'), + (r'operation timed out|connection timed out|execution expired', 'DEPENDENCY_COMMAND_TIMEOUT'), + (r'no space left on device', 'DISK_SPACE_EXHAUSTED'), + (r'requires a higher minimum deployment target', 'IOS_DEPLOYMENT_TARGET_UNSUPPORTED'), + (r'command not found|No such file or directory', 'DEPENDENCY_TOOL_OR_FILE_UNAVAILABLE'), + (r'\b(?:LoadError|NoMethodError|NameError|ArgumentError|SyntaxError)\b', 'RUBY_TOOL_FAILURE'), + (r'unable to find compatibility version string|unknown ISA', 'XCODE_PROJECT_FORMAT_UNSUPPORTED'), + (r'^fatal:.*(?:unable to access|could not read|repository|fetch|clone)|RPC failed', 'GIT_DOWNLOAD_FAILURE'), +) + + +def lock_state(root): + """Only fixed committed input names and exact byte hashes are returned.""" + values = {} + for name in ('Gemfile', 'Gemfile.lock', 'yarn.lock', 'ios/Podfile.lock'): + try: + path = root / name + if path.is_symlink() or not path.is_file(): + raise ValueError('Unsupported lock input.') + values[name] = hashlib.sha256(path.read_bytes()).hexdigest() + except (OSError, ValueError): + values[name] = None + return values + + +def locked_pods(root): + """Permit identifiers only from the committed lock's PODS section.""" + result = set() + path = root / 'ios/Podfile.lock' + if path.is_symlink(): + return result + try: + text = path.read_text() + except (OSError, UnicodeError): + return result + if not text.startswith('PODS:\n'): + return result + for line in text.splitlines()[1:]: + if line and not line[0].isspace(): + break + match = re.fullmatch(r' - ([A-Za-z0-9_+.-]+(?:/[A-Za-z0-9_+.-]+)*)(?: \([^()\r\n]+\))?:?', line) + if match: + result.add(match[1]) + return result + + +def observe(path, pods, before, after): + facts = {'schema': 1, 'scope': SCOPE, 'state': 'UNKNOWN', 'categories': [], + 'reported_failed_locked_pods': [], 'curl_exit_codes': [], 'http_statuses': [], + 'bytes_read': 0, 'input_truncated': False, 'line_limit_reached': False, + 'unclassified_error_lines': 0, 'locks_before': before, 'locks_after': after, + 'locks_unchanged': before == after and all(before.values())} + try: + with os.fdopen(os.open(path, os.O_RDONLY | os.O_NOFOLLOW | os.O_NONBLOCK), 'rb') as source: + if not stat.S_ISREG(os.fstat(source.fileno()).st_mode): + raise ValueError('Expected a private regular log.') + raw = source.read(LIMIT + 1) + facts['input_truncated'] = len(raw) > LIMIT + raw = raw[:LIMIT] + facts['bytes_read'] = len(raw) + lines = raw.decode('utf-8', errors='replace').splitlines() + facts['line_limit_reached'] = len(lines) > LINE_LIMIT + categories, failed, curl_codes, statuses = set(), set(), set(), set() + for raw_line in lines[:LINE_LIMIT]: + line = re.sub(r'\x1b\[[0-9;]*[A-Za-z]', '', raw_line).strip() + found = False + for pattern, category in PATTERNS: + if re.search(pattern, line, re.IGNORECASE): + categories.add(category) + found = True + match = re.match(r'^\[!\] Error installing ([A-Za-z0-9_+.-]+(?:/[A-Za-z0-9_+.-]+)*)(?:\s|$)', line) + if match and match[1] in pods: + failed.add(match[1]) + match = re.match(r'^curl: \(([1-9][0-9]?)\)(?:\s|$)', line) + if match: + code = int(match[1]) + curl_codes.add(code) + found = True + categories.add({5: 'DEPENDENCY_DNS_FAILURE', 6: 'DEPENDENCY_DNS_FAILURE', + 7: 'DEPENDENCY_CONNECTION_FAILURE', 28: 'DEPENDENCY_COMMAND_TIMEOUT', + 35: 'DEPENDENCY_TLS_FAILURE', 60: 'DEPENDENCY_TLS_FAILURE'}.get( + code, 'CURL_DOWNLOAD_FAILURE')) + match = re.search(r'\b(?:HTTP/[12](?:\.[01])?\s+|requested URL returned error:\s*)([1-5][0-9]{2})(?:\s|$)', line) + if match: + status = int(match[1]) + statuses.add(status) + if 400 <= status < 500: + categories.add('DEPENDENCY_HTTP_CLIENT_FAILURE') + elif status >= 500: + categories.add('DEPENDENCY_HTTP_SERVER_FAILURE') + found = True + if re.match(r'^(?:\[!\]|curl:|fatal:|error:)', line, re.IGNORECASE) and not found: + facts['unclassified_error_lines'] += 1 + facts.update(state='OBSERVED', categories=sorted(categories), + reported_failed_locked_pods=sorted(failed)[:16], + curl_exit_codes=sorted(curl_codes), http_statuses=sorted(statuses)) + except (OSError, ValueError): + pass + return facts diff --git a/.github/native/private_actions_origin.cjs b/.github/native/private_actions_origin.cjs new file mode 100644 index 0000000000..4921b6ef8b --- /dev/null +++ b/.github/native/private_actions_origin.cjs @@ -0,0 +1,316 @@ +'use strict'; +// Read-only private diagnostic origin. No installed/canonical/release authority. +const crypto = require('node:crypto'); +const { parseStrictJson } = require('./strict_json.cjs'); +const { RECOVERY_SOURCES } = require('./private_integration_receipts.cjs'); +const { WORKER_SOURCES } = require('./public_software_receipts.cjs'); +const PRIVATE = 'KeeperCommunity/bitcoin-keeper-relay', PRIVATE_ID = 496119449; +const APP = 'KeeperCommunity/bitcoin-keeper', APP_ID = 470904030; +const BACKEND = '29597b27eeeb299205bf7d0555db0a90c38cc4be'; +const REVIEWED_DRIVER = 'f0a695a6bba068ba93962e90ab52e8901ef4a852'; +const WORKFLOW = '.github/workflows/r5-native-integration.yml'; +const REF = 'refs/heads/codex/r5-cloud-native-execution-20261025'; +const POLICY = 'release/r5-native-integration/inputs.json'; +const MARKER = 'release/r5-native-integration/native-request.json'; +const MAX_AGE = 86400000, MAX_ARCHIVE = 12 * 1024 * 1024; +const SHA = /^[a-f0-9]{40}$/, issued = new WeakSet(); +const sha = value => typeof value === 'string' && SHA.test(value); +const instant = value => typeof value === 'string' ? Date.parse(value) : NaN; +// Final app/controller windows require separate source review and installation. +// No input, artifact, environment variable or API response may populate this. +const REVIEWED_WINDOWS = Object.freeze({}); +const id = n => Number.isSafeInteger(n) && n > 0; +const object = v => v !== null && typeof v === 'object' && !Array.isArray(v); +const keys = (v, names) => object(v) && Object.keys(v).sort().join('|') === [...names].sort().join('|'); +const demand = (yes, code = 'PRIVATE_ACTIONS_ORIGIN_INVALID') => { if (!yes) throw new Error(code); }; +const digest = bytes => crypto.createHash('sha256').update(bytes).digest('hex'); +function equal(a, b) { + if (a === b) return true; + if (Array.isArray(a)) return Array.isArray(b) && a.length === b.length && a.every((v, i) => equal(v, b[i])); + return object(a) && object(b) && keys(a, Object.keys(b)) && Object.keys(a).every(k => equal(a[k], b[k])); +} +function frozen(value) { + if (value && typeof value === 'object') { for (const child of Object.values(value)) frozen(child); Object.freeze(value); } + return value; +} +const safePath = p => typeof p === 'string' && p.length <= 4096 && /^[A-Za-z0-9_./@+ -]+$/.test(p) && + !p.startsWith('/') && p.split('/').every(s => s && s !== '.' && s !== '..'); +const api = repository => ({ owner: 'KeeperCommunity', repo: repository.split('/')[1], request: { timeout: 30000 } }); +function data(r) { demand(r?.status === 200 && object(r.data) && !/rel="next"/.test(r.headers?.link || ''), 'PRIVATE_ACTIONS_API_INCOMPLETE'); return r.data; } +async function complete(endpoint, args, field) { + const rows = [], ids = new Set(); + let total; + for (let page = 1; page <= 10; page++) { + // Construct each page against the same fixed SDK endpoint/arguments. Never + // follow an API-supplied URL or silently drop older/failed attempts. + const response = await endpoint({ ...args, per_page: 100, page }), value = response?.data; + demand(response?.status === 200 && object(value) && id(value.total_count) && value.total_count <= 1000 && + (total === undefined || value.total_count === total) && Array.isArray(value[field]) && + value[field].length > 0 && value[field].length <= 100, 'PRIVATE_ACTIONS_API_INCOMPLETE'); + total = value.total_count; + for (const row of value[field]) { + demand(id(row?.id) && !ids.has(row.id), 'PRIVATE_ACTIONS_API_INCOMPLETE'); + ids.add(row.id); rows.push(row); + } + const more = /rel="next"/.test(response.headers?.link || ''); + demand(rows.length <= total && more === (rows.length < total), 'PRIVATE_ACTIONS_API_INCOMPLETE'); + if (!more) return rows; + demand(value[field].length === 100, 'PRIVATE_ACTIONS_API_INCOMPLETE'); + } + throw new Error('PRIVATE_ACTIONS_API_INCOMPLETE'); +} +function repository(value, name, number) { return value?.id === number && value.full_name === name; } +function descriptor(value, platform) { + demand(keys(value, ['platform', 'app', 'control', 'backend', 'execution']) && value.platform === platform); + for (const [name, repo] of [['app', APP], ['control', PRIVATE]]) demand(keys(value[name], ['repository', 'sha', 'tree']) && + value[name].repository === repo && sha(value[name].sha) && sha(value[name].tree)); + demand(keys(value.backend, ['repository', 'sha']) && value.backend.repository === PRIVATE && value.backend.sha === BACKEND); + const e = value.execution; + demand(keys(e, ['repository', 'workflowPath', 'workflowSha', 'ref', 'runId', 'runAttempt']) && + e.repository === PRIVATE && e.workflowPath === WORKFLOW && e.workflowSha === value.control.sha && + e.ref === REF && id(e.runId) && id(e.runAttempt)); + return value; +} +function tree(value, expected) { + demand(value.sha === expected && value.truncated === false && Array.isArray(value.tree) && + value.tree.length > 0 && value.tree.length <= 20000, 'PRIVATE_ACTIONS_TREE_INVALID'); + const entries = new Map(); + for (const row of value.tree) { + demand(safePath(row.path) && sha(row.sha) && !entries.has(row.path) && + ((row.type === 'tree' && row.mode === '040000') || + (row.type === 'blob' && ['100644', '100755'].includes(row.mode))), 'PRIVATE_ACTIONS_TREE_INVALID'); + entries.set(row.path, row); + } + return new Map([...entries].filter(([, row]) => row.type === 'blob').sort(([a], [b]) => a < b ? -1 : a > b ? 1 : 0)); +} +async function contents(github, repo, ref, name, inventory) { + const value = data(await github.rest.repos.getContent({ ...api(repo), ref, path: name })); + demand(value.type === 'file' && value.path === name && value.encoding === 'base64' && + id(value.size) && value.size <= 2_000_000 && sha(value.sha || '') && typeof value.content === 'string' && + /^[A-Za-z0-9+/=\r\n]+$/.test(value.content), 'PRIVATE_ACTIONS_SOURCE_INVALID'); + const bytes = Buffer.from(value.content, 'base64'); + demand(bytes.length === value.size && bytes.toString('base64') === value.content.replace(/[\r\n]/g, '') && + crypto.createHash('sha1').update(`blob ${bytes.length}\0`).update(bytes).digest('hex') === value.sha && + (!inventory || inventory.get(name)?.sha === value.sha), 'PRIVATE_ACTIONS_SOURCE_INVALID'); + return bytes; +} +function fresh(run, now) { + const updated = instant(run.updated_at); + demand(Number.isFinite(now) && Number.isFinite(updated) && updated <= now && updated >= now - MAX_AGE, + 'PRIVATE_ACTIONS_RUN_STALE'); +} +function runIdentity(run, value, workflow, now) { + const e = value.execution; + demand(run.id === e.runId && run.run_attempt === e.runAttempt && run.workflow_id === workflow.id && + run.path === WORKFLOW && run.event === 'push' && run.head_sha === value.control.sha && run.head_branch === REF.slice(11) && + repository(run.repository, PRIVATE, PRIVATE_ID) && repository(run.head_repository, PRIVATE, PRIVATE_ID) && + run.status === 'completed' && run.conclusion === 'success' && id(run.check_suite_id) && + (run.referenced_workflows === undefined || (Array.isArray(run.referenced_workflows) && run.referenced_workflows.length === 0)), + 'PRIVATE_ACTIONS_EXACT_RUN_REQUIRED'); + fresh(run, now); +} +function artifactIdentity(info, value, artifactId, now, selected) { + const e = value.execution, name = `r5-integration-native-${value.platform}-${value.control.sha}-${e.runId}-${e.runAttempt}`; + demand(info.id === artifactId && info.name === name && info.expired === false && id(info.size_in_bytes) && info.size_in_bytes <= MAX_ARCHIVE && + typeof info.digest === 'string' && /^sha256:[a-f0-9]{64}$/.test(info.digest) && instant(info.expires_at) > now && + info.workflow_run?.id === e.runId && info.workflow_run.head_sha === value.control.sha && info.workflow_run.head_branch === REF.slice(11) && + info.workflow_run.repository_id === PRIVATE_ID && info.workflow_run.head_repository_id === PRIVATE_ID && + instant(info.created_at) >= instant(selected.started_at) && instant(info.created_at) <= instant(selected.completed_at), + 'PRIVATE_ACTIONS_ARTIFACT_INVALID'); +} +function stepNames(platform) { + return ['Record actual shared native window start before checkout or build', + 'Check out exact private tooling and only required runtime modules', 'Fetch fixed public app anonymously', + 'Set up reviewed Node runtime', 'Verify bounded exact source before installation', + 'Install frozen backend dependencies without scripts', 'Set up Java17', + 'Record actual hosted toolchain', 'Install immutable native app dependencies and synthetic configuration', + 'Verify exact sources after dependency preparation', + ...(platform === 'IOS' ? ['Set up Ruby for hosted iOS', 'Select reviewed Xcode16.4', + 'Check hosted iOS driver before the app build'] : ['Set up Android tools', 'Enable owned Android emulator acceleration']), + 'Start only this run\'s pinned owned loopback Mongo replica', 'Characterize the actual fixture and source contracts', + 'Build one unsigned development native candidate', platform === 'IOS' ? + 'Run one iOS startup then public phases and three serial private suites' : + 'Run one Android startup then public phases and three serial private suites', + 'Verify exact source after native attempts', 'Remove only this run\'s owned Mongo runtime', 'Retain metadata only']; +} +async function selectedJob(github, value, run, now) { + const { platform, control, execution: e } = value, args = api(PRIVATE); + const suite = data(await github.rest.checks.getSuite({ ...args, check_suite_id: run.check_suite_id })); + demand(suite.id === run.check_suite_id && suite.head_sha === control.sha && suite.status === 'completed' && + suite.conclusion === 'success' && suite.app?.id === 15368 && suite.app.slug === 'github-actions', 'PRIVATE_ACTIONS_SUITE_INVALID'); + const jobs = await complete(github.rest.actions.listJobsForWorkflowRunAttempt, { ...args, run_id: e.runId, + attempt_number: e.runAttempt }, 'jobs'); + demand(jobs.length === 1, 'PRIVATE_ACTIONS_JOB_INVALID'); + const job = jobs[0], match = job.check_run_url?.match(/^https:\/\/api\.github\.com\/repos\/KeeperCommunity\/bitcoin-keeper-relay\/check-runs\/([1-9]\d*)$/); + demand(job.name === 'native' && job.run_id === e.runId && job.run_attempt === e.runAttempt && job.head_sha === control.sha && + job.status === 'completed' && job.conclusion === 'success' && match && id(Number(match[1])) && + Array.isArray(job.labels) && job.labels.includes(platform === 'IOS' ? 'macos-15-intel' : 'ubuntu-24.04') && + !job.labels.includes('self-hosted') && instant(job.started_at) > 0 && + instant(job.completed_at) > instant(job.started_at) && instant(job.completed_at) <= now, 'PRIVATE_ACTIONS_JOB_INVALID'); + for (const name of stepNames(platform)) { + const found = job.steps?.filter(s => s.name === name) || []; + demand(found.length === 1 && found[0].status === 'completed' && found[0].conclusion === 'success', 'PRIVATE_ACTIONS_STEP_INVALID'); + } + const other = platform === 'IOS' ? 'Run one Android startup then public phases and three serial private suites' : + 'Run one iOS startup then public phases and three serial private suites'; + const skipped = job.steps?.filter(s => s.name === other) || []; + demand(skipped.length === 1 && skipped[0].status === 'completed' && skipped[0].conclusion === 'skipped', 'PRIVATE_ACTIONS_SELECTED_PLATFORM_INVALID'); + const check = data(await github.rest.checks.get({ ...args, check_run_id: Number(match[1]) })); + demand(check.id === Number(match[1]) && check.name === job.name && check.check_suite?.id === run.check_suite_id && + check.head_sha === control.sha && check.status === 'completed' && check.conclusion === 'success' && + check.app?.id === 15368 && check.app.slug === 'github-actions', 'PRIVATE_ACTIONS_CHECK_INVALID'); + return { check_suite_id: suite.id, job_id: job.id, check_run_id: check.id, started_at: job.started_at, completed_at: job.completed_at }; +} +async function windowOrigin(github, value, workflow, backendTree, baselinePolicy, now) { + const { platform, control, execution: e, app } = value, args = api(PRIVATE); + const runs = await complete(github.rest.actions.listWorkflowRuns, { ...args, workflow_id: workflow.id, + head_sha: control.sha, branch: REF.slice(11), event: 'push' }, 'workflow_runs'); + demand(Math.max(...runs.map(r => r.id)) === e.runId, 'PRIVATE_ACTIONS_LATEST_RUN_REQUIRED'); + const [run, commit, rawTree] = await Promise.all([ + github.rest.actions.getWorkflowRun({ ...args, run_id: e.runId }).then(data), + github.rest.git.getCommit({ ...args, commit_sha: control.sha }).then(data), + github.rest.git.getTree({ ...args, tree_sha: control.sha, recursive: '1' }).then(data), + ]); + runIdentity(run, value, workflow, now); + demand(runs.some(r => r.id === run.id && r.run_attempt === run.run_attempt && r.head_sha === run.head_sha), 'PRIVATE_ACTIONS_EXACT_RUN_REQUIRED'); + demand(commit.sha === control.sha && commit.tree?.sha === control.tree && + commit.message?.split('\n')[0] === `ci(r5): run combined native ${platform}` && + Array.isArray(commit.parents) && commit.parents.length === 1 && sha(commit.parents[0].sha), 'PRIVATE_ACTIONS_CONTROL_COMMIT_INVALID'); + const controlTree = tree(rawTree, control.tree); + demand(controlTree.get(MARKER)?.mode === '100644', 'PRIVATE_ACTIONS_MARKER_INVALID'); + const [markerBytes, policyBytes, workflowBytes] = await Promise.all([MARKER, POLICY, WORKFLOW].map(name => + contents(github, PRIVATE, control.sha, name, controlTree))); + const marker = parseStrictJson(markerBytes, 8192), policy = parseStrictJson(policyBytes, 131072); + demand(keys(marker, ['schema', 'app_sha', 'platform']) && marker.schema === 1 && marker.app_sha === app.sha && marker.platform === platform, + 'PRIVATE_ACTIONS_MARKER_INVALID'); + demand(policy.schema === 1 && policy.app_sha === app.sha && policy.app_tree === app.tree && policy.backend_base === BACKEND && + equal(policy.control_paths, baselinePolicy.control_paths) && equal(policy.materialized_control_paths, baselinePolicy.materialized_control_paths) && + Array.isArray(policy.control_paths) && new Set(policy.control_paths).size === policy.control_paths.length && + policy.control_paths.every(safePath) && policy.materialized_control_paths.every(p => controlTree.has(p)), 'PRIVATE_ACTIONS_POLICY_INVALID'); + const allowed = new Set(policy.control_paths), paths = new Set([...backendTree.keys(), ...controlTree.keys()]); + for (const p of paths) demand((controlTree.get(p)?.sha === backendTree.get(p)?.sha && + controlTree.get(p)?.mode === backendTree.get(p)?.mode) || allowed.has(p), 'PRIVATE_ACTIONS_BACKEND_DELTA_INVALID'); + // The reviewed control commit selects the app itself; the artifact cannot choose it. + const workflowText = workflowBytes.toString('utf8'); + demand(workflowText.includes(`KEEPER_SOURCE_SHA: ${app.sha}`) && + workflowText.includes(`origin ${app.sha}`) && workflowText.includes(`--detach ${app.sha}`), 'PRIVATE_ACTIONS_APP_SELECTION_INVALID'); + const selected = await selectedJob(github, value, run, now); + const artifacts = await complete(github.rest.actions.listWorkflowRunArtifacts, { ...args, run_id: e.runId }, 'artifacts'); + const name = `r5-integration-native-${platform}-${control.sha}-${e.runId}-${e.runAttempt}`; + const found = artifacts.filter(a => a.name === name); + demand(found.length === 1, 'PRIVATE_ACTIONS_ARTIFACT_INVALID'); + const info = data(await github.rest.actions.getArtifact({ ...args, artifact_id: found[0].id })); + artifactIdentity(info, value, found[0].id, now, selected); + return { descriptor: value, run_updated_at: run.updated_at, workflow_id: workflow.id, ...selected, artifact: { id: info.id, name: info.name, size: info.size_in_bytes, + sha256: info.digest.slice(7), expires_at: info.expires_at, created_at: info.created_at }, workflow_source_sha256: digest(workflowBytes), + policy_source_sha256: digest(policyBytes), marker_source_sha256: digest(markerBytes), + parent: commit.parents[0].sha, controlTree }; +} +async function authenticatePrivateActionsOrigins(github, supplied, { clock = Date.now } = {}) { + try { + demand(keys(supplied, ['IOS', 'ANDROID'])); + // Snapshot caller expectations; never populate them from a receipt or API response. + const expected = JSON.parse(JSON.stringify(supplied)); + for (const p of ['IOS', 'ANDROID']) descriptor(expected[p], p); + demand(equal(expected.IOS.app, expected.ANDROID.app) && equal(expected.IOS.backend, expected.ANDROID.backend) && + expected.IOS.control.sha !== expected.ANDROID.control.sha && expected.IOS.execution.runId !== expected.ANDROID.execution.runId); + const args = api(PRIVATE), appArgs = api(APP), app = expected.IOS.app; + const [privateRepo, appRepo, branch, workflow, appCommit, appRawTree, backendCommit, backendRawTree, policyBytes] = await Promise.all([ + github.rest.repos.get(args).then(data), github.rest.repos.get(appArgs).then(data), + github.rest.repos.getBranch({ ...args, branch: REF.slice(11) }).then(data), + github.rest.actions.getWorkflow({ ...args, workflow_id: 'r5-native-integration.yml' }).then(data), + github.rest.git.getCommit({ ...appArgs, commit_sha: app.sha }).then(data), + github.rest.git.getTree({ ...appArgs, tree_sha: app.sha, recursive: '1' }).then(data), + github.rest.git.getCommit({ ...args, commit_sha: BACKEND }).then(data), + github.rest.git.getTree({ ...args, tree_sha: BACKEND, recursive: '1' }).then(data), + contents(github, PRIVATE, REVIEWED_DRIVER, POLICY), + ]); + demand(repository(privateRepo, PRIVATE, PRIVATE_ID) && privateRepo.private === true && repository(appRepo, APP, APP_ID) && + appRepo.private === false, 'PRIVATE_ACTIONS_REPOSITORY_INVALID'); + demand(branch.name === REF.slice(11) && [expected.IOS.control.sha, expected.ANDROID.control.sha].includes(branch.commit?.sha), + 'PRIVATE_ACTIONS_BRANCH_MOVED'); + demand(id(workflow.id) && workflow.path === WORKFLOW && workflow.state === 'active', 'PRIVATE_ACTIONS_WORKFLOW_INVALID'); + demand(appCommit.sha === app.sha && appCommit.tree?.sha === app.tree && backendCommit.sha === BACKEND && sha(backendCommit.tree?.sha || ''), + 'PRIVATE_ACTIONS_COMMIT_INVALID'); + const appTree = tree(appRawTree, app.tree), backendTree = tree(backendRawTree, backendCommit.tree.sha); + const baselinePolicy = parseStrictJson(policyBytes, 131072); + const rows = {}; + for (const p of ['IOS', 'ANDROID']) rows[p] = await windowOrigin(github, expected[p], workflow, backendTree, baselinePolicy, clock()); + const current = rows.IOS.descriptor.control.sha === branch.commit.sha ? rows.IOS : rows.ANDROID; + const previous = current === rows.IOS ? rows.ANDROID : rows.IOS; + demand(current.parent === previous.descriptor.control.sha, 'PRIVATE_ACTIONS_CONTROL_ANCESTRY_INVALID'); + demand(instant(current.started_at) >= instant(previous.completed_at), 'PRIVATE_ACTIONS_NATIVE_WINDOWS_OVERLAP'); + const left = [...rows.IOS.controlTree].filter(([p]) => p !== MARKER).map(([p, v]) => [p, v.mode, v.sha]); + const right = [...rows.ANDROID.controlTree].filter(([p]) => p !== MARKER).map(([p, v]) => [p, v.mode, v.sha]); + demand(equal(left, right), 'PRIVATE_ACTIONS_CONTROL_EQUIVALENCE_INVALID'); + const sourceBytes = new Map(), workerSources = [], recoverySources = []; + for (const name of new Set([...RECOVERY_SOURCES, ...Object.keys(WORKER_SOURCES)])) + sourceBytes.set(name, await contents(github, APP, app.sha, name, appTree)); + for (const name of RECOVERY_SOURCES) recoverySources.push(Object.freeze({ path: name, sha256: digest(sourceBytes.get(name)) })); + for (const [name, names] of Object.entries(WORKER_SOURCES)) workerSources.push(Object.freeze({ path: name, + sha256: digest(sourceBytes.get(name)), names: Object.freeze([...names]) })); + // Source retrieval is network work. Revoke a moved branch, later failed run, + // rerun or changed artifact before issuing even this diagnostic-only origin. + for (const p of ['IOS', 'ANDROID']) { + const row = rows[p], e = expected[p].execution; + const latest = await complete(github.rest.actions.listWorkflowRuns, { ...args, workflow_id: workflow.id, + head_sha: expected[p].control.sha, branch: REF.slice(11), event: 'push' }, 'workflow_runs'); + const run = data(await github.rest.actions.getWorkflowRun({ ...args, run_id: e.runId })); + demand(Math.max(...latest.map(r => r.id)) === e.runId && run.check_suite_id === row.check_suite_id && + latest.some(r => r.id === e.runId && r.run_attempt === e.runAttempt && r.head_sha === expected[p].control.sha), 'PRIVATE_ACTIONS_EXACT_RUN_REQUIRED'); + runIdentity(run, expected[p], workflow, clock()); + demand(equal(await selectedJob(github, expected[p], run, clock()), { check_suite_id: row.check_suite_id, + job_id: row.job_id, check_run_id: row.check_run_id, started_at: row.started_at, completed_at: row.completed_at }), 'PRIVATE_ACTIONS_JOB_CHANGED'); + const currentArtifacts = await complete(github.rest.actions.listWorkflowRunArtifacts, { ...args, run_id: e.runId }, 'artifacts'); + const selected = currentArtifacts.filter(a => a.name === row.artifact.name); + demand(selected.length === 1 && selected[0].id === row.artifact.id, + 'PRIVATE_ACTIONS_ARTIFACT_INVALID'); + const currentArtifact = data(await github.rest.actions.getArtifact({ ...args, artifact_id: row.artifact.id })); + artifactIdentity(currentArtifact, expected[p], row.artifact.id, clock(), row); + demand(currentArtifact.digest === `sha256:${row.artifact.sha256}` && currentArtifact.size_in_bytes === row.artifact.size && + currentArtifact.expires_at === row.artifact.expires_at && currentArtifact.created_at === row.artifact.created_at, + 'PRIVATE_ACTIONS_ARTIFACT_INVALID'); + } + // Artifact readbacks above are network work too. Perform a terminal current + // window fence AFTER all of them, with no subsequent API reads before issue. + const [lastBranch, lastWorkflow, lastRepo, ...lastRuns] = await Promise.all([ + github.rest.repos.getBranch({ ...args, branch: REF.slice(11) }).then(data), + github.rest.actions.getWorkflow({ ...args, workflow_id: 'r5-native-integration.yml' }).then(data), + github.rest.repos.get(args).then(data), + ...['IOS', 'ANDROID'].flatMap(p => { + const e = expected[p].execution; + return [complete(github.rest.actions.listWorkflowRuns, { ...args, workflow_id: workflow.id, head_sha: expected[p].control.sha, + branch: REF.slice(11), event: 'push' }, 'workflow_runs'), + github.rest.actions.getWorkflowRun({ ...args, run_id: e.runId }).then(data)]; + }), + ]); + demand(lastBranch.name === branch.name && lastBranch.commit?.sha === branch.commit.sha, 'PRIVATE_ACTIONS_BRANCH_MOVED'); + demand(lastWorkflow.id === workflow.id && lastWorkflow.path === WORKFLOW && lastWorkflow.state === 'active', 'PRIVATE_ACTIONS_WORKFLOW_INVALID'); + demand(repository(lastRepo, PRIVATE, PRIVATE_ID) && lastRepo.private === true, 'PRIVATE_ACTIONS_REPOSITORY_INVALID'); + ['IOS', 'ANDROID'].forEach((p, index) => { + const [latest, run] = lastRuns.slice(index * 2, index * 2 + 2), e = expected[p].execution; + demand(Math.max(...latest.map(r => r.id)) === e.runId && run.check_suite_id === rows[p].check_suite_id && + latest.some(r => r.id === e.runId && r.run_attempt === e.runAttempt && r.head_sha === expected[p].control.sha), + 'PRIVATE_ACTIONS_EXACT_RUN_REQUIRED'); + runIdentity(run, expected[p], workflow, clock()); + demand(instant(rows[p].artifact.expires_at) > clock(), 'PRIVATE_ACTIONS_ARTIFACT_INVALID'); + }); + const platforms = {}; + for (const p of ['IOS', 'ANDROID']) { + const { controlTree, parent, ...row } = rows[p]; + platforms[p] = Object.freeze({ ...row, expected: Object.freeze({ ...expected[p], recoverySources: Object.freeze(recoverySources), + workerSources: Object.freeze(workerSources) }) }); + } + const result = frozen({ schema: 1, kind: 'private-actions-diagnostic-origin', current_controller_sha: branch.commit.sha, + control_equivalence: 'IDENTICAL_TRACKED_BLOBS_AND_MODES_EXCEPT_EXACT_PLATFORM_REQUEST', platforms: Object.freeze(platforms), + reviewed_window: false, installed_trust: 'UNVERIFIED', native_acceptance: false, release_ready: false, + authorization_eligible: false, release_authorized: false, execution_allowed: false }); + issued.add(result); return result; + } catch (error) { throw new Error(/^PRIVATE_ACTIONS_[A-Z0-9_]{1,80}$/.test(error?.message || '') ? error.message : 'PRIVATE_ACTIONS_ORIGIN_INVALID'); } +} +function isPrivateActionsOrigin(value) { return issued.has(value); } +async function authenticateReviewedPrivateActionsOrigins(github, windowKey, options) { + demand(typeof windowKey === 'string' && Object.hasOwn(REVIEWED_WINDOWS, windowKey), 'PRIVATE_ACTIONS_REVIEWED_WINDOW_REQUIRED'); + return authenticatePrivateActionsOrigins(github, REVIEWED_WINDOWS[windowKey], options); +} +module.exports = { authenticatePrivateActionsOrigins, authenticateReviewedPrivateActionsOrigins, isPrivateActionsOrigin }; diff --git a/.github/native/private_actions_software.cjs b/.github/native/private_actions_software.cjs new file mode 100644 index 0000000000..143414af90 --- /dev/null +++ b/.github/native/private_actions_software.cjs @@ -0,0 +1,133 @@ +'use strict'; +// Original D448 bytes and authenticated read-only Actions facts; never release authority. +const crypto = require('node:crypto'); +const path = require('node:path'); +const { execFileSync } = require('node:child_process'); +const { isDeepStrictEqual: equal } = require('node:util'); +const origin = require('./private_actions_origin.cjs'); +const { parentSoftwareMembers, validateParentSoftwareEvidence } = require('./parent_software_evidence.cjs'); +const PRIVATE = 'KeeperCommunity/bitcoin-keeper-relay'; +const MAX_ARCHIVE = 12 * 1024 * 1024; +const issued = new WeakSet(), authorities = new WeakMap(); +const demand = (yes, code) => { if (!yes) throw new Error(code); }; +const digest = bytes => crypto.createHash('sha256').update(bytes).digest('hex'); +const id = value => Number.isSafeInteger(value) && value > 0; +function frozen(value) { + if (value && typeof value === 'object') { Object.values(value).forEach(frozen); Object.freeze(value); } + return value; +} +function safeError(error) { + return new Error(/^PRIVATE_(?:ACTIONS|SOFTWARE)_[A-Z0-9_]{1,80}$/.test(error?.message || '') ? + error.message : 'PRIVATE_SOFTWARE_COLLECTION_INVALID'); +} +async function boundedBytes(response) { + let reader; + try { + demand(response?.ok === true && typeof response.body?.getReader === 'function', 'PRIVATE_SOFTWARE_ARCHIVE_READ_FAILED'); + const length = response.headers.get('content-length'); + demand(length === null || (/^\d+$/.test(length) && Number(length) <= MAX_ARCHIVE), 'PRIVATE_SOFTWARE_ARCHIVE_TOO_LARGE'); + reader = response.body.getReader(); + const chunks = []; + let size = 0; + for (;;) { + const { done, value } = await reader.read(); + if (done) break; + demand(value instanceof Uint8Array, 'PRIVATE_SOFTWARE_ARCHIVE_READ_FAILED'); + size += value.length; + demand(size <= MAX_ARCHIVE, 'PRIVATE_SOFTWARE_ARCHIVE_TOO_LARGE'); + chunks.push(Buffer.from(value)); + } + demand(length === null || Number(length) === size, 'PRIVATE_SOFTWARE_ARCHIVE_SIZE_MISMATCH'); + return Buffer.concat(chunks); + } finally { + if (reader) await reader.cancel(); + else if (response?.body?.cancel) await response.body.cancel(); + } +} +async function readPrivateArtifactBytes(github, artifactId, { fetchImpl = fetch } = {}) { + try { + demand(id(artifactId), 'PRIVATE_SOFTWARE_ARTIFACT_IDENTITY_INVALID'); + const apiURL = `https://api.github.com/repos/${PRIVATE}/actions/artifacts/${artifactId}/zip`; + let intercepted = false; + const response = await github.rest.actions.downloadArtifact({ owner: 'KeeperCommunity', repo: 'bitcoin-keeper-relay', + artifact_id: artifactId, archive_format: 'zip', request: { timeout: 30000, parseSuccessResponseBody: false, + fetch: async (url, init) => { + demand(!intercepted && url === apiURL && init?.method === 'GET' && init.body == null, + 'PRIVATE_SOFTWARE_ARCHIVE_API_REQUEST_INVALID'); + intercepted = true; + const apiResponse = await fetchImpl(url, { ...init, redirect: 'manual', signal: AbortSignal.timeout(30000) }); + try { + demand(apiResponse?.status === 302, 'PRIVATE_SOFTWARE_ARCHIVE_REDIRECT_REQUIRED'); + return apiResponse; + } finally { if (apiResponse?.body) await apiResponse.body.cancel(); } + } } }); + demand(intercepted && response.status === 302, 'PRIVATE_SOFTWARE_ARCHIVE_REDIRECT_REQUIRED'); + const location = new URL(response.headers.location); + demand(location.protocol === 'https:' && !location.username && !location.password && !location.hash && + (!location.port || location.port === '443') && + (location.hostname.endsWith('.blob.core.windows.net') || location.hostname.endsWith('.actions.githubusercontent.com')), + 'PRIVATE_SOFTWARE_ARCHIVE_HOST_INVALID'); + // Only the authenticated GitHub API hop receives SDK headers. Storage never does. + return await boundedBytes(await fetchImpl(location, { redirect: 'error', signal: AbortSignal.timeout(30000) })); + } catch (error) { throw safeError(error); } +} +function unpackPrivateArtifact(archive, platform) { + try { + demand(Buffer.isBuffer(archive) && archive.length > 0 && archive.length <= MAX_ARCHIVE, + 'PRIVATE_SOFTWARE_ARCHIVE_TOO_LARGE'); + const result = execFileSync('python3', ['-B', path.join(__dirname, 'private_software_artifact_zip.py'), platform], { + input: archive, timeout: 10000, maxBuffer: MAX_ARCHIVE * 2, stdio: ['pipe', 'pipe', 'pipe'], + env: { PATH: process.env.PATH, PYTHONDONTWRITEBYTECODE: '1', PYTHONNOUSERSITE: '1' }, + }); + const envelope = JSON.parse(result.toString('utf8')), names = parentSoftwareMembers(platform); + demand(envelope && !Array.isArray(envelope) && Object.keys(envelope).length === names.length && + Object.keys(envelope).every(name => names.includes(name)), 'PRIVATE_SOFTWARE_ARCHIVE_MEMBERS_INVALID'); + const members = {}; + for (const name of names) { + const value = envelope[name]; + demand(typeof value === 'string', 'PRIVATE_SOFTWARE_ARCHIVE_MEMBERS_INVALID'); + const bytes = Buffer.from(value, 'base64'); + demand(bytes.length > 0 && bytes.toString('base64') === value, 'PRIVATE_SOFTWARE_ARCHIVE_MEMBERS_INVALID'); + members[name] = bytes; + } + return members; + } catch { throw new Error('PRIVATE_SOFTWARE_ARCHIVE_MEMBERS_INVALID'); } +} +async function revalidatePrivateActionsSoftware(github, proof, { clock = Date.now } = {}) { + try { + demand(issued.has(proof), 'PRIVATE_SOFTWARE_CAPABILITY_REQUIRED'); + const original = authorities.get(proof); + const latest = await origin.authenticatePrivateActionsOrigins(github, original.expected, { clock }); + demand(origin.isPrivateActionsOrigin(latest) && equal(latest, original.origin), 'PRIVATE_SOFTWARE_ORIGIN_CHANGED'); + return true; + } catch (error) { throw safeError(error); } +} +async function collectPrivateActionsSoftware(github, expectedPerPlatform, { clock = Date.now, fetchImpl = fetch } = {}) { + try { + // Origin snapshots independent descriptors and validates complete lists; no success filter. + const first = await origin.authenticatePrivateActionsOrigins(github, expectedPerPlatform, { clock }); + demand(origin.isPrivateActionsOrigin(first), 'PRIVATE_SOFTWARE_ORIGIN_REQUIRED'); + const platforms = {}; + for (const platform of ['IOS', 'ANDROID']) { + const selected = first.platforms[platform]; + const archive = await readPrivateArtifactBytes(github, selected.artifact.id, { fetchImpl }); + demand(archive.length === selected.artifact.size && digest(archive) === selected.artifact.sha256, + 'PRIVATE_SOFTWARE_ARTIFACT_DIGEST_MISMATCH'); + const members = unpackPrivateArtifact(archive, platform); + const software = validateParentSoftwareEvidence(members, selected.expected); + platforms[platform] = { artifact_id: selected.artifact.id, archive_sha256: digest(archive), + job_id: selected.job_id, check_run_id: selected.check_run_id, software }; + } + // Later platform download/validation can revoke the first artifact or run. + const expected = Object.fromEntries(['IOS', 'ANDROID'].map(p => [p, first.platforms[p].descriptor])); + const latest = await origin.authenticatePrivateActionsOrigins(github, expected, { clock }); + demand(origin.isPrivateActionsOrigin(latest) && equal(latest, first), 'PRIVATE_SOFTWARE_ORIGIN_CHANGED'); + const result = frozen({ schema: 1, kind: 'private-actions-original-d13-software', platforms, + worker_sources: first.platforms.IOS.expected.workerSources, + installed_trust: 'UNVERIFIED', native_acceptance: false, release_ready: false, + authorization_eligible: false, execution_allowed: false, release_authorized: false }); + issued.add(result); authorities.set(result, { expected, origin: first }); return result; + } catch (error) { throw safeError(error); } +} +module.exports = { readPrivateArtifactBytes, unpackPrivateArtifact, collectPrivateActionsSoftware, + revalidatePrivateActionsSoftware, isPrivateActionsSoftware: value => issued.has(value) }; diff --git a/.github/native/private_integration_receipts.cjs b/.github/native/private_integration_receipts.cjs new file mode 100644 index 0000000000..14a605c6c0 --- /dev/null +++ b/.github/native/private_integration_receipts.cjs @@ -0,0 +1,272 @@ +'use strict'; +// Pure C2 receipt checks. Authenticated Actions origin and installed trust are external. +const crypto = require('node:crypto'); +const { parseStrictJson } = require('./strict_json.cjs'); +const SUITES = Object.freeze(['protected', 'recovery', 'account']); +const SOURCE_SCOPES = Object.freeze({ + protected: 'BOUNDED_PRIVATE_NATIVE_ACCEPTANCE_TOOLING', + recovery: 'BOUNDED_PRIVATE_FRESH_RECOVERY_TOOLING', + account: 'BOUNDED_PRIVATE_NORMAL_ACCOUNT_READ_ISOLATION_TOOLING', +}); +const RECOVERY_SOURCES = Object.freeze([ + 'src/constants/defaultData.tsx', 'src/hardware/index.ts', 'src/hardware/signerSetup.ts', + 'src/models/interfaces/HeathCheckTypes.ts', 'src/services/backup/authority.ts', + 'src/services/backup/encoding.ts', 'src/services/backup/image.ts', + 'src/services/wallets/enums/index.ts', 'src/services/wallets/factories/VaultFactory.ts', + 'src/services/wallets/factories/WalletFactory.ts', 'src/services/wallets/operations/BIP85.ts', + 'src/services/wallets/operations/index.ts', 'src/services/wallets/operations/taproot-utils/noble_ecc.ts', + 'src/services/wallets/operations/utils.ts', 'src/utils/service-utilities/config.ts', + 'src/utils/service-utilities/encryption.ts', 'src/utils/utilities.ts', + 'tests/recovery/support/protected-authority-esm-loader.mjs', +]); +const STAGES = Object.freeze({ + protected: Object.freeze(['normal_home_ready', 'legacy_consent_ready', 'never_enrolled_delete_feedback', 'never_enrolled_visible', + 'check_again_read_only', 'repair_verified', 'second_check_read_only', 'delete_confirmation_copy', + 'cancelled_deletion_preserved_backup', 'delete_confirmed', 'post_delete_check_read_only', + 'cold_relaunch_home', 'cold_deleted_status', 'local_recovery_preserved', 'version_history_checked']), + recovery: Object.freeze(['fresh_recovery_form', 'public_recovery_input_ready', 'protected_recovery_success', + 'restored_home', 'native_restored_verified', 'restored_history', 'cold_home', 'native_cold_verified', 'cold_history']), + account: Object.freeze(['fresh_recovery_form', 'public_recovery_input_ready', 'protected_recovery_success', + 'restored_home', 'native_restored_verified', 'restored_history', 'cold_home', 'native_cold_verified', 'cold_history', + 'a_add_user_ready', 'b_new_home', 'b_first_check_read', 'native_b_first_verified', 'b_new_history', + 'a_return_home', 'a_return_check_read', 'native_a_return_verified', 'a_return_history', + 'b_return_home', 'b_return_check_read', 'native_b_return_verified', 'b_return_history', + 'a_final_home', 'a_final_check_read', 'native_a_final_verified', 'a_final_history']), +}); +const SCOPES = Object.freeze({ protected: 'REAL_CURRENT_PROTOCOL_AND_NORMAL_UI_WITH_CONTROLLED_LEGACY_CONSENT', + recovery: 'FRESH_INSTALL_NORMAL_UI_CURRENT_PROTECTED_RECOVERY', account: 'NORMAL_UI_CURRENT_PROTOCOL_NATIVE_ACCOUNT_READ_ISOLATION' }); +const COUNTS = Object.freeze({ + protected: ['createNewAppCalls', 'protectedSnapshotCalls', 'protectedEnrollmentCalls', 'protectedRepairCalls', + 'protectedDeletionCalls', 'releasedSnapshotResponseGates', 'rejectedOtherBackendCalls', 'providerOperations'], + recovery: ['nativeSnapshotRequests', 'authenticatedPublicSnapshotReplies', 'nativeWriteAttempts', + 'rejectedLegacyReadAttempts', 'rejectedOtherCalls', 'providerOperations'], + account: ['nativeSnapshotRequests', 'authenticatedAReplies', 'authenticatedBReplies', 'nativeRegistrations', + 'nativeWriteAttempts', 'rejectedLegacyReads', 'rejectedOtherCalls', 'providerOperations', + 'recoveryWitnesses', 'accountWitnesses', 'gatesReleased'], +}); +const ACCOUNT_BOOLEANS = ['durableBRegistered', 'distinctProtectedNamespaces', 'bNeverEnrolled']; +const PHASES = ['initial', 'before_protected', 'after_protected', 'before_recovery', 'after_recovery', 'before_account', 'after_account']; +const METADATA_SCOPE = { ...SCOPES, protected: 'NORMAL_UI_AND_CURRENT_PROTOCOL_WITH_CONTROLLED_LEGACY_CONSENT' }; +const HISTORY = { + protected: 'Fresh fixture installed tuple and Initially installed row after cold login; no historical account-switch claim.', + recovery: 'Restored public account current installed tuple and Recovered Wallet row, after recovery and cold PIN login; no historical account-switch claim.', + account: 'Selected native A Recovered Wallet and B Initially installed rows, one exact current tuple each after cold PIN selection.', +}; +const IMAGE = 'mongo:7.0.16@sha256:c630c59342c1493d50345136df2af14a76b9e827dd5316bfabee07a0880a5f3a'; +const demand = condition => { if (!condition) throw new Error('PRIVATE_INTEGRATION_RECEIPT_INVALID'); }; +const object = value => value !== null && typeof value === 'object' && !Array.isArray(value); +const keys = (value, fields) => object(value) && Object.keys(value).sort().join('|') === [...fields].sort().join('|'); +const hex = (value, length) => typeof value === 'string' && new RegExp(`^[a-f0-9]{${length}}$`).test(value); +const integer = (value, min, max) => Number.isSafeInteger(value) && value >= min && value <= max; +const digest = bytes => crypto.createHash('sha256').update(bytes).digest('hex'); +// Compare JSON values independently of producer dictionary insertion order. +function same(a, b) { + if (a === b) return true; + if (Array.isArray(a)) return Array.isArray(b) && a.length === b.length && a.every((value, i) => same(value, b[i])); + return object(a) && object(b) && keys(a, Object.keys(b)) && Object.keys(a).every(key => same(a[key], b[key])); +} +function fixed(value, fields) { demand(object(value) && Object.entries(fields).every(([key, expected]) => same(value[key], expected))); } +function noFailure(value) { demand(object(value) && !Object.keys(value).some(key => /(?:^failure(?:_|$)|^cleanup_failure(?!_categories$))/.test(key)) && + (!Object.hasOwn(value, 'cleanup') || value.cleanup === 'PASS') && + (!Object.hasOwn(value, 'cleanup_failure_categories') || same(value.cleanup_failure_categories, [])) && + (!Object.hasOwn(value, 'startup_cleanup_errors') || same(value.startup_cleanup_errors, [])) && + !Object.hasOwn(value, 'startup_observation_cleanup') && + (!Object.hasOwn(value, 'install_succeeded') || value.install_succeeded === true) && + (!Object.hasOwn(value, 'device') || (object(value.device) && + (!Object.hasOwn(value.device, 'fresh_disposable_state') || value.device.fresh_disposable_state === true)))); } +function privateIntegrationMembers(platform) { + demand(['IOS', 'ANDROID'].includes(platform)); + const name = platform.toLowerCase(), base = `control/artifacts/native-integration/${name}`; + return Object.freeze([`${base}/metadata.json`, `${base}/startup.json`, ...SUITES.flatMap(suite => + ['metadata.json', 'fixture.json'].map(file => `control/artifacts/native-${suite}/${name}/${file}`)), + 'control/artifacts/native-protected/mongo.json', `app/artifacts/native/${name}/metadata.json`]); +} +function context(expected) { + const { app, control, backend, execution, platform, recoverySources } = expected; + const repository = value => typeof value === 'string' && /^[A-Za-z0-9_.-]+\/[A-Za-z0-9_.-]+$/.test(value); + demand(['IOS', 'ANDROID'].includes(platform) && app.repository === 'KeeperCommunity/bitcoin-keeper' && + [app, control].every(row => keys(row, ['repository', 'sha', 'tree']) && repository(row.repository) && hex(row.sha, 40) && hex(row.tree, 40)) && + keys(backend, ['repository', 'sha']) && repository(backend.repository) && hex(backend.sha, 40) && + keys(execution, ['repository', 'workflowPath', 'workflowSha', 'ref', 'runId', 'runAttempt']) && + execution.repository === control.repository && execution.workflowSha === control.sha && + execution.workflowPath === '.github/workflows/r5-native-integration.yml' && + typeof execution.ref === 'string' && /^refs\/heads\/[A-Za-z0-9_./-]+$/.test(execution.ref) && + !execution.ref.includes('..') && integer(execution.runId, 1, Number.MAX_SAFE_INTEGER) && integer(execution.runAttempt, 1, Number.MAX_SAFE_INTEGER)); + demand(Array.isArray(recoverySources) && recoverySources.length === RECOVERY_SOURCES.length && + recoverySources.every((row, i) => keys(row, ['path', 'sha256']) && row.path === RECOVERY_SOURCES[i] && hex(row.sha256, 64))); + return expected; +} +function installed(value, platform, built, previous) { + const fields = platform === 'IOS' ? ['inventory_sha256', 'executable_sha256'] : ['sha256']; + const method = platform === 'IOS' ? 'FULL_SIMULATOR_BUNDLE_TREE_AND_EXECUTABLE_SHA256' : 'SINGLE_INSTALLED_BASE_APK_SHA256'; + const proof = value.installed_artifact, binary = platform === 'IOS' ? 'executable_sha256' : 'sha256'; + demand(value.installed_artifact_verified === true && keys(proof, ['method', ...fields]) && proof.method === method && + fields.every(key => hex(proof[key], 64)) && proof[binary] === built[binary] && (!previous || same(proof, previous))); + return proof; +} +function builtIdentity(value, platform) { + demand(typeof value.path === 'string' && value.path.length > 0 && value.path.length <= 4096 && !/[\x00-\x1f]/.test(value.path)); + if (platform === 'IOS') demand(keys(value, ['path', 'executable', 'executable_sha256', 'bundle_tree_sha256']) && + typeof value.executable === 'string' && /^[A-Za-z0-9_.-]{1,256}$/.test(value.executable) && + hex(value.executable_sha256, 64) && hex(value.bundle_tree_sha256, 64)); + else demand(keys(value, ['path', 'sha256']) && value.path === 'android/app/build/outputs/apk/development/debug/app-development-debug.apk' && hex(value.sha256, 64)); +} +function foundation(value, expected, built, proof) { + const { platform, app, execution } = expected; + noFailure(value); + fixed(value, { schema: 1, platform, source_sha: app.sha, workflow_sha: execution.workflowSha, + workflow_ref: `${execution.repository}/${execution.workflowPath}@${execution.ref}`, + run_id: String(execution.runId), run_attempt: String(execution.runAttempt), diagnostic_only: false, + app_id: platform === 'IOS' ? 'io.hexawallet.hexakeeper.dev' : 'io.hexawallet.keeper.development', build: 'PASS', smoke: 'PASS' }); + demand(same(value.built_artifact, built)); + return installed(value, platform, built, proof); +} +function sourceProofs(value, expected) { + demand(keys(value, SUITES)); + SUITES.forEach(suite => { + const row = value[suite]; + demand(keys(row, ['sourceVerification', 'rawSources', 'app', 'backendBase', 'control', 'scope', 'nativeAcceptance'])); + fixed(row, { sourceVerification: 'PASS', app: expected.app.sha, backendBase: expected.backend.sha, control: expected.control.sha, + scope: SOURCE_SCOPES[suite], nativeAcceptance: 'NOT_ESTABLISHED_BY_SOURCE_CHECK' }); + demand(keys(row.rawSources, ['app', 'control'])); + for (const kind of ['app', 'control']) demand(keys(row.rawSources[kind], ['files', 'sha256']) && + integer(row.rawSources[kind].files, kind === 'app' ? 2404 : 98, 100000) && hex(row.rawSources[kind].sha256, 64)); + demand(same(row.rawSources, value.protected.rawSources)); + }); +} +function fixture(value, suite, expected) { + const common = { schema: 1, status: 'PASS', scope: SCOPES[suite], completedStages: STAGES[suite], failures: [], + physicalDevice: 'NOT_TESTED', releaseReadiness: 'NOT_ESTABLISHED' }; + const fields = suite === 'protected' ? { ...common, + historicalUpgradeInstallation: 'NOT_TESTED', normalAppEntry: 'INDEX_JS_IMPORTED_BY_CONTROLLED_FIXTURE', + witnessKind: 'FIXTURE_AND_PROTOCOL_RECEIPT_NOT_UI_ATTESTATION', retentionCanary: 'SYNTHETIC_OPAQUE_LEGACY_RECORD_SCOPE_ONLY', + accountIsolationCanary: 'INDEPENDENT_THROWAWAY_ES256K_NAMESPACE', + nativeAcceptance: 'REQUIRES_SEPARATE_EXACT_INSTALLED_ARTIFACT_AND_UI_RECEIPT' } : { ...common, + appSha: expected.app.sha, cryptoSourceBaseline: expected.app.sha, historicalUpgrade: 'NOT_TESTED', + fixture: suite === 'recovery' ? 'CONTROLLED_PUBLIC_BIP39_PACKET_WITH_PRODUCTION_CRYPTO' : 'CONTROLLED_PUBLIC_A_AND_NATIVE_CREATED_B', + witnessKind: 'PROTOCOL_AND_FIXED_STAGE_RECEIPT_REQUIRES_EXACT_NATIVE_UI_AND_BYTES', + ...(suite === 'recovery' ? { nativeAccountSwitching: 'NOT_TESTED' } : { bMutationIsolation: 'NOT_TESTED' }) }; + demand(keys(value, [...Object.keys(fields), ...COUNTS[suite], ...(suite === 'protected' ? [] : + ['ownerSha256', 'sourceMembers', ...(suite === 'account' ? ACCOUNT_BOOLEANS : [])])])); + fixed(value, fields); + demand(COUNTS[suite].every(key => integer(value[key], 0, 10000)) && value.providerOperations === 0); + if (suite === 'protected') { + // Five explicit fresh status gates, plus separate repair and Delete ACK/readbacks. + demand(value.createNewAppCalls === 1 && value.releasedSnapshotResponseGates === 5 && + value.protectedSnapshotCalls >= value.releasedSnapshotResponseGates + 2 && value.protectedEnrollmentCalls >= 1 && + value.protectedRepairCalls >= 1 && value.protectedDeletionCalls >= 1); + } else { + demand(hex(value.ownerSha256, 64) && same(value.sourceMembers, expected.recoverySources) && value.nativeWriteAttempts === 0); + if (suite === 'recovery') demand(value.authenticatedPublicSnapshotReplies >= 1 && + value.authenticatedPublicSnapshotReplies <= value.nativeSnapshotRequests && value.rejectedLegacyReadAttempts === 0); + else demand(value.authenticatedAReplies >= 3 && value.authenticatedBReplies >= 2 && + value.authenticatedAReplies + value.authenticatedBReplies <= value.nativeSnapshotRequests && value.nativeRegistrations === 1 && + ACCOUNT_BOOLEANS.every(key => value[key] === true) && value.recoveryWitnesses === 2 && value.accountWitnesses === 4 && + value.gatesReleased === 4 && value.rejectedLegacyReads === 0); + } +} +function suiteMetadata(value, suite, expected, built, proof, finalFixture) { + const snapshot = Object.fromEntries(Object.entries(finalFixture).filter(([key]) => !['ready', 'cleanup'].includes(key))); + fixture(snapshot, suite, expected); + noFailure(value); + const fields = { schema: 1, status: 'PASS', app_sha: expected.app.sha, backend_base: expected.backend.sha, + control_sha: expected.control.sha, platform: expected.platform, scope: METADATA_SCOPE[suite], + entrypoint: suite === 'protected' ? 'PRIVATE_WRAPPER_IMPORTS_UNCHANGED_INDEX_JS' : 'UNCHANGED_INDEX_JS_AND_DECLARED_READ_ONLY_NATIVE_OBSERVER', + fresh_install_recovery: suite === 'protected' ? 'NOT_TESTED' : 'CONTROLLED_PUBLIC_BIP39_PACKET_REQUIRES_NATIVE_UI_AND_REALM', + native_account_switch_A_B_A: suite === 'account' ? 'NORMAL_UI_ADD_USER_AND_COLD_PIN_SELECTION_REQUIRES_NATIVE_CHECKS' : 'NOT_TESTED', + physical_devices: 'NOT_TESTED', historical_upgrade_installation: 'NOT_TESTED', version_history_scope: HISTORY[suite], + performance_sample: false, release_readiness: 'NOT_ESTABLISHED', cleanup: 'PASS', cleanup_failure_categories: [], + fixture: snapshot, device_kind: expected.platform === 'IOS' ? 'SIMULATOR' : 'EMULATOR', + ...(suite === 'account' ? { b_backend_mutation_isolation: 'NOT_TESTED', biometrics: 'NOT_TESTED' } : {}) }; + demand(keys(value, [...Object.keys(fields), 'ui_test_cases', 'installed_artifact_verified', 'installed_artifact', + ...(suite === 'protected' ? [] : ['fixture_owner_sha256'])])); + fixed(value, fields); + demand(finalFixture.ready === false && finalFixture.cleanup === 'PASS' && integer(value.ui_test_cases, 1, Number.MAX_SAFE_INTEGER)); + if (suite !== 'protected') demand(value.fixture_owner_sha256 === snapshot.ownerSha256); + installed(value, expected.platform, built, proof); +} +function mongo(value, expected) { + const fields = { owner: `keeper-r5-native-protected-${expected.execution.runId}-${expected.execution.runAttempt}`, + version: '7.0.16', kind: expected.platform === 'ANDROID' ? 'DOCKER' : 'OWNED_MAC_PROCESS', cleanup: 'PASS', ready: true }; + if (expected.platform === 'ANDROID') fields.image = IMAGE; + else fields.checksum_scope = 'OFFICIAL_TLS_VENDOR_CHECKSUM_NOT_INDEPENDENT_CONTENT_PIN'; + demand(keys(value, [...Object.keys(fields), 'disposition', ...(expected.platform === 'IOS' ? ['archive_sha256'] : [])])); + fixed(value, fields); + demand(['removed', 'absent'].includes(value.disposition) && (expected.platform !== 'IOS' || hex(value.archive_sha256, 64))); +} +// Fixed object-level components for a separately reviewed raw D profile. These +// never parse, project or reserialize a C2 aggregate, or grant origin/authority. +function privateSuites(values, hashes, expected, built, proof, suites) { + demand(keys(suites, SUITES)); + const owners = new Set(); + for (const suite of SUITES) { + const prefix = `control/artifacts/native-${suite}/${expected.platform.toLowerCase()}`, row = suites[suite]; + demand(keys(row, ['status', 'scope', 'execution_started', 'cleanup', 'process_exit_code', 'receipt_sha256', 'post_suite_barrier'])); + fixed(row, { status: 'PASS', scope: SCOPES[suite], execution_started: true, cleanup: 'PASS', process_exit_code: 0, + post_suite_barrier: 'PASS', receipt_sha256: { 'metadata.json': hashes[`${prefix}/metadata.json`], 'fixture.json': hashes[`${prefix}/fixture.json`] } }); + suiteMetadata(values[`${prefix}/metadata.json`], suite, expected, built, proof, values[`${prefix}/fixture.json`]); + if (suite !== 'protected') { + const owner = values[`${prefix}/fixture.json`].ownerSha256; + demand(!owners.has(owner)); owners.add(owner); + } + } +} +function validatePrivateIntegration(rawFiles, suppliedExpected) { + try { + const expected = context(suppliedExpected), members = privateIntegrationMembers(expected.platform); + demand(keys(rawFiles, members)); + const values = {}, hashes = {}; + for (const name of members) { + const bytes = rawFiles[name], limit = name.startsWith('app/') || name.endsWith('/startup.json') ? 2_000_000 : + name.endsWith('/fixture.json') || name.endsWith('/mongo.json') ? 8192 : name === members[0] ? 131072 : 32768; + demand(Buffer.isBuffer(bytes) && bytes.length > 0 && bytes.length <= limit); + values[name] = parseStrictJson(bytes, limit); + demand(object(values[name])); hashes[name] = digest(bytes); + } + const [aggregateName, startupName] = members, aggregate = values[aggregateName], startup = values[startupName]; + noFailure(aggregate); + fixed(aggregate, { schema: 1, status: 'PASS', app_sha: expected.app.sha, app_tree: expected.app.tree, + control_sha: expected.control.sha, control_tree: expected.control.tree, backend_base: expected.backend.sha, + platform: expected.platform, run_id: String(expected.execution.runId), run_attempt: String(expected.execution.runAttempt), + scope: 'THREE_SERIAL_NORMAL_UI_FEATURE_SUITES_ONE_BUILD', physical_devices: 'NOT_TESTED', historical_upgrade: 'NOT_TESTED', + b_backend_mutation_isolation: 'NOT_TESTED', biometrics: 'NOT_TESTED', + five_startup_measurements: 'REQUIRES_SEPARATE_RAW_MEASUREMENT_EVIDENCE', trusted_acceptance: 'NOT_ESTABLISHED', release_readiness: 'NOT_ESTABLISHED' }); + demand(keys(aggregate, ['schema', 'status', 'app_sha', 'app_tree', 'control_sha', 'control_tree', 'backend_base', 'platform', + 'run_id', 'run_attempt', 'built_artifact', 'scope', 'source_admission', 'barriers', 'suites', 'physical_devices', + 'historical_upgrade', 'b_backend_mutation_isolation', 'biometrics', 'five_startup_measurements', 'trusted_acceptance', + 'release_readiness', 'startup_receipt_sha256'])); + const built = aggregate.built_artifact; + builtIdentity(built, expected.platform); + const proof = foundation(startup, expected, built); + foundation(values[members.at(-1)], expected, built, proof); + demand(aggregate.startup_receipt_sha256 === hashes[startupName]); + sourceProofs(aggregate.source_admission, expected); + demand(keys(aggregate.suites, SUITES) && Array.isArray(aggregate.barriers) && aggregate.barriers.length === PHASES.length); + const retained = {}; + aggregate.barriers.forEach((row, index) => { + if (index === 1) retained[startupName.slice('control/'.length)] = hashes[startupName]; + if (index >= 2 && index % 2 === 0) { + const suite = SUITES[index / 2 - 1], prefix = `control/artifacts/native-${suite}/${expected.platform.toLowerCase()}`; + for (const file of ['metadata.json', 'fixture.json']) retained[`${prefix}/${file}`.slice('control/'.length)] = hashes[`${prefix}/${file}`]; + } + demand(keys(row, ['phase', 'source_proofs', 'built_artifact', 'resources', 'retained_receipt_sha256'])); + fixed(row, { phase: PHASES[index], source_proofs: aggregate.source_admission, built_artifact: built, + retained_receipt_sha256: retained, resources: { + overlays_absent: SUITES.map(suite => `.github/native/.private-${suite}`), + listeners: [8081, 8765, 8768, 8769].map(port => ({ port, result: 'ECONNREFUSED' })) } }); + }); + privateSuites(values, hashes, expected, built, proof, aggregate.suites); + mongo(values['control/artifacts/native-protected/mongo.json'], expected); + const identity = Object.freeze({ app: Object.freeze({ ...expected.app }), control: Object.freeze({ ...expected.control }), + backend: Object.freeze({ ...expected.backend }), execution: Object.freeze({ ...expected.execution }) }); + return Object.freeze({ schema: 1, platform: expected.platform, identity, raw_sha256: Object.freeze(hashes), + installed_artifact: Object.freeze({ ...proof }), source_observations: Object.freeze({ + app: Object.freeze({ ...aggregate.source_admission.protected.rawSources.app }), + control: Object.freeze({ ...aggregate.source_admission.protected.rawSources.control }) }), + scope: 'three serial private UI/protocol suites; public phases and authenticated workflow cleanup remain separate', + public_parent_composition: 'NOT_FROZEN', installed_trust: 'UNVERIFIED', release_ready: false, + authorization_eligible: false, execution_allowed: false, release_authorized: false }); + } catch { throw new Error('PRIVATE_INTEGRATION_RECEIPT_INVALID'); } +} +const PRIVATE_INTEGRATION_CORE = Object.freeze({ context, builtIdentity, foundation, sourceProofs, privateSuites, mongo }); +module.exports = { validatePrivateIntegration, privateIntegrationMembers, RECOVERY_SOURCES, PRIVATE_INTEGRATION_CORE }; diff --git a/.github/native/private_software_artifact_zip.md b/.github/native/private_software_artifact_zip.md new file mode 100644 index 0000000000..4d0f5a7d14 --- /dev/null +++ b/.github/native/private_software_artifact_zip.md @@ -0,0 +1,64 @@ +# Private D13 artifact transport preparation + +`private_software_artifact_zip.py` is a separate, pure ZIP32 reader for the +thirteen selected-platform metadata members emitted by Relay D +`448e09d7a373fc7e9d9c540880b45e2c742997f7` (#187). It does not change the +existing single-member `software_artifact_zip.py` API, limits or consumers. +This source package does not enable an Actions route, collect artifacts, +authenticate source identities, validate receipt contents, grant native or +installed trust, or authorize an operation. + +`read_archive(raw: bytes, platform: str) -> dict[str, bytes]` accepts exactly +`IOS` or `ANDROID`. `member_limits(platform)` declares the finite thirteen +names and their canonical return order. The returned values are the original +uncompressed bytes, including invalid JSON or binary bytes. Receipt schema, +identity, source hashes, native proofs, stage outcomes and cleanup must be +validated separately against an independently authenticated expected context. +The reader never parses or reserializes member contents and never extracts +paths to disk. + +The fixed limits match the parent software validator's D profile: + +| Members | Count | Maximum uncompressed bytes each | +| --- | ---: | ---: | +| Integration aggregate, startup, final app metadata, three public wrappers | 6 | 2,000,000 | +| Protected, recovery and account metadata | 3 | 32,768 | +| Three fixture receipts and Mongo receipt | 4 | 8,192 | + +Every member must be nonempty. The member caps total 12,131,072 bytes. Both +the archive input and total inflated output have a fixed 12 MiB (12,582,912 +byte) ceiling. The 451,840 byte margin accommodates ZIP headers for an archive +with every member at its individual limit. In particular, the aggregate has +the same 2,000,000 byte limit as the raw validator; the actual producer's +2,048-event budget ledger can exceed 262,144 bytes before its other fields. +There are no caller-provided names, limits or bypass modes. + +Any physical permutation of the exact thirteen selected-platform names is +accepted. Central records must follow that physical order and describe every +contiguous local record. Return order alone is canonical: original C2 ten +members, followed by public-probe, public-features and public-product. The +reader rejects prefixes, gaps, overlaps, unlisted records, hidden compressed +tails, trailing data, duplicate or unknown names, alternate paths, links, +directories, ZIP64, encryption, multidisk archives and comments. Only stored +or raw-deflate payloads, matching ZIP32 headers/descriptors, valid CRCs and +finite timestamp extensions are accepted. Deflate output is bounded during +decompression, then checked against the declared size and member cap. + +The CLI takes one platform argument and reads at most the archive limit plus +one byte from stdin. Its successful stdout is a compact base64 mapping for +captured IPC, with no member-content JSON interpretation. A consumer must +capture this envelope and must not print it in logs. Failure returns exit 1, +empty stdout and the fixed `PRIVATE_SOFTWARE_ARTIFACT_ZIP_INVALID` error without +a payload, path or traceback. No source identity is inferred from the CLI. + +Run the synthetic host controls with: + +```sh +PYTHONDONTWRITEBYTECODE=1 python3 -m unittest discover -s .github/tests -p 'test*software_artifact_zip.py' -v +``` + +These controls use generated disposable byte archives. They prove transport +behavior only. They do not run an app, simulator, emulator, backend, provider, +Actions API, native suite or release operation. Live transport and the private +Actions adapter remain separate packages with their own reviewed identities +and evidence requirements. diff --git a/.github/native/private_software_artifact_zip.py b/.github/native/private_software_artifact_zip.py new file mode 100644 index 0000000000..3e60651d0f --- /dev/null +++ b/.github/native/private_software_artifact_zip.py @@ -0,0 +1,175 @@ +#!/usr/bin/env python3 +"""Finite D13 ZIP32 transport. Return original bytes; never extract or grant trust.""" +import base64 +import json +import stat +import struct +import sys +import zlib + +MAX_ARCHIVE = MAX_TOTAL = 12 * 1024 * 1024 +ERROR = 'PRIVATE_SOFTWARE_ARTIFACT_ZIP_INVALID' +PLATFORMS = ('IOS', 'ANDROID') + + +def member_limits(platform): + """Fixed D448 profile, in canonical return order, without caller overrides.""" + _require(type(platform) is str and platform in PLATFORMS) + p = platform.lower() + return { + f'control/artifacts/native-integration/{p}/metadata.json': 2000000, + f'control/artifacts/native-integration/{p}/startup.json': 2000000, + f'control/artifacts/native-protected/{p}/metadata.json': 32768, + f'control/artifacts/native-protected/{p}/fixture.json': 8192, + f'control/artifacts/native-recovery/{p}/metadata.json': 32768, + f'control/artifacts/native-recovery/{p}/fixture.json': 8192, + f'control/artifacts/native-account/{p}/metadata.json': 32768, + f'control/artifacts/native-account/{p}/fixture.json': 8192, + 'control/artifacts/native-protected/mongo.json': 8192, + f'app/artifacts/native/{p}/metadata.json': 2000000, + f'control/artifacts/native-integration/{p}/public-probe.json': 2000000, + f'control/artifacts/native-integration/{p}/public-features.json': 2000000, + f'control/artifacts/native-integration/{p}/public-product.json': 2000000, + } + + +def _require(condition): + if not condition: + raise ValueError(ERROR) + + +def _extras(raw, central=False): + """Only bounded, non-path-bearing ZIP32 timestamp extensions are accepted.""" + offset, seen = 0, set() + while offset < len(raw): + _require(offset + 4 <= len(raw)) + kind, size = struct.unpack_from('= 1 and 0 < value[0] <= 7) + lengths = {1 + 4 * value[0].bit_count()} + if central: + lengths.add(1 + 4 * (value[0] & 1)) + _require(size in lengths) + elif kind == 0x000a: + _require(size == 32 and value[:4] == b'\0' * 4 and + value[4:8] == struct.pack('= 30 and + length >= 46 * count and start + length == end) + cursor, total, entries, seen = start, 0, [], set() + for _ in range(count): + _require(cursor + 46 <= end) + fields = struct.unpack_from('<4s6H3I5H2I', raw, cursor) + (signature, made, needed, flags, method, mtime, mdate, crc, packed, size, + namesize, extrasize, commentsize, disk, _internal, external, local) = fields + stop = cursor + 46 + namesize + extrasize + _require(stop <= end) + name = raw[cursor + 46:cursor + 46 + namesize].decode('ascii') + mode = external >> 16 + _require(signature == b'PK\x01\x02' and name in limits and name not in seen and + made >> 8 in (0, 3) and 10 <= needed <= 20 and + flags & ~0x080e == 0 and method in (0, 8) and + (method != 0 or flags & 6 == 0) and 0 < size <= limits[name] and + 0 < packed <= MAX_ARCHIVE and commentsize == disk == 0 and + stat.S_IFMT(mode) in (0, stat.S_IFREG) and external & 0x10 == 0) + _extras(raw[cursor + 46 + namesize:stop], central=True) + seen.add(name) + total += size + _require(total <= MAX_TOTAL) + entries.append((name, needed, flags, method, mtime, mdate, crc, packed, size, local)) + cursor = stop + _require(cursor == end and seen == set(limits)) + return start, entries + + +def _member(raw, entry, following, cap): + name, needed, flags, method, mtime, mdate, crc, packed, size, local = entry + _require(local + 30 <= following) + fields = struct.unpack_from('<4s5H3I2H', raw, local) + (signature, version, local_flags, local_method, local_time, local_date, + local_crc, local_packed, local_size, namesize, extrasize) = fields + start = local + 30 + namesize + extrasize + stop = start + packed + _require(signature == b'PK\x03\x04' and version == needed and local_flags == flags and + local_method == method and local_time == mtime and local_date == mdate and + start <= following and stop <= following and + raw[local + 30:local + 30 + namesize] == name.encode('ascii')) + _extras(raw[local + 30 + namesize:start]) + if flags & 8: + _require(local_crc in (0, crc) and local_packed in (0, packed) and local_size in (0, size)) + descriptor = raw[stop:following] + if len(descriptor) == 16: + _require(descriptor[:4] == b'PK\x07\x08') + descriptor = descriptor[4:] + _require(len(descriptor) == 12 and struct.unpack('