Skip to content
37 changes: 25 additions & 12 deletions src/services/backup/image.ts
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,8 @@ export const pauseBackup = () =>
export const MAX_RECORD_BYTES = 8 * 1024 * 1024;
export const MAX_BACKUP_BYTES = 32 * 1024 * 1024;

export class BackupSnapshotValidationError extends Error {}

export function canonical(value: any): string {
if (value === undefined) return 'null';
if (value === null || typeof value !== 'object') return JSON.stringify(value);
Expand Down Expand Up @@ -211,7 +213,7 @@ export async function encryptRecord(key: string, record: any): Promise<string> {

export async function decryptRecord(key: string, value: unknown) {
if (typeof value !== 'string' || value.length > MAX_RECORD_BYTES * 2)
throw new Error('Invalid backup record');
throw new BackupSnapshotValidationError('Invalid backup record');
// Parsing/format decoding still require native performance validation at the
// supported size ceiling. Yield before and after each record.
await pauseBackup();
Expand All @@ -238,7 +240,7 @@ export async function decryptRecord(key: string, value: unknown) {
}
const record = JSON.parse(text);
if (!record || typeof record !== 'object' || Array.isArray(record))
throw new Error('Invalid backup record');
throw new BackupSnapshotValidationError('Invalid backup record');
await pauseBackup();
return record;
}
Expand All @@ -247,19 +249,24 @@ export async function decodeImage(
key: string,
response: any,
appId: string,
checkpoint = () => {}
checkpoint = () => {},
verifyWalletKeys = false
): Promise<BackupImage> {
const app = response?.appImage;
if (
!app ||
app.appId !== appId ||
!app.wallets ||
typeof app.wallets !== 'object' ||
Array.isArray(app.wallets) ||
!app.signers ||
typeof app.signers !== 'object' ||
Array.isArray(app.signers) ||
!Array.isArray(app.nodes) ||
!Array.isArray(response.allVaultImages) ||
!Array.isArray(response.labels)
)
throw new Error('Incomplete backup response');
throw new BackupSnapshotValidationError('Incomplete backup response');
const checkReferences = (expected: unknown, actual: unknown[]) => {
if (expected === undefined) return; // Legacy backups can omit reference lists.
if (
Expand All @@ -269,7 +276,7 @@ export async function decodeImage(
new Set(actual).size !== actual.length ||
actual.some((id) => typeof id !== 'string' || !expected.includes(id))
)
throw new Error('Incomplete backup references');
throw new BackupSnapshotValidationError('Incomplete backup references');
};
checkReferences(
app.vaults,
Expand All @@ -283,17 +290,23 @@ export async function decodeImage(
let totalBytes = 0;
const decode = async (value: unknown) => {
checkpoint();
if (typeof value !== 'string') throw new Error('Invalid backup record');
if (typeof value !== 'string') throw new BackupSnapshotValidationError('Invalid backup record');
totalBytes += value.length;
if (totalBytes > MAX_BACKUP_BYTES) throw new Error('Backup exceeds supported size');
if (totalBytes > MAX_BACKUP_BYTES)
throw new BackupSnapshotValidationError('Backup exceeds supported size');
return decryptRecord(key, value);
};
const add = (kind: keyof BackupImage, id: string, record: any) => {
if (!id || image[kind][id]) throw new Error('Duplicate or unidentified backup record');
if (!id || image[kind][id])
throw new BackupSnapshotValidationError('Duplicate or unidentified backup record');
image[kind][id] = record;
};
for (const value of Object.values(app.wallets)) {
for (const [storedId, value] of Object.entries(app.wallets)) {
const record = await decode(value);
if (typeof record.id !== 'string' || !record.id)
throw new BackupSnapshotValidationError('Invalid wallet backup identity');
if (verifyWalletKeys && storedId !== record.id)
throw new BackupSnapshotValidationError('Wallet backup identity mismatch');
add('wallets', record.id, record);
}
for (const [id, value] of Object.entries(app.signers)) {
Expand All @@ -303,7 +316,7 @@ export async function decodeImage(
for (const value of response.allVaultImages) {
const record = await decode(value?.vault);
if (value.vaultId !== undefined && value.vaultId !== record.id)
throw new Error('Vault backup identity mismatch');
throw new BackupSnapshotValidationError('Vault backup identity mismatch');
add('vaults', record.id, record);
}
for (const value of app.nodes) {
Expand All @@ -316,8 +329,8 @@ export async function decodeImage(
}
// A partial relay response must never look like an empty backup.
if (Array.isArray(app.vaults) && app.vaults.length !== Object.keys(image.vaults).length)
throw new Error('Incomplete vault backup');
throw new BackupSnapshotValidationError('Incomplete vault backup');
if (Array.isArray(app.labels) && app.labels.length !== Object.keys(image.labels).length)
throw new Error('Incomplete label backup');
throw new BackupSnapshotValidationError('Incomplete label backup');
return image;
}
89 changes: 80 additions & 9 deletions src/services/backup/repair.ts
Original file line number Diff line number Diff line change
Expand Up @@ -10,12 +10,15 @@ import {
} from 'src/utils/utilities';
import {
BACKUP_DEADLINE_MS,
BackupRequestError,
backupQueuedRevision,
backupRevision,
boundedBackupPost,
withBackupSession,
} from './transport';
import {
BackupImage,
BackupSnapshotValidationError,
canonical,
compareImages,
decodeImage,
Expand All @@ -35,6 +38,16 @@ export type RepairPhase =
| 'verified'
| 'conflict'
| 'unverified';
type InspectionStage =
| 'local-capture'
| 'snapshot-request'
| 'snapshot-validation'
| 'snapshot-decryption'
| 'comparison'
| 'repair-preparation'
| 'repair-upload'
| 'repair-readback'
| 'local-recheck';
const inFlight = new Map<
string,
{
Expand Down Expand Up @@ -148,16 +161,24 @@ export function inspectBackup(
if (active) active.phase = phase;
listeners.forEach((listener) => listener(phase));
};
const result = withBackupSession(appId, async () => {
const deadline = Date.now() + BACKUP_DEADLINE_MS;
let deadline: number;
const inspectOnce = async (allowRetry: boolean): Promise<RepairPhase | 'retry'> => {
let stage: InspectionStage = 'local-capture';
let changedRevision = 0;
// One deadline covers both read-only attempts; a retry must not double it.
deadline ??= Date.now() + BACKUP_DEADLINE_MS;
const revision = backupRevision(appId);
const assertCurrent = () => {
if (
Date.now() >= deadline ||
(dbManager.getObjectByIndex(RealmSchema.KeeperApp) as any)?.id !== appId ||
backupRevision(appId) !== revision
(dbManager.getObjectByIndex(RealmSchema.KeeperApp) as any)?.id !== appId
)
throw new Error('Backup changed during check');
const currentRevision = backupRevision(appId);
if (currentRevision !== revision) {
changedRevision = currentRevision;
throw new Error('Backup changed during check');
}
};
try {
notify('checking');
Expand All @@ -168,12 +189,14 @@ export function inspectBackup(
assertCurrent();
let expectedRevision: string;
let needsVaultRepair = false;
const read = async () => {
const read = async (readback = false) => {
stage = readback ? 'repair-readback' : 'snapshot-request';
const response = await boundedBackupPost(
`${config.RELAY}getBackupSnapshot`,
{ appId },
deadline
);
if (!readback) stage = 'snapshot-validation';
assertCurrent();
if (!/^[a-f0-9]{64}$/.test(response.data?.revision))
throw new Error('Backup snapshot revision unavailable');
Expand Down Expand Up @@ -208,7 +231,10 @@ export function inspectBackup(
},
}
: response.data;
const decoded = await decodeImage(key, availableResponse, appId, assertCurrent);
if (!readback) stage = 'snapshot-decryption';
// Restore can salvage legacy map aliases. Inspection requires the
// stored key to match the ID inside each encrypted wallet record.
const decoded = await decodeImage(key, availableResponse, appId, assertCurrent, true);
needsVaultRepair =
unavailable.length > 0 ||
response.data.allVaultImages.some(
Expand All @@ -219,6 +245,7 @@ export function inspectBackup(
return decoded;
};
const remote = await read();
stage = 'comparison';
const contentComparison = await compareImages(local, remote, assertCurrent);
const comparison =
contentComparison === 'matched' && needsVaultRepair ? 'different' : contentComparison;
Expand All @@ -232,6 +259,7 @@ export function inspectBackup(
return 'different';
}
if (comparison === 'different') {
stage = 'repair-preparation';
notify('preparing');
const payload = await encodeImage(app, local, assertCurrent);
assertCurrent();
Expand All @@ -243,6 +271,7 @@ export function inspectBackup(
throw new Error('Local backup changed');
assertCurrent();
notify('uploading');
stage = 'repair-upload';
try {
const response = await boundedBackupPost(
`${config.RELAY}repairAppBackup`,
Expand All @@ -257,13 +286,16 @@ export function inspectBackup(
// A lost response is not proof the upload failed. Always read back.
}
notify('verifying');
const verifiedRemote = await read();
stage = 'repair-readback';
const verifiedRemote = await read(true);
stage = 'repair-readback';
if (
needsVaultRepair ||
(await compareImages(local, verifiedRemote, assertCurrent)) !== 'matched'
)
throw new Error('Backup not verified');
}
stage = 'local-recheck';
if (
(await compareImages(local, await captureBackupImage(assertCurrent), assertCurrent)) !==
'matched'
Expand All @@ -272,11 +304,50 @@ export function inspectBackup(
assertCurrent();
notify('verified');
return 'verified';
} catch {
} catch (error) {
// A routine incremental write can be queued while a read-only check is
// running. A transient snapshot request can also fail while the backup is
// healthy. Retry either case once behind queued writes, within the same
// deadline. Never retry a repair, invalid response, or account switch.
let accountIsCurrent = false;
try {
accountIsCurrent = (dbManager.getObjectByIndex(RealmSchema.KeeperApp) as any)?.id === appId;
} catch {
// Realm may already be closed during an account switch.
}
const queuedWriteChanged = changedRevision && backupQueuedRevision(appId) >= changedRevision;
const transientSnapshotFailure =
stage === 'snapshot-request' &&
error instanceof BackupRequestError &&
['http-5xx', 'network', 'timeout'].includes(error.category);
if (
allowRetry &&
!repair &&
accountIsCurrent &&
Date.now() < deadline &&
(queuedWriteChanged || transientSnapshotFailure)
)
return 'retry';
// Only log fixed categories. Errors and responses can contain encrypted
// backup data, account identifiers, headers, or request bodies.
const category = error instanceof BackupRequestError ? error.category : 'non-transport';
const diagnosticStage =
stage === 'snapshot-decryption' && error instanceof BackupSnapshotValidationError
? 'snapshot-validation'
: stage;
globalThis.console.warn(
'Assisted Server Backup check failed at stage:',
diagnosticStage,
category
);
notify('unverified');
return 'unverified';
}
});
};
const runOnce = (allowRetry: boolean) => withBackupSession(appId, () => inspectOnce(allowRetry));
const result: Promise<RepairPhase> = runOnce(true).then(async (phase) =>
phase === 'retry' ? ((await runOnce(false)) as RepairPhase) : phase
);
inFlight.set(appId, { repair, promise: result, listeners });
const clean = () => {
if (inFlight.get(appId)?.promise === result) inFlight.delete(appId);
Expand Down
6 changes: 5 additions & 1 deletion src/services/backup/restore.ts
Original file line number Diff line number Diff line change
Expand Up @@ -11,12 +11,16 @@ export async function prepareRecoveryImage(
labels: any[]
) {
if (!appImage || appImage.appId !== appId) throw new Error('Recovery data unavailable');
// Older Relay registrations and deletions stored empty maps as empty arrays.
// Keep those recoverable, but let decodeImage reject nonempty malformed arrays.
const legacyEmptyMap = (value: any) => (Array.isArray(value) && !value.length ? {} : value);
return decodeImage(
encryptionKey,
{
appImage: {
...appImage,
signers: appImage.signers === undefined ? {} : appImage.signers,
wallets: legacyEmptyMap(appImage.wallets),
signers: legacyEmptyMap(appImage.signers === undefined ? {} : appImage.signers),
nodes: appImage.nodes === undefined ? [] : appImage.nodes,
},
allVaultImages,
Expand Down
36 changes: 32 additions & 4 deletions src/services/backup/transport.ts
Original file line number Diff line number Diff line change
Expand Up @@ -3,11 +3,37 @@ import RestClient from '../rest/RestClient';
// One queue per account covers repair, incremental writes, labels and deletion.
const queues = new Map<string, Promise<unknown>>();
const revisions = new Map<string, number>();
const queuedMutationRevisions = new Map<string, number>();
export const backupRevision = (id: string) => revisions.get(id) || 0;
export const backupQueuedRevision = (id: string) => queuedMutationRevisions.get(id) || 0;
export const markBackupMutation = (id: string) => revisions.set(id, backupRevision(id) + 1);
export const BACKUP_STALL_MS = 30_000;
export const BACKUP_DEADLINE_MS = 10 * 60_000;

export type BackupRequestFailure = 'http-4xx' | 'http-5xx' | 'timeout' | 'network' | 'other';

export class BackupRequestError extends Error {
constructor(public readonly category: BackupRequestFailure) {
super('Backup request could not be completed');
}
}

function requestFailureCategory(error: unknown, aborted: boolean): BackupRequestFailure {
if (aborted) return 'timeout';
if (!error || typeof error !== 'object') return 'other';
const failure = error as { response?: { status?: unknown }; status?: unknown; code?: unknown; name?: unknown; message?: unknown };
const status = failure.response?.status ?? failure.status;
if (typeof status === 'number') {
if (status >= 400 && status < 500) return 'http-4xx';
if (status >= 500 && status < 600) return 'http-5xx';
return 'other';
}
if (failure.code === 'ECONNABORTED' || failure.code === 'ETIMEDOUT' || failure.name === 'AbortError')
return 'timeout';
if (failure.code === 'ERR_NETWORK' || failure.message === 'Network Error') return 'network';
return 'other';
}

export function withBackupSession<T>(id: string, work: () => Promise<T>): Promise<T> {
const previous = queues.get(id) || Promise.resolve();
const result = previous.catch(() => undefined).then(work);
Expand Down Expand Up @@ -35,7 +61,7 @@ export async function boundedBackupPost(
idle = setTimeout(() => controller.abort(), BACKUP_STALL_MS);
};
const remaining = deadline - Date.now();
if (remaining <= 0) throw new Error('Backup deadline exceeded');
if (remaining <= 0) throw new BackupRequestError('timeout');
const total = setTimeout(() => controller.abort(), remaining);
resetIdle();
try {
Expand All @@ -54,9 +80,9 @@ export async function boundedBackupPost(
}
},
});
} catch {
} catch (error) {
// Do not propagate Axios errors: they can contain request bodies and headers.
throw new Error('Backup request could not be completed');
throw new BackupRequestError(requestFailureCategory(error, controller.signal.aborted));
} finally {
clearTimeout(idle);
clearTimeout(total);
Expand All @@ -67,5 +93,7 @@ export function backupPost(path: string, body: any) {
const appId = body.appId || body.appID;
if (!appId) return Promise.reject(new Error('Missing backup account'));
markBackupMutation(appId);
return withBackupSession(appId, () => boundedBackupPost(path, body));
const result = withBackupSession(appId, () => boundedBackupPost(path, body));
queuedMutationRevisions.set(appId, backupRevision(appId));
return result;
}
Loading
Loading