diff --git a/.gitignore b/.gitignore index bbdb5c99f..1dc084d97 100644 --- a/.gitignore +++ b/.gitignore @@ -175,3 +175,6 @@ e2e/reports/ # maestro-runner iOS driver artifacts drivers + +# hdiffz/hpatchz built by scripts/binary-patch/build-hdiffpatch.sh +.hdiffpatch-tools/ diff --git a/THIRD-PARTY-NOTICES b/THIRD-PARTY-NOTICES new file mode 100644 index 000000000..3784382d0 --- /dev/null +++ b/THIRD-PARTY-NOTICES @@ -0,0 +1,93 @@ +THIRD PARTY NOTICES +=================== + +This package includes third party source code. The notices below apply to that +code only; the rest of this project is covered by LICENSE.md. + + +------------------------------------------------------------------------------- +HDiffPatch +------------------------------------------------------------------------------- + +Version: v5.1.3 +Source: https://github.com/sisong/HDiffPatch/tree/v5.1.3 +License: MIT +Vendored: cpp/binarypatch/vendor/HDiffPatch/ + +Only the files required to apply a patch on device are vendored, from +libHDiffPatch/HPatch: patch.c, patch.h, patch_private.h, patch_types.h, +checksum_plugin.h and hpatch_mt/hpatch_mt.h. The diff side of HDiffPatch is not +vendored; the `hdiffz` command line tool is built from upstream sources by +scripts/binary-patch/build-hdiffpatch.sh and is not distributed with this package. + +MIT License + +HDiffPatch +Copyright (c) 2012-2025 housisong + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. + + +------------------------------------------------------------------------------- +Zstandard (zstd) +------------------------------------------------------------------------------- + +Version: 1.5.7 +Source: https://github.com/sisong/zstd + (the fork HDiffPatch builds against, derived from + https://github.com/facebook/zstd) + The files here are a snapshot of that fork's dev branch; the copies in + this repository are the version the appliers are built from. +License: BSD-3-Clause +Vendored: cpp/binarypatch/vendor/zstd/ + +Only the decompression path is vendored (lib/common and lib/decompress). The +amd64 assembly implementation of the Huffman decoder is not vendored because the +appliers are compiled with ZSTD_DISABLE_ASM=1. + +BSD License + +For Zstandard software + +Copyright (c) Meta Platforms, Inc. and affiliates. All rights reserved. + +Redistribution and use in source and binary forms, with or without modification, +are permitted provided that the following conditions are met: + + * Redistributions of source code must retain the above copyright notice, this + list of conditions and the following disclaimer. + + * Redistributions in binary form must reproduce the above copyright notice, + this list of conditions and the following disclaimer in the documentation + and/or other materials provided with the distribution. + + * Neither the name Facebook, nor Meta, nor the names of its contributors may + be used to endorse or promote products derived from this software without + specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR +ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON +ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/cli/fixtures/binary-patch/base.bundle b/cli/fixtures/binary-patch/base.bundle new file mode 100644 index 000000000..bf87b00a4 Binary files /dev/null and b/cli/fixtures/binary-patch/base.bundle differ diff --git a/cli/fixtures/binary-patch/target.bundle b/cli/fixtures/binary-patch/target.bundle new file mode 100644 index 000000000..354372b0b Binary files /dev/null and b/cli/fixtures/binary-patch/target.bundle differ diff --git a/cli/fixtures/binary-patch/update.patch b/cli/fixtures/binary-patch/update.patch new file mode 100644 index 000000000..f5609754b Binary files /dev/null and b/cli/fixtures/binary-patch/update.patch differ diff --git a/cli/utils/binaryPatch.test.ts b/cli/utils/binaryPatch.test.ts new file mode 100644 index 000000000..6d68b1b43 --- /dev/null +++ b/cli/utils/binaryPatch.test.ts @@ -0,0 +1,254 @@ +import { spawnSync } from "child_process"; +import crypto from "crypto"; +import fs from "fs"; +import os from "os"; +import path from "path"; +import { afterAll, beforeAll, describe, expect, it } from "@jest/globals"; +import { applyPatch, generatePatch, resolveBinaryPatchTool } from "./binaryPatch.js"; + +/** + * Locks the binary patch codec contract: which bytes a patch restores, and which + * kinds of broken input the appliers do and do not catch. + * + * Everything here runs against real binaries and real bytes - hdiffz/hpatchz and a + * host build of the native applier - because the whole point is to prove that the + * committed patch format really is what every applier understands. + */ + +/** Cloning and building hdiffz/hpatchz from source takes a while on a cold machine. */ +const BUILD_TOOLS_TIMEOUT_MS = 10 * 60 * 1000; +const BUILD_APPLIER_TIMEOUT_MS = 5 * 60 * 1000; + +function findRepoRoot(): string { + let dir = process.cwd(); + for (;;) { + if (fs.existsSync(path.join(dir, "scripts", "binary-patch", "build-hdiffpatch.sh"))) { + return dir; + } + const parent = path.dirname(dir); + if (parent === dir) { + throw new Error(`cannot locate the repository root from ${process.cwd()}`); + } + dir = parent; + } +} + +const repoRoot = findRepoRoot(); +const fixtureDir = path.join(repoRoot, "cli", "fixtures", "binary-patch"); +const baseFixture = path.join(fixtureDir, "base.bundle"); +const targetFixture = path.join(fixtureDir, "target.bundle"); +const patchFixture = path.join(fixtureDir, "update.patch"); +/** + * One shared tree builds the applier for both platforms: iOS pulls it in through the + * podspec at the repository root, Android through externalNativeBuild. + */ +const applierDir = path.join(repoRoot, "cpp", "binarypatch"); + +let workDir: string; + +function sha256(filePath: string): string { + return crypto.createHash("sha256").update(fs.readFileSync(filePath)).digest("hex"); +} + +function workPath(name: string): string { + return path.join(workDir, name); +} + +/** Writes a copy of `sourcePath` with `mutate` applied to its bytes. */ +function writeMutatedCopy(sourcePath: string, name: string, mutate: (bytes: Buffer) => Buffer): string { + const destination = workPath(name); + fs.writeFileSync(destination, mutate(fs.readFileSync(sourcePath))); + return destination; +} + +/** + * Builds hdiffz/hpatchz when they are missing so a clean checkout - and CI - can run + * this suite without a manual setup step. + */ +function ensureBinaryPatchTools(): void { + try { + resolveBinaryPatchTool("hdiffz"); + resolveBinaryPatchTool("hpatchz"); + return; + } catch { + // Not built yet; build them once below. + } + const script = path.join(repoRoot, "scripts", "binary-patch", "build-hdiffpatch.sh"); + const result = spawnSync(script, { encoding: "utf8", timeout: BUILD_TOOLS_TIMEOUT_MS }); + if (result.status !== 0) { + throw new Error(`${script} failed:\n${result.stdout ?? ""}${result.stderr ?? ""}`); + } +} + +function cCompiler(): string { + return process.env.CC || "cc"; +} + +function hasCCompiler(): boolean { + return spawnSync(cCompiler(), ["--version"], { stdio: "ignore" }).status === 0; +} + +beforeAll(() => { + workDir = fs.mkdtempSync(path.join(os.tmpdir(), "codepush-binary-patch-")); + ensureBinaryPatchTools(); +}, BUILD_TOOLS_TIMEOUT_MS); + +afterAll(() => { + fs.rmSync(workDir, { recursive: true, force: true }); +}); + +describe("generatePatch/applyPatch", () => { + it("restores the target bytes from a freshly generated patch", () => { + const generatedPatch = workPath("generated.patch"); + const restored = workPath("restored-from-generated.bundle"); + + generatePatch(baseFixture, targetFixture, generatedPatch); + applyPatch(baseFixture, generatedPatch, restored); + + expect(fs.statSync(generatedPatch).size).toBeGreaterThan(0); + expect(sha256(restored)).toBe(sha256(targetFixture)); + }); + + it("restores the target bytes from the committed patch fixture", () => { + const restored = workPath("restored-from-fixture.bundle"); + + applyPatch(baseFixture, patchFixture, restored); + + expect(sha256(restored)).toBe(sha256(targetFixture)); + }); + + it("reports a missing patch tool with a message that says how to get it", () => { + const previous = process.env.HDIFFPATCH_TOOLS_DIR; + process.env.HDIFFPATCH_TOOLS_DIR = workPath("no-such-dir"); + try { + expect(() => resolveBinaryPatchTool("hdiffz")).toThrow(/build-hdiffpatch\.sh/); + } finally { + if (previous === undefined) { + delete process.env.HDIFFPATCH_TOOLS_DIR; + } else { + process.env.HDIFFPATCH_TOOLS_DIR = previous; + } + } + }); +}); + +describe("applyPatch failure modes", () => { + it("fails on a patch with a corrupted header", () => { + const corrupted = writeMutatedCopy(patchFixture, "corrupt-header.patch", (bytes) => { + // Byte 0 is inside the patch type string, which the header parser reads first. + bytes[0] ^= 0xff; + return bytes; + }); + + expect(() => applyPatch(baseFixture, corrupted, workPath("out-corrupt-header.bundle"))).toThrow(/hpatchz/); + }); + + it("fails on a truncated patch", () => { + const truncated = writeMutatedCopy(patchFixture, "truncated.patch", (bytes) => + bytes.subarray(0, Math.floor(bytes.length / 2)), + ); + + expect(() => applyPatch(baseFixture, truncated, workPath("out-truncated.bundle"))).toThrow(/hpatchz/); + }); +}); + +/** + * A compressed diff carries no checksum of the base data, and the zstd streams + * inside it are written without content checksums, so an applier cannot tell every + * kind of wrong input apart from a legitimate one. These tests pin that down: the + * apply step may report success, but it never produces the target bytes. Callers + * must therefore verify the base and target hashes themselves rather than treating + * a successful apply as proof of a correct result. + */ +describe("errors the codec cannot detect", () => { + it("never produces the target bytes when a byte inside the compressed diff data is corrupted", () => { + const corrupted = writeMutatedCopy(patchFixture, "corrupt-body.patch", (bytes) => { + bytes[Math.floor(bytes.length / 2)] ^= 0xff; + return bytes; + }); + const restored = workPath("out-corrupt-body.bundle"); + + // Whether the corruption is reported depends on where it lands: a flipped bit + // inside a zstd literals block decodes to different bytes without any error, + // while one that invalidates a block header does fail. Either way the result + // is not the target, which is all the caller can rely on. + let restoredHash: string | null = null; + try { + applyPatch(baseFixture, corrupted, restored); + restoredHash = sha256(restored); + } catch { + restoredHash = null; + } + + expect(restoredHash).not.toBe(sha256(targetFixture)); + }); + + it("applies but produces wrong bytes when the base has the expected size and different content", () => { + const wrongBase = writeMutatedCopy(baseFixture, "wrong-base.bundle", (bytes) => { + bytes[1000] ^= 0xff; + return bytes; + }); + const restored = workPath("out-wrong-base.bundle"); + + applyPatch(wrongBase, patchFixture, restored); + + expect(fs.statSync(wrongBase).size).toBe(fs.statSync(baseFixture).size); + expect(sha256(restored)).not.toBe(sha256(targetFixture)); + }); +}); + +/** + * The native appliers on Android and iOS are built from the vendored sources, so a + * host build of the same sources is what proves they agree with hdiffz/hpatchz. + */ +const describeNativeApplier = hasCCompiler() ? describe : describe.skip; + +describeNativeApplier("host build of the native applier", () => { + let hostApplier: string; + + const runHostApplier = (oldPath: string, patch: string, outPath: string): number | null => { + const result = spawnSync(hostApplier, [oldPath, patch, outPath], { encoding: "utf8" }); + return result.status; + }; + + beforeAll(() => { + hostApplier = workPath("apply_patch_host"); + const script = path.join(applierDir, "host", "build.sh"); + const result = spawnSync(script, [hostApplier], { encoding: "utf8", timeout: BUILD_APPLIER_TIMEOUT_MS }); + if (result.status !== 0) { + throw new Error(`${script} failed:\n${result.stdout ?? ""}${result.stderr ?? ""}`); + } + }, BUILD_APPLIER_TIMEOUT_MS); + + it("restores the same target bytes as the CLI applier", () => { + const restored = workPath("out-host.bundle"); + + expect(runHostApplier(baseFixture, patchFixture, restored)).toBe(0); + expect(sha256(restored)).toBe(sha256(targetFixture)); + }); + + it("reports a corrupted patch header as exit code 3", () => { + const corrupted = writeMutatedCopy(patchFixture, "host-corrupt-header.patch", (bytes) => { + bytes[0] ^= 0xff; + return bytes; + }); + + expect(runHostApplier(baseFixture, corrupted, workPath("out-host-corrupt-header.bundle"))).toBe(3); + }); + + it("reports a base of unexpected size as exit code 5", () => { + const shortBase = writeMutatedCopy(baseFixture, "host-short-base.bundle", (bytes) => + bytes.subarray(0, bytes.length - 16), + ); + + expect(runHostApplier(shortBase, patchFixture, workPath("out-host-short-base.bundle"))).toBe(5); + }); + + it("reports a truncated patch as exit code 6", () => { + const truncated = writeMutatedCopy(patchFixture, "host-truncated.patch", (bytes) => + bytes.subarray(0, Math.floor(bytes.length / 2)), + ); + + expect(runHostApplier(baseFixture, truncated, workPath("out-host-truncated.bundle"))).toBe(6); + }); +}); diff --git a/cli/utils/binaryPatch.ts b/cli/utils/binaryPatch.ts new file mode 100644 index 000000000..aec7852b8 --- /dev/null +++ b/cli/utils/binaryPatch.ts @@ -0,0 +1,103 @@ +/** + * Binary patch codec for CodePush updates. + * + * A binary patch update carries only the difference between the bundle that is + * already inside the app binary and the bundle a release wants to run, instead of + * the whole bundle. Patches are produced by HDiffPatch's `hdiffz` and applied on + * device by the native appliers; `hpatchz` is the reference applier used here so a + * patch can be verified right after it is produced. + * + * The tool options below are part of the format contract, not tuning knobs: + * -m-6 the base bundle is held in memory while patching, which is what + * the native appliers do, and match score 6 suits bytecode + * -c-zstd-21-24 zstd is the only decompressor the native appliers link + * -f overwrite the output file if it already exists + * Changing them produces patches the native appliers cannot read. + * + * A patch contains no checksum of the base data, and its zstd streams carry no + * content checksums, so a successful apply is not proof of a correct result: a base + * of the right size but the wrong content, or a corrupted patch body, can both apply + * "successfully" and yield wrong bytes. Callers must verify the base and target + * hashes themselves. + */ + +import { spawnSync } from "child_process"; +import fs from "fs"; +import path from "path"; + +/** + * Identifies the patch format so a client can refuse a patch it cannot apply. + * Bump the format version whenever the tool options above change. + */ +export const BINARY_PATCH_ALGORITHM = 'hdiffpatch-m-zstd'; +export const BINARY_PATCH_FORMAT_VERSION = 1; + +export type BinaryPatchTool = 'hdiffz' | 'hpatchz'; + +const HDIFFZ_OPTIONS = ['-f', '-m-6', '-c-zstd-21-24']; +const HPATCHZ_OPTIONS = ['-f', '-m']; + +const TOOLS_DIR_ENV_NAME = 'HDIFFPATCH_TOOLS_DIR'; +const TOOLS_DIR_NAME = '.hdiffpatch-tools'; +const BUILD_SCRIPT_PATH = 'scripts/binary-patch/build-hdiffpatch.sh'; + +/** + * Finds the hdiffz/hpatchz executable, looking at `HDIFFPATCH_TOOLS_DIR` first and + * then at a `.hdiffpatch-tools` directory in the working directory or any directory + * above it. The tools are built from source rather than installed as a package + * dependency, so the error explains how to get them. + */ +export function resolveBinaryPatchTool(tool: BinaryPatchTool): string { + const configuredDir = process.env[TOOLS_DIR_ENV_NAME]; + if (configuredDir) { + const configured = path.join(configuredDir, tool); + if (fs.existsSync(configured)) { + return configured; + } + throw new Error( + `${TOOLS_DIR_ENV_NAME} is set to '${configuredDir}' but it does not contain '${tool}'. ` + + `Build the tools with '${BUILD_SCRIPT_PATH}'.`, + ); + } + + let directory = path.resolve(process.cwd()); + for (;;) { + const candidate = path.join(directory, TOOLS_DIR_NAME, tool); + if (fs.existsSync(candidate)) { + return candidate; + } + const parent = path.dirname(directory); + if (parent === directory) { + break; + } + directory = parent; + } + + throw new Error( + `'${tool}' not found in any '${TOOLS_DIR_NAME}' directory at or above '${process.cwd()}'. ` + + `Build it with '${BUILD_SCRIPT_PATH}', or set ${TOOLS_DIR_ENV_NAME} to a directory that contains it.`, + ); +} + +/** Writes the patch that turns the bundle at `basePath` into the one at `targetPath`. */ +export function generatePatch(basePath: string, targetPath: string, patchPath: string): void { + runTool('hdiffz', [...HDIFFZ_OPTIONS, basePath, targetPath, patchPath]); +} + +/** Writes the bundle that `patchPath` produces from the bundle at `basePath`. */ +export function applyPatch(basePath: string, patchPath: string, outputPath: string): void { + runTool('hpatchz', [...HPATCHZ_OPTIONS, basePath, patchPath, outputPath]); +} + +function runTool(tool: BinaryPatchTool, args: string[]): void { + const executable = resolveBinaryPatchTool(tool); + const result = spawnSync(executable, args, { encoding: 'utf8' }); + + if (result.error) { + throw new Error(`failed to run ${tool}: ${result.error.message}`); + } + if (result.status !== 0) { + const reason = result.status === null ? `signal ${result.signal}` : `exit code ${result.status}`; + throw new Error(`${tool} failed with ${reason}\n${result.stdout ?? ''}${result.stderr ?? ''}`); + } +} diff --git a/cpp/binarypatch/binarypatch_zstd_decompressor.c b/cpp/binarypatch/binarypatch_zstd_decompressor.c new file mode 100644 index 000000000..91d96b2e3 --- /dev/null +++ b/cpp/binarypatch/binarypatch_zstd_decompressor.c @@ -0,0 +1,187 @@ +#include "binarypatch_zstd_decompressor.h" + +#include +#include + +#include "zstd.h" + +/* + * Largest window a decompression session will accept, pinned to the window that the + * `-c-zstd-21-24` generation options use: no patch this project produces declares a + * window above 2^24. Keeping the bound at the generation limit means a corrupted + * frame header that asks for a wider window is rejected before anything is + * allocated, instead of being answered with an allocation that large - and applying + * a patch keeps several decompression sessions open at once, so the bound applies + * several times over. Changing the generation options is a patch format version + * change, and this bound moves with them. + */ +#define BINARYPATCH_ZSTD_WINDOW_LOG_MAX 24 + +/* + * One decompression session. The input and output buffers are allocated in the + * same block as the struct (`buffers` is their start) to keep the number of + * allocations per patch session down. + */ +typedef struct { + const hpatch_TStreamInput* codeStream; + hpatch_StreamPos_t codeReadPos; + hpatch_StreamPos_t codeEnd; + + ZSTD_DStream* dstream; + ZSTD_inBuffer in; + size_t inCapacity; + ZSTD_outBuffer out; + size_t outReadPos; /* bytes of `out` already handed to the caller */ + + hpatch_dec_error_t decError; + unsigned char buffers[1]; /* [inCapacity bytes][out.size bytes] */ +} binarypatch_zstd_session; + +static hpatch_BOOL _zstd_fail(binarypatch_zstd_session* self) { + if (self->decError == hpatch_dec_ok) { + self->decError = hpatch_dec_error; + } + return hpatch_FALSE; +} + +static hpatch_BOOL _zstd_is_can_open(const char* compressType) { + return (0 == strcmp(compressType, BINARYPATCH_ZSTD_COMPRESS_TYPE)) ? hpatch_TRUE : hpatch_FALSE; +} + +static hpatch_decompressHandle _zstd_open(struct hpatch_TDecompress* decompressPlugin, + hpatch_StreamPos_t dataSize, + const struct hpatch_TStreamInput* codeStream, + hpatch_StreamPos_t code_begin, + hpatch_StreamPos_t code_end) { + const size_t inCapacity = ZSTD_DStreamInSize(); + const size_t outCapacity = ZSTD_DStreamOutSize(); + binarypatch_zstd_session* self; + + /* The uncompressed size is tracked by the caller's patch stream, not here. */ + (void)dataSize; + + self = (binarypatch_zstd_session*)malloc(sizeof(binarypatch_zstd_session) + inCapacity + outCapacity); + if (!self) { + _hpatch_update_decError(decompressPlugin, hpatch_dec_mem_error); + return 0; + } + memset(self, 0, sizeof(binarypatch_zstd_session)); + self->codeStream = codeStream; + self->codeReadPos = code_begin; + self->codeEnd = code_end; + self->in.src = self->buffers; + self->in.size = 0; + self->in.pos = 0; /* pos == size: the input buffer starts empty and is refilled on demand */ + self->inCapacity = inCapacity; + self->out.dst = self->buffers + inCapacity; + self->out.size = outCapacity; + self->out.pos = 0; + self->outReadPos = 0; + + self->dstream = ZSTD_createDStream(); + if (!self->dstream) { + free(self); + _hpatch_update_decError(decompressPlugin, hpatch_dec_open_error); + return 0; + } + /* + * The window bound is applied here rather than left at zstd's default, and a + * session that cannot take it is refused: without the bound a corrupted frame + * header could ask for a much larger window than any patch legitimately needs. + */ + if (ZSTD_isError(ZSTD_initDStream(self->dstream)) || + ZSTD_isError(ZSTD_DCtx_setParameter(self->dstream, ZSTD_d_windowLogMax, + BINARYPATCH_ZSTD_WINDOW_LOG_MAX))) { + ZSTD_freeDStream(self->dstream); + free(self); + _hpatch_update_decError(decompressPlugin, hpatch_dec_open_error); + return 0; + } + return self; +} + +static hpatch_BOOL _zstd_close(struct hpatch_TDecompress* decompressPlugin, + hpatch_decompressHandle decompressHandle) { + binarypatch_zstd_session* self = (binarypatch_zstd_session*)decompressHandle; + hpatch_BOOL result = hpatch_TRUE; + if (!self) { + return hpatch_TRUE; + } + /* Report the session's failure through the plugin so the caller can read it. */ + if (self->decError != hpatch_dec_ok) { + _hpatch_update_decError(decompressPlugin, self->decError); + } + if (0 != ZSTD_freeDStream(self->dstream)) { + result = hpatch_FALSE; + _hpatch_update_decError(decompressPlugin, hpatch_dec_close_error); + } + free(self); + return result; +} + +/* + * Must fill the whole `[out_part_data, out_part_data_end)` range; anything less is + * an error by the `hpatch_TDecompress` contract. + */ +static hpatch_BOOL _zstd_decompress_part(hpatch_decompressHandle decompressHandle, + unsigned char* out_part_data, + unsigned char* out_part_data_end) { + binarypatch_zstd_session* self = (binarypatch_zstd_session*)decompressHandle; + + while (out_part_data < out_part_data_end) { + size_t ready = self->out.pos - self->outReadPos; + size_t ret; + + if (ready > 0) { + const size_t wanted = (size_t)(out_part_data_end - out_part_data); + if (ready > wanted) { + ready = wanted; + } + memcpy(out_part_data, (const unsigned char*)self->out.dst + self->outReadPos, ready); + out_part_data += ready; + self->outReadPos += ready; + continue; + } + + if (self->in.pos == self->in.size) { + const hpatch_StreamPos_t remaining = self->codeEnd - self->codeReadPos; + unsigned char* inBuf = (unsigned char*)self->in.src; + size_t toRead = self->inCapacity; + if (remaining < (hpatch_StreamPos_t)toRead) { + toRead = (size_t)remaining; + } + self->in.pos = 0; + self->in.size = toRead; + if (toRead > 0) { + if (!self->codeStream->read(self->codeStream, self->codeReadPos, inBuf, inBuf + toRead)) { + return _zstd_fail(self); + } + self->codeReadPos += toRead; + } + } + + self->out.pos = 0; + self->outReadPos = 0; + ret = ZSTD_decompressStream(self->dstream, &self->out, &self->in); + if (ZSTD_isError(ret)) { + return _zstd_fail(self); + } + /* + * No progress with no input left means the compressed stream ended before + * the patch asked for its last byte - a truncated or corrupt patch. + */ + if (self->out.pos == 0) { + return _zstd_fail(self); + } + } + return hpatch_TRUE; +} + +void binarypatch_zstd_decompressor_init(hpatch_TDecompress* out_decompressor) { + memset(out_decompressor, 0, sizeof(hpatch_TDecompress)); + out_decompressor->is_can_open = _zstd_is_can_open; + out_decompressor->open = _zstd_open; + out_decompressor->close = _zstd_close; + out_decompressor->decompress_part = _zstd_decompress_part; + /* reset_code stays NULL: it is only needed for vcdiff style patches. */ +} diff --git a/cpp/binarypatch/binarypatch_zstd_decompressor.h b/cpp/binarypatch/binarypatch_zstd_decompressor.h new file mode 100644 index 000000000..e76b80002 --- /dev/null +++ b/cpp/binarypatch/binarypatch_zstd_decompressor.h @@ -0,0 +1,47 @@ +/* + * zstd decompressor for HDiffPatch. + * + * HDiffPatch's patch functions take the decompressor as a plugin, so every codec + * the appliers must understand has to be supplied through `hpatch_TDecompress`. + * CodePush patches are always produced with `hdiffz -f -m-6 -c-zstd-21-24`, so + * zstd is the only codec implemented here; upstream ships a demo header covering + * a dozen codecs, but pulling that in would drag along headers for codecs the + * appliers never see. + * + * `hpatch_TDecompress` carries a mutable `decError` field, so each patch session + * must own its instance. Callers therefore declare the struct themselves and + * initialize it through `binarypatch_zstd_decompressor_init()`. + * + * This file and the vendored sources next to it sit outside the platform + * directories because both platforms compile them: iOS through the podspec at the + * repository root, Android through externalNativeBuild. One shared copy is what + * keeps the two appliers from drifting apart. + */ + +#ifndef BINARYPATCH_ZSTD_DECOMPRESSOR_H +#define BINARYPATCH_ZSTD_DECOMPRESSOR_H + +#include "libHDiffPatch/HPatch/patch_types.h" + +#ifdef __cplusplus +extern "C" { +#endif + +/* + * The `compressType` string that `hdiffz -c-zstd-...` writes into the patch + * header, and the only value this decompressor accepts. + */ +#define BINARYPATCH_ZSTD_COMPRESS_TYPE "zstd" + +/* + * Fills `out_decompressor` with the zstd plugin implementation. The struct is + * fully overwritten, so it needs no prior initialization, and it holds no state + * that must be released. + */ +void binarypatch_zstd_decompressor_init(hpatch_TDecompress* out_decompressor); + +#ifdef __cplusplus +} +#endif + +#endif /* BINARYPATCH_ZSTD_DECOMPRESSOR_H */ diff --git a/cpp/binarypatch/host/apply_patch_host.c b/cpp/binarypatch/host/apply_patch_host.c new file mode 100644 index 000000000..cb652c013 --- /dev/null +++ b/cpp/binarypatch/host/apply_patch_host.c @@ -0,0 +1,197 @@ +/* + * Host build of the CodePush binary patch applier. + * + * This is the reference implementation of the native applier contract, compiled + * for the development machine so the CLI test suite can verify that the vendored + * sources plus the zstd decompressor restore the exact target bytes. The Android + * and iOS wrappers apply patches the same way. + * + * Memory contract: + * - old (the base bundle) is loaded whole; `hdiffz -m` patches require random + * access to the base data + * - patch is loaded whole; patches stay far smaller than the bundles + * - new (the target bundle) is written sequentially to a file, so the process + * never holds both bundles in memory + * + * Usage: + * apply_patch_host + * + * Exit codes let a caller tell why an apply failed, which is what decides between + * retrying and falling back to a full bundle download: + * 0 success + * 1 bad arguments + * 2 input/output file error + * 3 patch header could not be parsed (corrupt header) + * 4 patch uses a compression codec this applier does not support + * 5 the base file size does not match what the patch expects + * 6 applying the patch failed (corrupt or truncated patch body) + * + * Note: a compressed diff carries no checksum of the base data. Applying a patch + * to a *different* base of the *same* size succeeds and silently produces wrong + * output, so the caller must verify the base and target hashes itself. + */ + +#include +#include +#include + +#include "binarypatch_zstd_decompressor.h" +#include "libHDiffPatch/HPatch/patch.h" + +/* Scratch buffer handed to patch_decompress_with_cache() to reduce stream reads. */ +#define APPLY_CACHE_SIZE (4 * 1024 * 1024) + +typedef struct { + FILE* file; + hpatch_StreamPos_t writtenSize; +} TSequentialFileWriter; + +static hpatch_BOOL _write_sequential(const hpatch_TStreamOutput* stream, + hpatch_StreamPos_t writeToPos, + const unsigned char* data, + const unsigned char* data_end) { + TSequentialFileWriter* self = (TSequentialFileWriter*)stream->streamImport; + const size_t length = (size_t)(data_end - data); + /* patch_decompress_with_cache() only ever appends; anything else is a bug. */ + if (writeToPos != self->writtenSize) { + return hpatch_FALSE; + } + if (fwrite(data, 1, length, self->file) != length) { + return hpatch_FALSE; + } + self->writtenSize += length; + return hpatch_TRUE; +} + +static unsigned char* read_whole_file(const char* path, size_t* out_size) { + FILE* file = fopen(path, "rb"); + long size; + unsigned char* buffer; + if (!file) { + return NULL; + } + if (fseek(file, 0, SEEK_END) != 0) { + fclose(file); + return NULL; + } + size = ftell(file); + if ((size < 0) || (fseek(file, 0, SEEK_SET) != 0)) { + fclose(file); + return NULL; + } + /* malloc(0) may return NULL, which would be indistinguishable from failure. */ + buffer = (unsigned char*)malloc(size ? (size_t)size : 1); + if (!buffer) { + fclose(file); + return NULL; + } + if (size && (fread(buffer, 1, (size_t)size, file) != (size_t)size)) { + free(buffer); + fclose(file); + return NULL; + } + fclose(file); + *out_size = (size_t)size; + return buffer; +} + +int main(int argc, char* argv[]) { + const char* oldPath; + const char* patchPath; + const char* newPath; + unsigned char* oldData = NULL; + unsigned char* patchData = NULL; + unsigned char* cache = NULL; + size_t oldSize = 0; + size_t patchSize = 0; + hpatch_compressedDiffInfo diffInfo; + hpatch_TDecompress decompressor; + hpatch_TStreamInput oldStream; + hpatch_TStreamInput patchStream; + hpatch_TStreamOutput newStream; + TSequentialFileWriter writer; + hpatch_BOOL applied; + int exitCode = 0; + + if (argc != 4) { + fprintf(stderr, "usage: %s \n", argv[0]); + return 1; + } + oldPath = argv[1]; + patchPath = argv[2]; + newPath = argv[3]; + + oldData = read_whole_file(oldPath, &oldSize); + if (!oldData) { + fprintf(stderr, "error: cannot read old file: %s\n", oldPath); + return 2; + } + patchData = read_whole_file(patchPath, &patchSize); + if (!patchData) { + fprintf(stderr, "error: cannot read patch file: %s\n", patchPath); + free(oldData); + return 2; + } + + binarypatch_zstd_decompressor_init(&decompressor); + + if (!getCompressedDiffInfo_mem(&diffInfo, patchData, patchData + patchSize)) { + fprintf(stderr, "error: cannot read patch header\n"); + exitCode = 3; + goto cleanup; + } + if ((strlen(diffInfo.compressType) > 0) && !decompressor.is_can_open(diffInfo.compressType)) { + fprintf(stderr, "error: unsupported compressType: %s\n", diffInfo.compressType); + exitCode = 4; + goto cleanup; + } + if (diffInfo.oldDataSize != (hpatch_StreamPos_t)oldSize) { + fprintf(stderr, "error: old size mismatch: patch expects %llu, file is %llu\n", + (unsigned long long)diffInfo.oldDataSize, (unsigned long long)oldSize); + exitCode = 5; + goto cleanup; + } + + mem_as_hStreamInput(&oldStream, oldData, oldData + oldSize); + mem_as_hStreamInput(&patchStream, patchData, patchData + patchSize); + + writer.file = fopen(newPath, "wb"); + writer.writtenSize = 0; + if (!writer.file) { + fprintf(stderr, "error: cannot open output file: %s\n", newPath); + exitCode = 2; + goto cleanup; + } + memset(&newStream, 0, sizeof(newStream)); + newStream.streamImport = &writer; + newStream.streamSize = diffInfo.newDataSize; + newStream.write = _write_sequential; + + cache = (unsigned char*)malloc(APPLY_CACHE_SIZE); + if (!cache) { + fprintf(stderr, "error: out of memory\n"); + fclose(writer.file); + exitCode = 2; + goto cleanup; + } + + applied = patch_decompress_with_cache(&newStream, &oldStream, &patchStream, &decompressor, + cache, cache + APPLY_CACHE_SIZE); + if (fclose(writer.file) != 0) { + fprintf(stderr, "error: cannot flush output file: %s\n", newPath); + exitCode = 2; + goto cleanup; + } + if (!applied) { + fprintf(stderr, "error: patch apply failed (decError=%d)\n", (int)decompressor.decError); + exitCode = 6; + goto cleanup; + } + printf("ok: wrote %llu bytes to %s\n", (unsigned long long)writer.writtenSize, newPath); + +cleanup: + free(cache); + free(patchData); + free(oldData); + return exitCode; +} diff --git a/cpp/binarypatch/host/build.sh b/cpp/binarypatch/host/build.sh new file mode 100755 index 000000000..148aa0bd4 --- /dev/null +++ b/cpp/binarypatch/host/build.sh @@ -0,0 +1,50 @@ +#!/usr/bin/env bash +# +# Compiles the host build of the binary patch applier from the vendored sources in +# the tree this script lives in. The CLI test suite runs it to prove that the +# committed sources really do restore the target bundle byte for byte. +# +# Only the zstd decompress path of the vendored zstd copy is compiled; the +# compressor and the assembly fast path are not part of the applier. +# +# Usage: +# build.sh [output-binary-path] (default: