From 597967bd6e091ac2648b92d5bd3b70908a0ec741 Mon Sep 17 00:00:00 2001 From: Eric Seidel Date: Fri, 14 Nov 2025 14:22:52 -0800 Subject: [PATCH 01/95] chore: squash all commits on top of 3.35.7 --- .github/workflows/ci.yml | 43 +++ .github/workflows/shorebird_ci.yml | 60 ++++ DEPS | 13 +- engine/src/build/config/compiler/BUILD.gn | 10 +- engine/src/build/toolchain/win/BUILD.gn | 5 +- .../src/build/toolchain/win/tool_wrapper.py | 11 + engine/src/flutter/BUILD.gn | 27 +- engine/src/flutter/build/archives/BUILD.gn | 18 +- .../src/flutter/build/dart/tools/dart_pkg.py | 10 +- engine/src/flutter/common/config.gni | 8 + engine/src/flutter/lib/snapshot/BUILD.gn | 76 +++- engine/src/flutter/runtime/dart_snapshot.cc | 104 ++++-- engine/src/flutter/shell/common/BUILD.gn | 27 ++ engine/src/flutter/shell/common/shell.cc | 10 + .../flutter/shell/common/shorebird/BUILD.gn | 60 ++++ .../shell/common/shorebird/shorebird.cc | 335 ++++++++++++++++++ .../shell/common/shorebird/shorebird.h | 47 +++ .../common/shorebird/shorebird_unittests.cc | 21 ++ .../common/shorebird/snapshots_data_handle.cc | 144 ++++++++ .../common/shorebird/snapshots_data_handle.h | 48 +++ .../snapshots_data_handle_unittests.cc | 177 +++++++++ .../flutter/shell/platform/android/BUILD.gn | 20 +- .../platform/android/android_exports.lst | 12 + .../shell/platform/android/flutter_main.cc | 21 +- .../shell/platform/android/flutter_main.h | 3 + .../flutter/embedding/engine/FlutterJNI.java | 50 ++- .../shell/platform/darwin/ios/BUILD.gn | 10 + .../framework/Source/FlutterDartProject.mm | 33 ++ .../framework/Source/FlutterViewController.mm | 1 + .../shell/platform/darwin/macos/BUILD.gn | 2 + .../macos/framework/Source/FlutterEngine.mm | 58 ++- .../src/flutter/shell/platform/linux/BUILD.gn | 3 + .../flutter/shell/platform/linux/fl_engine.cc | 20 +- .../shell/platform/linux/fl_shorebird.cc | 56 +++ .../shell/platform/linux/fl_shorebird.h | 13 + .../flutter/shell/platform/windows/BUILD.gn | 8 + .../platform/windows/flutter_project_bundle.h | 4 + .../platform/windows/flutter_windows.dll.def | 17 + .../windows/flutter_windows_engine.cc | 150 ++++++++ .../flutter/sky/tools/create_ios_framework.py | 12 +- engine/src/flutter/testing/run_tests.py | 1 + .../flutter_tools/gradle/build.gradle.kts | 1 + .../flutter_tools/lib/src/base/build.dart | 24 +- .../lib/src/build_system/targets/assets.dart | 14 + .../lib/src/build_system/targets/common.dart | 35 ++ .../lib/src/build_system/targets/ios.dart | 6 + .../lib/src/build_system/targets/macos.dart | 6 + packages/flutter_tools/lib/src/cache.dart | 5 + .../lib/src/ios/application_package.dart | 12 + .../lib/src/shorebird/shorebird_yaml.dart | 64 ++++ packages/flutter_tools/lib/src/version.dart | 13 + .../lib/src/windows/build_windows.dart | 1 + .../hermetic/build_macos_test.dart | 8 + .../hermetic/build_windows_test.dart | 126 +++++-- .../build_system/targets/macos_test.dart | 174 +++++++-- .../shorebird/shorebird_yaml_test.dart | 101 ++++++ packages/shorebird_tests/.gitignore | 3 + packages/shorebird_tests/README.md | 2 + .../shorebird_tests/analysis_options.yaml | 2 + packages/shorebird_tests/pubspec.yaml | 16 + .../shorebird_tests/test/android_test.dart | 92 +++++ packages/shorebird_tests/test/base_test.dart | 15 + packages/shorebird_tests/test/ios_test.dart | 91 +++++ .../shorebird_tests/test/shorebird_tests.dart | 281 +++++++++++++++ 64 files changed, 2723 insertions(+), 117 deletions(-) create mode 100644 .github/workflows/ci.yml create mode 100644 .github/workflows/shorebird_ci.yml create mode 100644 engine/src/flutter/shell/common/shorebird/BUILD.gn create mode 100644 engine/src/flutter/shell/common/shorebird/shorebird.cc create mode 100644 engine/src/flutter/shell/common/shorebird/shorebird.h create mode 100644 engine/src/flutter/shell/common/shorebird/shorebird_unittests.cc create mode 100644 engine/src/flutter/shell/common/shorebird/snapshots_data_handle.cc create mode 100644 engine/src/flutter/shell/common/shorebird/snapshots_data_handle.h create mode 100644 engine/src/flutter/shell/common/shorebird/snapshots_data_handle_unittests.cc create mode 100644 engine/src/flutter/shell/platform/linux/fl_shorebird.cc create mode 100644 engine/src/flutter/shell/platform/linux/fl_shorebird.h create mode 100644 engine/src/flutter/shell/platform/windows/flutter_windows.dll.def create mode 100644 packages/flutter_tools/lib/src/shorebird/shorebird_yaml.dart create mode 100644 packages/flutter_tools/test/general.shard/shorebird/shorebird_yaml_test.dart create mode 100644 packages/shorebird_tests/.gitignore create mode 100644 packages/shorebird_tests/README.md create mode 100644 packages/shorebird_tests/analysis_options.yaml create mode 100644 packages/shorebird_tests/pubspec.yaml create mode 100644 packages/shorebird_tests/test/android_test.dart create mode 100644 packages/shorebird_tests/test/base_test.dart create mode 100644 packages/shorebird_tests/test/ios_test.dart create mode 100644 packages/shorebird_tests/test/shorebird_tests.dart diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000000000..ade159ddabcb9 --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,43 @@ +name: ci + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +on: + pull_request: + push: + branches: + - main + +jobs: + test: + strategy: + fail-fast: false + matrix: + os: [ubuntu-latest, macos-latest, windows-latest] + + runs-on: ${{ matrix.os }} + + name: ๐Ÿงช Test + + env: + FLUTTER_STORAGE_BASE_URL: https://download.shorebird.dev + + steps: + - name: ๐Ÿ“š Git Checkout + uses: actions/checkout@v4 + with: + # Fetch all branches and tags to ensure that Flutter can determine its version + fetch-depth: 0 + + - name: ๐ŸŽฏ Setup Dart + uses: dart-lang/setup-dart@v1 + + - name: ๐Ÿ“ฆ Install Dependencies + run: | + dart pub get -C ./dev/bots + dart pub get -C ./dev/tools + + - name: ๐Ÿงช Run Tests + run: dart ./dev/bots/test.dart diff --git a/.github/workflows/shorebird_ci.yml b/.github/workflows/shorebird_ci.yml new file mode 100644 index 0000000000000..ec147b4489255 --- /dev/null +++ b/.github/workflows/shorebird_ci.yml @@ -0,0 +1,60 @@ +name: shorebird_ci + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +on: + pull_request: + push: + branches: + - shorebird/dev + +jobs: + test: + strategy: + fail-fast: false + matrix: + os: [ubuntu-latest, macos-latest, windows-latest] + + runs-on: ${{ matrix.os }} + + name: ๐Ÿฆ Shorebird Test + + # TODO(eseidel): This is also set inside shorebird_tests, unclear if + # if it's needed here as well. + env: + FLUTTER_STORAGE_BASE_URL: https://download.shorebird.dev + + steps: + - name: ๐Ÿ“š Git Checkout + uses: actions/checkout@v4 + with: + # Fetch all branches and tags to ensure that Flutter can determine its version + fetch-depth: 0 + + # TODO(eseidel): shorebird_tests seems to assume flutter is available + # yet it doesn't seem to set it up here? + - name: ๐ŸŽฏ Setup Dart + uses: dart-lang/setup-dart@v1 + + - uses: actions/setup-java@v4 + with: + distribution: "zulu" + java-version: "17" + + - name: ๐Ÿฆ Run Flutter Tools Tests + # TODO(eseidel): Find a nice way to run this on windows. + if: ${{ matrix.os == 'ubuntu-latest' || matrix.os == 'macos-latest' }} + # TODO(eseidel): We can't run all flutter_tools tests until we make + # our changes not throw exceptions on missing shorebird.yaml. + # https://github.com/shorebirdtech/shorebird/issues/2392 + run: ../../bin/flutter test test/general.shard/shorebird + working-directory: packages/flutter_tools + + - name: ๐Ÿฆ Run Shorebird Tests + # TODO(felangel): These tests have a dependency on pkg:flutter_flavorizr which + # requires XCode -- therefore they don't work on Windows. + if: ${{ matrix.os == 'ubuntu-latest' || matrix.os == 'macos-latest' }} + run: dart test + working-directory: packages/shorebird_tests diff --git a/DEPS b/DEPS index ab57c4a8e1242..26bfea672a03e 100644 --- a/DEPS +++ b/DEPS @@ -17,6 +17,14 @@ vars = { 'llvm_git': 'https://llvm.googlesource.com', 'dart_ai_rev': '9c96bfe5f091c9451eff5b59c9bffeb2e806b875', 'skia_revision': '8df24be66531469e576a806749a0202ae26b8d08', + "dart_sdk_revision": "b65ce89c8057d6880e00693a7b0ecd7b9e5f61ca", + "dart_sdk_git": "git@github.com:shorebirdtech/dart-sdk.git", + "updater_git": "https://github.com/shorebirdtech/updater.git", + "updater_rev": "76f005940db57c38b479cee858abc0cfbd12ac28", + + # WARNING: DO NOT EDIT canvaskit_cipd_instance MANUALLY + # See `lib/web_ui/README.md` for how to roll CanvasKit to a new version. + 'canvaskit_cipd_instance': '61aeJQ9laGfEFF_Vlc_u0MCkqB6xb2hAYHRBxKH-Uw4C', # Do not download the Emscripten SDK by default. # This prevents us from downloading the Emscripten toolchain for builds @@ -286,7 +294,7 @@ deps = { Var('dart_git') + '/ai.git' + '@' + Var('dart_ai_rev'), 'engine/src/flutter/third_party/dart': - Var('dart_git') + '/sdk.git' + '@' + Var('dart_revision'), + Var('dart_sdk_git') + '@' + Var('dart_sdk_revision'), # WARNING: Unused Dart dependencies in the list below till "WARNING:" marker are removed automatically - see create_updated_flutter_deps.py. @@ -477,6 +485,9 @@ deps = { 'engine/src/flutter/third_party/ocmock': Var('flutter_git') + '/third_party/ocmock' + '@' + Var('ocmock_rev'), + 'engine/src/flutter/third_party/updater': + Var('updater_git') + '@' + Var('updater_rev'), + 'engine/src/flutter/third_party/libjpeg-turbo/src': Var('flutter_git') + '/third_party/libjpeg-turbo' + '@' + '0fb821f3b2e570b2783a94ccd9a2fb1f4916ae9f', diff --git a/engine/src/build/config/compiler/BUILD.gn b/engine/src/build/config/compiler/BUILD.gn index e06cf625abe8f..23ed57a7a006b 100644 --- a/engine/src/build/config/compiler/BUILD.gn +++ b/engine/src/build/config/compiler/BUILD.gn @@ -443,7 +443,10 @@ config("compiler") { # Example PR: https://github.com/dart-lang/native/pull/1615 ldflags += [ "-Wl,--no-undefined", - "-Wl,--exclude-libs,ALL", + + # TODO: Terrible hack, but otherwise libupdater.a symbols can't + # be exported from libflutter.so, even when added to android_exports.lst. + # "-Wl,--exclude-libs,ALL", # Enable identical code folding to reduce size. "-Wl,--icf=all", @@ -646,6 +649,8 @@ config("runtime_library") { ldflags += [ "-Wl,--warn-shared-textrel" ] libs += [ + # Rust requires libunwind. + "unwind", "c", "dl", "m", @@ -660,6 +665,9 @@ config("runtime_library") { } else if (current_cpu == "x86") { current_android_cpu = "i686" } + # libunwind.a is located in the respective android cpu subdirectories. + # The clang version needs to match the version in the lib_dirs line above. + lib_dirs += [ "${android_toolchain_root}/lib/clang/18/lib/linux/${current_android_cpu}/" ] libs += [ "clang_rt.builtins-${current_android_cpu}-android" ] } diff --git a/engine/src/build/toolchain/win/BUILD.gn b/engine/src/build/toolchain/win/BUILD.gn index 45a98b1ecd64b..b5073fb22277b 100644 --- a/engine/src/build/toolchain/win/BUILD.gn +++ b/engine/src/build/toolchain/win/BUILD.gn @@ -204,8 +204,11 @@ template("msvc_toolchain") { expname = "${dllname}.exp" pdbname = "${dllname}.pdb" rspfile = "${dllname}.rsp" + # .def files are used to export symbols from the DLL. This arg will be + # removed by the python tool wrapper if the .def file doesn't exist. + deffile = "${dllname}.def" - link_command = "\"$python_path\" $tool_wrapper_path link-wrapper $env False link.exe /nologo /IMPLIB:$libname /DLL /OUT:$dllname /PDB:${dllname}.pdb @$rspfile" + link_command = "\"$python_path\" $tool_wrapper_path link-wrapper $env False link.exe /nologo /IMPLIB:$libname /DLL /OUT:$dllname /PDB:${dllname}.pdb /DEF:$deffile @$rspfile" # TODO(brettw) support manifests #manifest_command = "\"$python_path\" $tool_wrapper_path manifest-wrapper $env mt.exe -nologo -manifest $manifests -out:${dllname}.manifest" diff --git a/engine/src/build/toolchain/win/tool_wrapper.py b/engine/src/build/toolchain/win/tool_wrapper.py index b4fc8485ffa17..7866c6122d832 100644 --- a/engine/src/build/toolchain/win/tool_wrapper.py +++ b/engine/src/build/toolchain/win/tool_wrapper.py @@ -121,6 +121,17 @@ def ExecLinkWrapper(self, arch, use_separate_mspdbsrv, *args): self._UseSeparateMspdbsrv(env, args) if sys.platform == 'win32': args = list(args) # *args is a tuple by default, which is read-only. + + # Remove the /DEF arg if not provided. We would ideally be able to do this + # in build\toolchain\win\BUILD.gn, but there doesn't seem to be a way to + # conditionally add args to the command line based on whether a file exists + # or not, so we do it here instead. + def_arg_prefix = "/DEF:" + for arg in args: + if arg.startswith(def_arg_prefix): + def_file = arg[len(def_arg_prefix):] + if not os.path.exists(def_file): + args.remove(arg) args[0] = args[0].replace('/', '\\') # https://docs.python.org/2/library/subprocess.html: # "On Unix with shell=True [...] if args is a sequence, the first item diff --git a/engine/src/flutter/BUILD.gn b/engine/src/flutter/BUILD.gn index 417a4b7848c8b..d0f629fc96f1a 100644 --- a/engine/src/flutter/BUILD.gn +++ b/engine/src/flutter/BUILD.gn @@ -119,6 +119,9 @@ group("flutter") { # path_ops "//flutter/tools/path_ops", + + # Built alongside gen_snapshot arm64 targets. + "$dart_src/runtime/bin:analyze_snapshot", ] if (host_os == "linux" || host_os == "mac") { @@ -128,13 +131,6 @@ group("flutter") { ] } - if (host_os == "linux") { - public_deps += [ - # Built alongside gen_snapshot for 64 bit targets - "$dart_src/runtime/bin:analyze_snapshot", - ] - } - if (full_dart_sdk) { public_deps += [ "//flutter/web_sdk" ] } @@ -217,6 +213,7 @@ group("unittests") { "//flutter/shell/common:shell_unittests", "//flutter/shell/geometry:geometry_unittests", "//flutter/shell/gpu:gpu_surface_unittests", + "//flutter/shell/common/shorebird:shorebird_unittests", "//flutter/shell/platform/embedder:embedder_a11y_unittests", "//flutter/shell/platform/embedder:embedder_proctable_unittests", "//flutter/shell/platform/embedder:embedder_unittests", @@ -350,3 +347,19 @@ if (host_os == "win") { outputs = [ "$root_build_dir/gen_snapshot/gen_snapshot.exe" ] } } + +# A top-level target for analyze_snapshot, modeled after the gen_snapshot +# target above. +if (host_os == "win") { + _analyze_snapshot_target = + "$dart_src/runtime/bin:analyze_snapshot($host_toolchain)" + + copy("analyze_snapshot") { + deps = [ _analyze_snapshot_target ] + + analyze_snapshot_out_dir = + get_label_info(_analyze_snapshot_target, "root_out_dir") + sources = [ "$analyze_snapshot_out_dir/analyze_snapshot.exe" ] + outputs = [ "$root_build_dir/analyze_snapshot/analyze_snapshot.exe" ] + } +} diff --git a/engine/src/flutter/build/archives/BUILD.gn b/engine/src/flutter/build/archives/BUILD.gn index 8b4b8f01ab767..6b9f940756a12 100644 --- a/engine/src/flutter/build/archives/BUILD.gn +++ b/engine/src/flutter/build/archives/BUILD.gn @@ -59,6 +59,7 @@ generated_file("artifacts_without_entitlement_config") { if (build_engine_artifacts) { zip_bundle("artifacts") { deps = [ + "$dart_src/runtime/bin:analyze_snapshot", "$dart_src/runtime/bin:gen_snapshot", "//flutter/flutter_frontend_server:frontend_server", "//flutter/impeller/compiler:impellerc", @@ -159,6 +160,10 @@ if (build_engine_artifacts) { ":artifacts_without_entitlement_config", ] files += [ + { + source = "$root_out_dir/analyze_snapshot$exe" + destination = "analyze_snapshot$exe" + }, { source = "$target_gen_dir/entitlements.txt" destination = "entitlements.txt" @@ -388,14 +393,25 @@ if (is_mac) { } if (host_os == "win") { + # This rule archives both gen_snapshot *and* analyze_snapshot. The name is + # misleading. We (shorebird) have updated this rule to include + # analyze_snapshot but did not update the name because it is referenced + # elsewhere in the tooling. zip_bundle("archive_win_gen_snapshot") { - deps = [ "//flutter:gen_snapshot" ] + deps = [ + "//flutter:analyze_snapshot", + "//flutter:gen_snapshot", + ] output = "$full_target_platform_name-$flutter_runtime_mode/windows-x64.zip" files = [ { source = "$root_out_dir/gen_snapshot/gen_snapshot.exe" destination = "gen_snapshot.exe" }, + { + source = "$root_out_dir/analyze_snapshot/analyze_snapshot.exe" + destination = "analyze_snapshot.exe" + }, ] } } diff --git a/engine/src/flutter/build/dart/tools/dart_pkg.py b/engine/src/flutter/build/dart/tools/dart_pkg.py index c60e3e02431e4..5b3e0a8b5f3ee 100755 --- a/engine/src/flutter/build/dart/tools/dart_pkg.py +++ b/engine/src/flutter/build/dart/tools/dart_pkg.py @@ -163,7 +163,10 @@ def main(): for source in args.package_sources: relative_source = os.path.relpath(source, common_source_prefix) target = os.path.join(target_dir, relative_source) - copy(source, target) + try: + copy(source, target) + except shutil.SameFileError: + pass # Copy sdk-ext sources into pkg directory sdk_ext_dir = os.path.join(target_dir, 'sdk_ext') @@ -179,7 +182,10 @@ def main(): for source in args.sdk_ext_files: relative_source = os.path.relpath(source, common_source_prefix) target = os.path.join(sdk_ext_dir, relative_source) - copy(source, target) + try: + copy(source, target) + except shutil.SameFileError: + pass # Write stamp file. with open(args.stamp_file, 'w'): diff --git a/engine/src/flutter/common/config.gni b/engine/src/flutter/common/config.gni index 318d305bd37fc..35ec1c7e058dc 100644 --- a/engine/src/flutter/common/config.gni +++ b/engine/src/flutter/common/config.gni @@ -67,6 +67,14 @@ if (slimpeller) { feature_defines_list += [ "SLIMPELLER=1" ] } +if (is_android || is_ios || is_linux || is_mac || is_win) { + feature_defines_list += [ "SHOREBIRD_PLATFORM_SUPPORTED=1" ] +} + +if (is_ios) { + feature_defines_list += [ "SHOREBIRD_USE_INTERPRETER=1" ] +} + if (is_ios || is_mac) { flutter_cflags_objc = [ "-Werror=overriding-method-mismatch", diff --git a/engine/src/flutter/lib/snapshot/BUILD.gn b/engine/src/flutter/lib/snapshot/BUILD.gn index d08c7d5fd3a48..7b935b4fedac1 100644 --- a/engine/src/flutter/lib/snapshot/BUILD.gn +++ b/engine/src/flutter/lib/snapshot/BUILD.gn @@ -35,7 +35,10 @@ group("generate_snapshot_bins") { if (host_os == "mac" && (target_os == "mac" || target_os == "ios" || target_os == "android")) { # For macOS target builds: needed for both target CPUs (arm64, x64). - public_deps += [ ":create_macos_gen_snapshots" ] + public_deps += [ + ":create_macos_analyze_snapshots", + ":create_macos_gen_snapshots", + ] } else if (host_os == "mac" && (target_cpu == "arm" || target_cpu == "arm64")) { # For iOS, Android target builds: all AOT target CPUs are arm/arm64. @@ -46,9 +49,11 @@ group("generate_snapshot_bins") { public_deps = [ "$dart_src/runtime/bin:gen_snapshot($host_toolchain)" ] } - # Build analyze_snapshot for 64-bit target CPUs. - if (host_os == "linux" && (target_cpu == "x64" || target_cpu == "arm64" || - target_cpu == "riscv64")) { + # Build analyze_snapshot for 64-bit target CPUs on linux. + # Or always targeting arm64 for Shorebird builds. + if ((host_os == "linux" && + (target_cpu == "x64" || target_cpu == "riscv64")) || + target_cpu == "arm64") { public_deps += [ "$dart_src/runtime/bin:analyze_snapshot($host_toolchain)" ] } } @@ -254,6 +259,69 @@ if (host_os == "mac" && ":create_macos_gen_snapshot_x64${gen_snapshot_suffix}", ] } + + # Added by shorebird. + # analyze_snapshot targets below were copied from the gen_snapshot targets + # above to allow us to include analyze_snapshot in the artifacts generated + # for create_ios_framework.py. + template("build_mac_analyze_snapshot") { + assert(defined(invoker.host_arch)) + host_cpu = invoker.host_arch + + build_toolchain = "//build/toolchain/mac:clang_$host_cpu" + analyze_snapshot_target_name = "analyze_snapshot" + + # At this point, the gen_snapshot equivalent changes + # gen_ snapshot_target_name to "gen_snapshot_host_targeting_host". There is + # no equivalent for analyze_snapshot, so we don't do that here. + # + # It's unclear whether we need to do so now, but we didn't previously, so + # we're not doing it now until we have a reason to. + + analyze_snapshot_target = + "$dart_src/runtime/bin:$analyze_snapshot_target_name($build_toolchain)" + + copy(target_name) { + # The toolchain-specific output directory. For cross-compiles, this is a + # clang-x64 or clang-arm64 subdirectory of the top-level build directory. + output_dir = get_label_info(analyze_snapshot_target, "root_out_dir") + + sources = [ "${output_dir}/${analyze_snapshot_target_name}" ] + outputs = [ + "${root_out_dir}/artifacts_$host_cpu/analyze_snapshot_${target_cpu}", + ] + deps = [ analyze_snapshot_target ] + } + } + + build_mac_analyze_snapshot( + "create_macos_analyze_snapshot_arm64_${target_cpu}") { + host_arch = "arm64" + } + + build_mac_analyze_snapshot( + "create_macos_analyze_snapshot_x64_${target_cpu}") { + host_arch = "x64" + } + + action("create_macos_analyze_snapshots") { + script = "//flutter/sky/tools/create_macos_binary.py" + outputs = [ "${root_out_dir}/analyze_snapshot_${target_cpu}" ] + args = [ + "--in-arm64", + rebase_path( + "${root_out_dir}/artifacts_arm64/analyze_snapshot_${target_cpu}"), + "--in-x64", + rebase_path( + "${root_out_dir}/artifacts_x64/analyze_snapshot_${target_cpu}"), + "--out", + rebase_path("${root_out_dir}/analyze_snapshot_${target_cpu}"), + ] + deps = [ + ":create_macos_analyze_snapshot_arm64_${target_cpu}", + ":create_macos_analyze_snapshot_x64_${target_cpu}", + ] + } } source_set("snapshot") { diff --git a/engine/src/flutter/runtime/dart_snapshot.cc b/engine/src/flutter/runtime/dart_snapshot.cc index 293ed44cc5fe2..6188180860f5f 100644 --- a/engine/src/flutter/runtime/dart_snapshot.cc +++ b/engine/src/flutter/runtime/dart_snapshot.cc @@ -55,33 +55,93 @@ static std::shared_ptr SearchMapping( const std::vector& native_library_paths, const char* native_library_symbol_name, bool is_executable) { - // Ask the embedder. There is no fallback as we expect the embedders (via - // their embedding APIs) to just specify the mappings directly. - if (embedder_mapping_callback) { - // Note that mapping will be nullptr if the mapping callback returns an - // invalid mapping. If all the other methods for resolving the data also - // fail, the engine will stop with accompanying error logs. - if (auto mapping = embedder_mapping_callback()) { - return mapping; +#if SHOREBIRD_USE_INTERPRETER + // Detect when we're trying to load a Shorebird patch. + auto patch_path = native_library_path.front(); + bool is_patch = patch_path.find(".vmcode") != std::string::npos; + if (is_patch) { + // We use this terrible hack to load in the patch and then extract the + // symbols from it when the path is not App.framework/App but rather + // foo.vmcode, etc. We read the symbols into static variables, but then I + // believe we need to hold onto the ELF itself, otherwise the symbols + // become invalid. + // "leaked_elf" is meant to indicate that we're not freeing the ELF. + static Dart_LoadedElf* leaked_elf = nullptr; + // The VM Snapshot is identical for all binaries produced by a given version + // of Dart. Our linker checks this and will fail to link if ever the VM + // snapshot changes. + const uint8_t* ignored_vm_data = nullptr; + const uint8_t* ignored_vm_instrs = nullptr; + static const uint8_t* isolate_data = nullptr; + static const uint8_t* isolate_instrs = nullptr; + if (leaked_elf == nullptr) { + const char* error = nullptr; + // vmcode files are elf files prefixed with a shorebird linker header. + auto elf_mapping = GetFileMapping(patch_path, false /* executable */); + int elf_file_offset = Shorebird_ReadLinkHeader(elf_mapping->GetMapping(), + elf_mapping->GetSize()); + + leaked_elf = Dart_LoadELF(patch_path.c_str(), elf_file_offset, &error, + &ignored_vm_data, &ignored_vm_instrs, + &isolate_data, &isolate_instrs, + /* load as read-only, not rx */ false); + if (leaked_elf != nullptr) { + FML_LOG(INFO) << "Loaded ELF"; + } else { + FML_LOG(FATAL) << "Failed to load ELF at " << patch_path + << " error: " << error; + abort(); + } } - } - // Attempt to open file at path specified. - if (!file_path.empty()) { - if (auto file_mapping = GetFileMapping(file_path, is_executable)) { - return file_mapping; + FML_LOG(INFO) << "Loading symbol from ELF " << native_library_symbol_name; + + if (native_library_symbol_name == DartSnapshot::kIsolateDataSymbol) { + return std::make_unique(isolate_data, 0, + nullptr, true); + } else if (native_library_symbol_name == + DartSnapshot::kIsolateInstructionsSymbol) { + return std::make_unique(isolate_instrs, 0, + nullptr, true); + } + // Fall through to normal lookups for VM data and instructions. + // This fallthrough depends on the fact that NativeLibrary below can't + // read the ELF out of our .vmcode files. + } else { + // Only try to open the file if we're not loading a patch. +#endif + + // Ask the embedder. There is no fallback as we expect the embedders (via + // their embedding APIs) to just specify the mappings directly. + if (embedder_mapping_callback) { + // Note that mapping will be nullptr if the mapping callback returns an + // invalid mapping. If all the other methods for resolving the data also + // fail, the engine will stop with accompanying error logs. + if (auto mapping = embedder_mapping_callback()) { + return mapping; + } } - } - // Look in application specified native library if specified. - for (const std::string& path : native_library_paths) { - auto native_library = fml::NativeLibrary::Create(path.c_str()); - auto symbol_mapping = std::make_unique( - native_library, native_library_symbol_name); - if (symbol_mapping->GetMapping() != nullptr) { - return symbol_mapping; + // Attempt to open file at path specified. + if (!file_path.empty()) { + if (auto file_mapping = GetFileMapping(file_path, is_executable)) { + return file_mapping; + } } - } + + // Look in application specified native library if specified. + for (const std::string& path : native_library_paths) { + auto native_library = fml::NativeLibrary::Create(path.c_str()); + auto symbol_mapping = std::make_unique( + native_library, native_library_symbol_name); + if (symbol_mapping->GetMapping() != nullptr) { + return symbol_mapping; + } + } + +#if SHOREBIRD_USE_INTERPRETER + } // !is_patch +#endif // Look inside the currently loaded process. { diff --git a/engine/src/flutter/shell/common/BUILD.gn b/engine/src/flutter/shell/common/BUILD.gn index e1aba738f17be..bbe42f5a987f8 100644 --- a/engine/src/flutter/shell/common/BUILD.gn +++ b/engine/src/flutter/shell/common/BUILD.gn @@ -158,6 +158,8 @@ source_set("common") { "//flutter/skia", ] + include_dirs = [ "//flutter/updater" ] + if (impeller_supports_rendering) { sources += [ "snapshot_controller_impeller.cc", @@ -166,6 +168,31 @@ source_set("common") { deps += [ "//flutter/impeller" ] } + + # Needed to compile flutter_tester for macOS. + if (host_os == "mac" && target_os == "mac") { + if (target_cpu == "arm64") { + libs = [ "//flutter/third_party/updater/target/aarch64-apple-darwin/release/libupdater.a" ] + } else if (target_cpu == "x64") { + libs = [ "//flutter/third_party/updater/target/x86_64-apple-darwin/release/libupdater.a" ] + } + } + + # Needed to compile flutter_tester for Windows. + if (host_os == "win" && target_os == "win") { + if (target_cpu == "x64") { + libs = [ + "userenv.lib", + "//flutter/third_party/updater/target/x86_64-pc-windows-msvc/release/updater.lib", + ] + } + } + + if (host_os == "linux" && target_os == "linux") { + if (target_cpu == "x64") { + libs = [ "//flutter/third_party/updater/target/x86_64-unknown-linux-gnu/release/libupdater.a" ] + } + } } # These are in their own source_set to avoid a dependency cycle with //common/graphics diff --git a/engine/src/flutter/shell/common/shell.cc b/engine/src/flutter/shell/common/shell.cc index 526d2a7984125..01d7595e2647d 100644 --- a/engine/src/flutter/shell/common/shell.cc +++ b/engine/src/flutter/shell/common/shell.cc @@ -47,6 +47,8 @@ #include "third_party/skia/include/core/SkGraphics.h" #include "third_party/tonic/common/log.h" +#include "third_party/updater/library/include/updater.h" + namespace flutter { constexpr char kSkiaChannel[] = "flutter/skia"; @@ -522,6 +524,14 @@ Shell::Shell(DartVMRef vm, is_gpu_disabled_sync_switch_(new fml::SyncSwitch(is_gpu_disabled)), weak_factory_gpu_(nullptr), weak_factory_(this) { + // FIXME: This is probably the wrong place to hook into. +#if SHOREBIRD_PLATFORM_SUPPORTED + if (!vm_) { + shorebird_report_launch_failure(); + } else { + shorebird_report_launch_success(); + } +#endif FML_CHECK(!settings.enable_software_rendering || !settings.enable_impeller) << "Software rendering is incompatible with Impeller."; if (!settings.enable_impeller && settings.warn_on_impeller_opt_out) { diff --git a/engine/src/flutter/shell/common/shorebird/BUILD.gn b/engine/src/flutter/shell/common/shorebird/BUILD.gn new file mode 100644 index 0000000000000..2c7def6991542 --- /dev/null +++ b/engine/src/flutter/shell/common/shorebird/BUILD.gn @@ -0,0 +1,60 @@ +import("//flutter/common/config.gni") +import("//flutter/testing/testing.gni") + +source_set("snapshots_data_handle") { + sources = [ + "snapshots_data_handle.cc", + "snapshots_data_handle.h", + ] + + deps = [ + "//flutter/fml", + "//flutter/runtime", + "//flutter/runtime:libdart", + "//flutter/shell/common", + ] +} + +source_set("shorebird") { + sources = [ + "shorebird.cc", + "shorebird.h", + ] + + deps = [ + ":snapshots_data_handle", + "//flutter/fml", + "//flutter/runtime", + "//flutter/runtime:libdart", + "//flutter/shell/common", + "//flutter/shell/platform/embedder:embedder_headers", + ] + + include_dirs = [ "//flutter/updater" ] +} + +if (enable_unittests) { + test_fixtures("shorebird_fixtures") { + fixtures = [] + } + + executable("shorebird_unittests") { + testonly = true + + sources = [ + "shorebird_unittests.cc", + "snapshots_data_handle_unittests.cc", + ] + + # This only includes snapshots_data_handle and not shorebird because + # shorebird fails to link due to a missing updater lib. + deps = [ + ":shorebird", + ":shorebird_fixtures", + ":snapshots_data_handle", + "//flutter/runtime", + "//flutter/testing", + "//flutter/testing:fixture_test", + ] + } +} diff --git a/engine/src/flutter/shell/common/shorebird/shorebird.cc b/engine/src/flutter/shell/common/shorebird/shorebird.cc new file mode 100644 index 0000000000000..a2b2f382820a1 --- /dev/null +++ b/engine/src/flutter/shell/common/shorebird/shorebird.cc @@ -0,0 +1,335 @@ + +#include "flutter/shell/common/shorebird/shorebird.h" + +#include +#include +#include +#include +#include + +#include "flutter/fml/command_line.h" +#include "flutter/fml/file.h" +#include "flutter/fml/macros.h" +#include "flutter/fml/mapping.h" +#include "flutter/fml/message_loop.h" +#include "flutter/fml/native_library.h" +#include "flutter/fml/paths.h" +#include "flutter/lib/ui/plugins/callback_cache.h" +#include "flutter/runtime/dart_snapshot.h" +#include "flutter/runtime/dart_vm.h" +#include "flutter/shell/common/shell.h" +#include "flutter/shell/common/shorebird/snapshots_data_handle.h" +#include "flutter/shell/common/switches.h" +#include "fml/logging.h" +#include "shell/platform/embedder/embedder.h" +#include "third_party/dart/runtime/include/dart_tools_api.h" + +#include "third_party/updater/library/include/updater.h" + +// Namespaced to avoid Google style warnings. +namespace flutter { + +// Old Android versions (e.g. the v16 ndk Flutter uses) don't always include a +// getauxval symbol, but the Rust ring crate assumes it exists: +// https://github.com/briansmith/ring/blob/fa25bf3a7403c9fe6458cb87bd8427be41225ca2/src/cpu/arm.rs#L22 +// It uses it to determine if the CPU supports AES instructions. +// Making this a weak symbol allows the linker to use a real version instead +// if it can find one. +// BoringSSL just reads from procfs instead, which is what we would do if +// we needed to implement this ourselves. Implementation looks straightforward: +// https://lwn.net/Articles/519085/ +// https://github.com/google/boringssl/blob/6ab4f0ae7f2db96d240eb61a5a8b4724e5a09b2f/crypto/cpu_arm_linux.c +#if defined(__ANDROID__) && defined(__arm__) +extern "C" __attribute__((weak)) unsigned long getauxval(unsigned long type) { + return 0; +} +#endif + +// TODO(eseidel): I believe we need to leak these or we'll sometimes crash +// when using the base snapshot in mixed mode. This likely will not play +// nicely with multi-engine support and will need to be refactored. +static fml::RefPtr vm_snapshot; +static fml::RefPtr isolate_snapshot; + +void SetBaseSnapshot(Settings& settings) { + // These mappings happen to be to static data in the App.framework, but + // we still need to seem to hold onto the DartSnapshot objects to keep + // the mappings alive. + vm_snapshot = DartSnapshot::VMSnapshotFromSettings(settings); + isolate_snapshot = DartSnapshot::IsolateSnapshotFromSettings(settings); + Shorebird_SetBaseSnapshots(isolate_snapshot->GetDataMapping(), + isolate_snapshot->GetInstructionsMapping(), + vm_snapshot->GetDataMapping(), + vm_snapshot->GetInstructionsMapping()); +} + +class FileCallbacksImpl { + public: + static void* Open(); + static uintptr_t Read(void* file, uint8_t* buffer, uintptr_t length); + static int64_t Seek(void* file, int64_t offset, int32_t whence); + static void Close(void* file); +}; + +FileCallbacks ShorebirdFileCallbacks() { + return { + .open = FileCallbacksImpl::Open, + .read = FileCallbacksImpl::Read, + .seek = FileCallbacksImpl::Seek, + .close = FileCallbacksImpl::Close, + }; +} + +// Given the contents of a yaml file, return the given value if it exists, +// otherwise return an empty string. +// Does not support nested keys. +std::string GetValueFromYaml(const std::string& yaml, const std::string& key) { + std::stringstream ss(yaml); + std::string line; + std::string prefix = key + ":"; + while (std::getline(ss, line, '\n')) { + if (line.find(prefix) != std::string::npos) { + auto ret = line.substr(line.find(prefix) + prefix.size()); + + // Remove leading and trailing spaces + while (!ret.empty() && std::isspace(ret.front())) { + ret.erase(0, 1); + } + while (!ret.empty() && std::isspace(ret.back())) { + ret.pop_back(); + } + return ret; + } + } + return ""; +} + +// FIXME: consolidate this with the other ConfigureShorebird +bool ConfigureShorebird(const ShorebirdConfigArgs& args, + std::string& patch_path) { + patch_path = fml::PathToUtf8(args.release_app_library_path); + auto shorebird_updater_dir_name = "shorebird_updater"; + + // Parse app id from shorebird.yaml + std::string app_id = GetValueFromYaml(args.shorebird_yaml, "app_id"); + if (app_id.empty()) { + FML_LOG(ERROR) << "Shorebird updater: appid not found in shorebird.yaml"; + return false; + } + + auto code_cache_dir = fml::paths::JoinPaths( + {std::move(args.code_cache_path), shorebird_updater_dir_name, app_id}); + auto app_storage_dir = fml::paths::JoinPaths( + {std::move(args.app_storage_path), shorebird_updater_dir_name, app_id}); + + fml::CreateDirectory(fml::paths::GetCachesDirectory(), + {shorebird_updater_dir_name}, + fml::FilePermission::kReadWrite); + + bool init_result; + // Using a block to make AppParameters lifetime explicit. + { + AppParameters app_parameters; + // Combine version and version_code into a single string. + // We could also pass these separately through to the updater if needed. + auto release_version = args.release_version.version; + if (!args.release_version.build_number.empty()) { + release_version += "+" + args.release_version.build_number; + } + + app_parameters.release_version = release_version.c_str(); + app_parameters.code_cache_dir = code_cache_dir.c_str(); + app_parameters.app_storage_dir = app_storage_dir.c_str(); + + // https://stackoverflow.com/questions/26032039/convert-vectorstring-into-char-c + std::vector c_paths{}; + c_paths.push_back(args.release_app_library_path.c_str()); + // Do not modify application_library_path or c_strings will invalidate. + + app_parameters.original_libapp_paths = c_paths.data(); + app_parameters.original_libapp_paths_size = c_paths.size(); + + // shorebird_init copies from app_parameters and shorebirdYaml. + init_result = shorebird_init(&app_parameters, ShorebirdFileCallbacks(), + args.shorebird_yaml.c_str()); + } + + // We've decided not to support synchronous updates on launch for now. + // It's a terrible user experience (having the app hang on launch) and + // instead we will provide examples of how to build a custom update UI + // within Dart, including updating as part of login, etc. + // https://github.com/shorebirdtech/shorebird/issues/950 + + FML_LOG(INFO) << "Checking for active patch"; + shorebird_validate_next_boot_patch(); + char* c_active_path = shorebird_next_boot_patch_path(); + if (c_active_path != NULL) { + patch_path = c_active_path; + shorebird_free_string(c_active_path); + FML_LOG(INFO) << "Shorebird updater: patch path: " << patch_path; + } else { + FML_LOG(INFO) << "Shorebird updater: no active patch."; + } + + // We are careful only to report a launch start in the case where it's the + // first time we've configured shorebird this process. Otherwise we could end + // up in a case where we report a launch start, but never a completion (e.g. + // from package:flutter_work_manager which sometimes creates a FlutterEngine + // (and thus configures shorebird) but never runs it. The proper fix for this + // is probably to move the launch_start() call to be later in the lifecycle + // (when the snapshot is loaded and run, rather than when FlutterEngine is + // initialized). This "hack" will still have a problem where FlutterEngine is + // initialized but never run before the app is quit, could still cause us to + // suddenly mark-bad a patch that was never actually attempted to launch. + if (!init_result) { + return false; + } + + // Once start_update_thread is called, the next_boot_patch* functions may + // change their return values if the shorebird_report_launch_failed + // function is called. + shorebird_report_launch_start(); + + if (shorebird_should_auto_update()) { + FML_LOG(INFO) << "Starting Shorebird update"; + shorebird_start_update_thread(); + } else { + FML_LOG(INFO) + << "Shorebird auto_update disabled, not checking for updates."; + } + + return true; +} + +void ConfigureShorebird(std::string code_cache_path, + std::string app_storage_path, + Settings& settings, + const std::string& shorebird_yaml, + const std::string& version, + const std::string& version_code) { + // If you are crashing here, you probably are running Shorebird in a Debug + // config, where the AOT snapshot won't be linked into the process, and thus + // lookups will fail. Change your Scheme to Release to fix: + // https://github.com/flutter/flutter/wiki/Debugging-the-engine#debugging-ios-builds-with-xcode + FML_CHECK(DartSnapshot::VMSnapshotFromSettings(settings)) + << "XCode Scheme must be set to Release to use Shorebird"; + + auto shorebird_updater_dir_name = "shorebird_updater"; + + auto code_cache_dir = fml::paths::JoinPaths( + {std::move(code_cache_path), shorebird_updater_dir_name}); + auto app_storage_dir = fml::paths::JoinPaths( + {std::move(app_storage_path), shorebird_updater_dir_name}); + + fml::CreateDirectory(fml::paths::GetCachesDirectory(), + {shorebird_updater_dir_name}, + fml::FilePermission::kReadWrite); + + bool init_result; + // Using a block to make AppParameters lifetime explicit. + { + AppParameters app_parameters; + // Combine version and version_code into a single string. + // We could also pass these separately through to the updater if needed. + auto release_version = version + "+" + version_code; + app_parameters.release_version = release_version.c_str(); + app_parameters.code_cache_dir = code_cache_dir.c_str(); + app_parameters.app_storage_dir = app_storage_dir.c_str(); + + // https://stackoverflow.com/questions/26032039/convert-vectorstring-into-char-c + std::vector c_paths{}; + for (const auto& string : settings.application_library_path) { + c_paths.push_back(string.c_str()); + } + // Do not modify application_library_path or c_strings will invalidate. + + app_parameters.original_libapp_paths = c_paths.data(); + app_parameters.original_libapp_paths_size = c_paths.size(); + + // shorebird_init copies from app_parameters and shorebirdYaml. + init_result = shorebird_init(&app_parameters, ShorebirdFileCallbacks(), + shorebird_yaml.c_str()); + } + + // We've decided not to support synchronous updates on launch for now. + // It's a terrible user experience (having the app hang on launch) and + // instead we will provide examples of how to build a custom update UI + // within Dart, including updating as part of login, etc. + // https://github.com/shorebirdtech/shorebird/issues/950 + + // We only set the base snapshot on iOS for now. +#if SHOREBIRD_USE_INTERPRETER + SetBaseSnapshot(settings); +#endif + + shorebird_validate_next_boot_patch(); + char* c_active_path = shorebird_next_boot_patch_path(); + if (c_active_path != NULL) { + std::string active_path = c_active_path; + shorebird_free_string(c_active_path); + FML_LOG(INFO) << "Shorebird updater: active path: " << active_path; + +#if SHOREBIRD_USE_INTERPRETER + // On iOS we add the patch to the front of the list instead of clearing + // the list, to allow dart_snapshot.cc to still find the base snapshot + // for the vm isolate. + settings.application_library_path.insert( + settings.application_library_path.begin(), active_path); +#else + settings.application_library_path.clear(); + settings.application_library_path.emplace_back(active_path); +#endif + } else { + FML_LOG(INFO) << "Shorebird updater: no active patch."; + } + + // We are careful only to report a launch start in the case where it's the + // first time we've configured shorebird this process. Otherwise we could end + // up in a case where we report a launch start, but never a completion (e.g. + // from package:flutter_work_manager which sometimes creates a FlutterEngine + // (and thus configures shorebird) but never runs it. The proper fix for this + // is probably to move the launch_start() call to be later in the lifecycle + // (when the snapshot is loaded and run, rather than when FlutterEngine is + // initialized). This "hack" will still have a problem where FlutterEngine is + // initialized but never run before the app is quit, could still cause us to + // suddenly mark-bad a patch that was never actually attempted to launch. + if (!init_result) { + return; + } + + // Once start_update_thread is called, the next_boot_patch* functions may + // change their return values if the shorebird_report_launch_failed + // function is called. + shorebird_report_launch_start(); + + if (shorebird_should_auto_update()) { + FML_LOG(INFO) << "Starting Shorebird update"; + shorebird_start_update_thread(); + } else { + FML_LOG(INFO) + << "Shorebird auto_update disabled, not checking for updates."; + } +} + +void* FileCallbacksImpl::Open() { + return SnapshotsDataHandle::createForSnapshots(*vm_snapshot, + *isolate_snapshot) + .release(); +} + +uintptr_t FileCallbacksImpl::Read(void* file, + uint8_t* buffer, + uintptr_t length) { + return reinterpret_cast(file)->Read(buffer, length); +} + +int64_t FileCallbacksImpl::Seek(void* file, int64_t offset, int32_t whence) { + // Currently we only support blob handles. + return reinterpret_cast(file)->Seek(offset, whence); +} + +void FileCallbacksImpl::Close(void* file) { + delete reinterpret_cast(file); +} + +} // namespace flutter \ No newline at end of file diff --git a/engine/src/flutter/shell/common/shorebird/shorebird.h b/engine/src/flutter/shell/common/shorebird/shorebird.h new file mode 100644 index 0000000000000..c6873c5014a00 --- /dev/null +++ b/engine/src/flutter/shell/common/shorebird/shorebird.h @@ -0,0 +1,47 @@ +#ifndef FLUTTER_SHELL_COMMON_SHOREBIRD_SHOREBIRD_H_ +#define FLUTTER_SHELL_COMMON_SHOREBIRD_SHOREBIRD_H_ + +#include "flutter/common/settings.h" +#include "shell/platform/embedder/embedder.h" + +namespace flutter { + +struct ReleaseVersion { + std::string version; + std::string build_number; +}; + +struct ShorebirdConfigArgs { + std::string code_cache_path; + std::string app_storage_path; + std::string release_app_library_path; + std::string shorebird_yaml; + ReleaseVersion release_version; + + ShorebirdConfigArgs(std::string code_cache_path, + std::string app_storage_path, + std::string release_app_library_path, + std::string shorebird_yaml, + ReleaseVersion release_version) + : code_cache_path(code_cache_path), + app_storage_path(app_storage_path), + release_app_library_path(release_app_library_path), + shorebird_yaml(shorebird_yaml), + release_version(release_version) {} +}; + +bool ConfigureShorebird(const ShorebirdConfigArgs& args, + std::string& patch_path); + +void ConfigureShorebird(std::string code_cache_path, + std::string app_storage_path, + Settings& settings, + const std::string& shorebird_yaml, + const std::string& version, + const std::string& version_code); + +std::string GetValueFromYaml(const std::string& yaml, const std::string& key); + +} // namespace flutter + +#endif // FLUTTER_SHELL_COMMON_SHOREBIRD_SHOREBIRD_H_ diff --git a/engine/src/flutter/shell/common/shorebird/shorebird_unittests.cc b/engine/src/flutter/shell/common/shorebird/shorebird_unittests.cc new file mode 100644 index 0000000000000..7c108ac1e6231 --- /dev/null +++ b/engine/src/flutter/shell/common/shorebird/shorebird_unittests.cc @@ -0,0 +1,21 @@ +#include "flutter/shell/common/shorebird/shorebird.h" + +#include "gtest/gtest.h" + +namespace flutter { +namespace testing { +TEST(Shorebird, GetValueFromYamlValueExists) { + std::string yaml = "appid: com.example.app\nversion: 1.0.0\n"; + std::string key = "appid"; + std::string value = GetValueFromYaml(yaml, key); + EXPECT_EQ(value, "com.example.app"); +} + +TEST(Shorebird, GetValueFromYamlValueDoesNotExist) { + std::string yaml = "appid: com.example.app\nversion: 1.0.0\n"; + std::string key = "appid2"; + std::string value = GetValueFromYaml(yaml, key); + EXPECT_EQ(value, ""); +} +} // namespace testing +} // namespace flutter \ No newline at end of file diff --git a/engine/src/flutter/shell/common/shorebird/snapshots_data_handle.cc b/engine/src/flutter/shell/common/shorebird/snapshots_data_handle.cc new file mode 100644 index 0000000000000..0c6c5a45450a1 --- /dev/null +++ b/engine/src/flutter/shell/common/shorebird/snapshots_data_handle.cc @@ -0,0 +1,144 @@ +#include "flutter/shell/common/shorebird/snapshots_data_handle.h" + +#include "third_party/dart/runtime/include/dart_native_api.h" + +namespace flutter { + +static std::unique_ptr DataMapping(const DartSnapshot& snapshot) { + auto ptr = snapshot.GetDataMapping(); + return std::make_unique(ptr, + Dart_SnapshotDataSize(ptr)); +} + +static std::unique_ptr InstructionsMapping( + const DartSnapshot& snapshot) { + auto ptr = snapshot.GetInstructionsMapping(); + return std::make_unique(ptr, + Dart_SnapshotInstrSize(ptr)); +} + +// The size of the snapshot data is the sum of the sizes of the blobs. +size_t SnapshotsDataHandle::FullSize() const { + size_t size = 0; + for (const auto& blob : blobs_) { + size += blob->GetSize(); + } + return size; +} + +// The offset into the snapshots data blobs as though they were a single +// contiguous buffer. +size_t SnapshotsDataHandle::AbsoluteOffsetForIndex(BlobsIndex index) { + if (index.blob >= blobs_.size()) { + FML_LOG(WARNING) << "Blob index " << index.blob + << " is larger than the number of blobs (" << blobs_.size() + << "). Returning full size (" << FullSize() << ")"; + return FullSize(); + } + if (index.offset > blobs_[index.blob]->GetSize()) { + FML_LOG(WARNING) << "Offset for blob " << index.blob << " (" << index.offset + << ") is larger than the blob size (" + << blobs_[index.blob]->GetSize() + << "). Returning index start of next blob"; + return AbsoluteOffsetForIndex({index.blob + 1, 0}); + } + size_t offset = 0; + for (size_t i = 0; i < index.blob; i++) { + offset += blobs_[i]->GetSize(); + } + offset += index.offset; + return offset; +} + +BlobsIndex SnapshotsDataHandle::IndexForAbsoluteOffset(int64_t offset, + BlobsIndex start_index) { + size_t start_offset = AbsoluteOffsetForIndex(start_index); + if (offset < 0) { + if ((size_t)abs(offset) > start_offset) { + FML_LOG(WARNING) + << "Offset is before the beginning of SnapshotsData. Returning 0, 0"; + return {0, 0}; + } + } else if (offset + start_offset >= FullSize()) { + FML_LOG(WARNING) << "Target offset is past the end of SnapshotsData (" + << offset + start_offset << ", blobs size:" << FullSize() + << "). Returning last blob index and offset"; + return {blobs_.size(), blobs_.back()->GetSize()}; + } + + size_t dest_offset = start_offset + offset; + BlobsIndex index = {0, 0}; + for (const auto& blob : blobs_) { + if (dest_offset < blob->GetSize()) { + // The remaining offset is within this blob. + index.offset = dest_offset; + break; + } + + index.blob++; + dest_offset -= blob->GetSize(); + } + return index; +} + +std::unique_ptr SnapshotsDataHandle::createForSnapshots( + const DartSnapshot& vm_snapshot, + const DartSnapshot& isolate_snapshot) { + // This needs to match the order in which the blobs are written out in + // analyze_snapshot --dump_blobs + std::vector> blobs; + blobs.push_back(DataMapping(vm_snapshot)); + blobs.push_back(DataMapping(isolate_snapshot)); + blobs.push_back(InstructionsMapping(vm_snapshot)); + blobs.push_back(InstructionsMapping(isolate_snapshot)); + return std::make_unique(std::move(blobs)); +} + +uintptr_t SnapshotsDataHandle::Read(uint8_t* buffer, uintptr_t length) { + uintptr_t bytes_read = 0; + // Copy current blob from current offset and possibly into the next blob + // until we have read length bytes. + while (bytes_read < length) { + if (current_index_.blob >= blobs_.size()) { + // We have read all blobs. + break; + } + intptr_t remaining_blob_length = + blobs_[current_index_.blob]->GetSize() - current_index_.offset; + if (remaining_blob_length <= 0) { + // We have read all bytes in this blob. + current_index_.blob++; + current_index_.offset = 0; + continue; + } + intptr_t bytes_to_read = fmin(length - bytes_read, remaining_blob_length); + memcpy(buffer + bytes_read, + blobs_[current_index_.blob]->GetMapping() + current_index_.offset, + bytes_to_read); + bytes_read += bytes_to_read; + current_index_.offset += bytes_to_read; + } + + return bytes_read; +} + +int64_t SnapshotsDataHandle::Seek(int64_t offset, int32_t whence) { + BlobsIndex start_index; + switch (whence) { + case SEEK_CUR: + start_index = current_index_; + break; + case SEEK_SET: + start_index = {0, 0}; + break; + case SEEK_END: + start_index = {blobs_.size(), blobs_.back()->GetSize()}; + break; + default: + FML_CHECK(false) << "Unrecognized whence value in Seek: " << whence; + } + current_index_ = IndexForAbsoluteOffset(offset, start_index); + return current_index_.offset; +} + +} // namespace flutter \ No newline at end of file diff --git a/engine/src/flutter/shell/common/shorebird/snapshots_data_handle.h b/engine/src/flutter/shell/common/shorebird/snapshots_data_handle.h new file mode 100644 index 0000000000000..50c4b8179b412 --- /dev/null +++ b/engine/src/flutter/shell/common/shorebird/snapshots_data_handle.h @@ -0,0 +1,48 @@ +#ifndef FLUTTER_SHELL_COMMON_SHOREBIRD_SNAPSHOTS_DATA_HANDLE_H_ +#define FLUTTER_SHELL_COMMON_SHOREBIRD_SNAPSHOTS_DATA_HANDLE_H_ + +#include +#include "flutter/fml/file.h" +#include "flutter/runtime/dart_snapshot.h" +#include "third_party/dart/runtime/include/dart_tools_api.h" + +namespace flutter { + +// An offset into an indexed collection of buffers. blob is the index of the +// buffer, and offset is the offset into that buffer. +struct BlobsIndex { + size_t blob; + size_t offset; +}; + +// Implements a POSIX file I/O interface which allows us to provide the four +// data blobs of a Dart snapshot (vm_data, vm_instructions, isolate_data, +// isolate_instructions) to Rust as though it were a single piece of memory. +class SnapshotsDataHandle { + public: + // This would ideally be private, but we need to be able to call it from the + // static createForSnapshots method. + explicit SnapshotsDataHandle(std::vector> blobs) + : blobs_(std::move(blobs)) {} + + static std::unique_ptr createForSnapshots( + const DartSnapshot& vm_snapshot, + const DartSnapshot& isolate_snapshot); + + uintptr_t Read(uint8_t* buffer, uintptr_t length); + int64_t Seek(int64_t offset, int32_t whence); + + // The sum of all the blobs' sizes. + size_t FullSize() const; + + private: + size_t AbsoluteOffsetForIndex(BlobsIndex index); + BlobsIndex IndexForAbsoluteOffset(int64_t offset, BlobsIndex startIndex); + + BlobsIndex current_index_ = {0, 0}; + std::vector> blobs_; +}; + +} // namespace flutter + +#endif // FLUTTER_SHELL_COMMON_SHOREBIRD_SNAPSHOTS_DATA_HANDLE_H_ diff --git a/engine/src/flutter/shell/common/shorebird/snapshots_data_handle_unittests.cc b/engine/src/flutter/shell/common/shorebird/snapshots_data_handle_unittests.cc new file mode 100644 index 0000000000000..2acf44a7b8973 --- /dev/null +++ b/engine/src/flutter/shell/common/shorebird/snapshots_data_handle_unittests.cc @@ -0,0 +1,177 @@ +#include +#include +#include + +#include "flutter/shell/common/shorebird/snapshots_data_handle.h" + +#include "flutter/fml/mapping.h" +#include "flutter/runtime/dart_snapshot.h" +#include "flutter/testing/testing.h" +#include "gmock/gmock.h" +#include "gtest/gtest.h" +#include "testing/fixture_test.h" + +namespace flutter { +namespace testing { + +std::unique_ptr MakeHandle( + std::vector& blobs) { + // Map the strings into non-owned mappings: + std::vector> mappings = {}; + for (auto& blob : blobs) { + std::unique_ptr mapping = + std::make_unique( + reinterpret_cast(blob.data()), blob.size()); + mappings.push_back(std::move(mapping)); + } + auto handle = + std::make_unique(std::move(mappings)); + return handle; +} + +TEST(SnapshotsDataHandle, Read) { + std::vector blobs = {"abc", "def", "ghi", "jkl"}; + std::unique_ptr blobs_handle = MakeHandle(blobs); + + const size_t buffer_size = 12; + uint8_t buffer[buffer_size]; + std::fill(buffer, buffer + buffer_size, 0); + blobs_handle->Read(buffer, 6); + + EXPECT_EQ(buffer[0], 'a'); + EXPECT_EQ(buffer[1], 'b'); + EXPECT_EQ(buffer[2], 'c'); + EXPECT_EQ(buffer[3], 'd'); + EXPECT_EQ(buffer[4], 'e'); + EXPECT_EQ(buffer[5], 'f'); + + // Only the first 6 bytes should have been read. + EXPECT_EQ(buffer[6], 0); +} + +TEST(SnapshotsDataHandle, ReadAfterSeekWithPositiveOffset) { + std::vector blobs = {"abc", "def", "ghi", "jkl"}; + std::unique_ptr blobs_handle = MakeHandle(blobs); + + const size_t buffer_size = 20; + uint8_t buffer[buffer_size]; + std::fill(buffer, buffer + buffer_size, 0); + + blobs_handle->Seek(4, SEEK_CUR); + blobs_handle->Read(buffer, 6); + + EXPECT_EQ(buffer[0], 'e'); + EXPECT_EQ(buffer[1], 'f'); + EXPECT_EQ(buffer[2], 'g'); + EXPECT_EQ(buffer[3], 'h'); + EXPECT_EQ(buffer[4], 'i'); + EXPECT_EQ(buffer[5], 'j'); + + // Only the first 6 bytes should have been read. + EXPECT_EQ(buffer[6], 0); +} + +TEST(SnapshotsDataHandle, ReadAfterSeekWithNegativeOffset) { + std::vector blobs = {"abc", "def", "ghi", "jkl"}; + std::unique_ptr blobs_handle = MakeHandle(blobs); + + const size_t buffer_size = 20; + uint8_t buffer[buffer_size]; + std::fill(buffer, buffer + buffer_size, 0); + + blobs_handle->Read(buffer, 5); + EXPECT_EQ(buffer[0], 'a'); + EXPECT_EQ(buffer[1], 'b'); + EXPECT_EQ(buffer[2], 'c'); + EXPECT_EQ(buffer[3], 'd'); + EXPECT_EQ(buffer[4], 'e'); + EXPECT_EQ(buffer[5], 0); + + // Reset buffer + std::fill(buffer, buffer + buffer_size, 0); + + // Read 5, seeked back 4, should start reading at offset 1 ('b') + blobs_handle->Seek(-4, SEEK_CUR); + blobs_handle->Read(buffer, 6); + + EXPECT_EQ(buffer[0], 'b'); + EXPECT_EQ(buffer[1], 'c'); + EXPECT_EQ(buffer[2], 'd'); + EXPECT_EQ(buffer[3], 'e'); + EXPECT_EQ(buffer[4], 'f'); + EXPECT_EQ(buffer[5], 'g'); + EXPECT_EQ(buffer[6], 0); +} + +TEST(SnapshotsDataHandle, SeekPastEnd) { + std::vector blobs = {"abc", "def", "ghi", "jkl"}; + std::unique_ptr blobs_handle = MakeHandle(blobs); + + const size_t buffer_size = 20; + uint8_t buffer[buffer_size]; + std::fill(buffer, buffer + buffer_size, 0); + + // Seek 1 past the end + blobs_handle->Seek(blobs_handle->FullSize() + 1, SEEK_CUR); + + // Seek back 2 bytes and read 2 bytes + blobs_handle->Seek(-2, SEEK_CUR); + blobs_handle->Read(buffer, 2); + + EXPECT_EQ(buffer[0], 'k'); + EXPECT_EQ(buffer[1], 'l'); +} + +TEST(SnapshotsDataHandle, SeekBeforeBeginning) { + std::vector blobs = {"abc", "def", "ghi", "jkl"}; + std::unique_ptr blobs_handle = MakeHandle(blobs); + + const size_t buffer_size = 20; + uint8_t buffer[buffer_size]; + std::fill(buffer, buffer + buffer_size, 0); + + // Seek before the start of the blobs and read the first 2 bytes. + blobs_handle->Seek(-2, SEEK_CUR); + blobs_handle->Read(buffer, 2); + + EXPECT_EQ(buffer[0], 'a'); + EXPECT_EQ(buffer[1], 'b'); +} + +TEST(SnapshotsDataHandle, SeekFromBeginning) { + std::vector blobs = {"abc", "def", "ghi", "jkl"}; + std::unique_ptr blobs_handle = MakeHandle(blobs); + + const size_t buffer_size = 20; + uint8_t buffer[buffer_size]; + std::fill(buffer, buffer + buffer_size, 0); + + // Seek 10 bytes from current (the beginning) + blobs_handle->Seek(10, SEEK_CUR); + + // Seek 2 bytes from the beginning and read 2 bytes + blobs_handle->Seek(2, SEEK_SET); + blobs_handle->Read(buffer, 2); + + EXPECT_EQ(buffer[0], 'c'); + EXPECT_EQ(buffer[1], 'd'); +} + +TEST(SnapshotsDataHandle, SeekFromEnd) { + std::vector blobs = {"abc", "def", "ghi", "jkl"}; + std::unique_ptr blobs_handle = MakeHandle(blobs); + + const size_t buffer_size = 20; + uint8_t buffer[buffer_size]; + std::fill(buffer, buffer + buffer_size, 0); + + // Seek 2 bytes from the end and read 2 bytes + blobs_handle->Seek(-2, SEEK_END); + blobs_handle->Read(buffer, 2); + + EXPECT_EQ(buffer[0], 'k'); + EXPECT_EQ(buffer[1], 'l'); +} + +} // namespace testing +} // namespace flutter diff --git a/engine/src/flutter/shell/platform/android/BUILD.gn b/engine/src/flutter/shell/platform/android/BUILD.gn index 40c209ca92511..3b41ad6dd3b7a 100644 --- a/engine/src/flutter/shell/platform/android/BUILD.gn +++ b/engine/src/flutter/shell/platform/android/BUILD.gn @@ -174,6 +174,7 @@ source_set("flutter_shell_native_src") { "//flutter/runtime", "//flutter/runtime:libdart", "//flutter/shell/common", + "//flutter/shell/common/shorebird", "//flutter/shell/platform/android/context", "//flutter/shell/platform/android/external_view_embedder", "//flutter/shell/platform/android/jni", @@ -187,6 +188,8 @@ source_set("flutter_shell_native_src") { public_configs = [ "//flutter:config" ] + include_dirs = [ "//flutter/updater" ] + defines = [] libs = [ @@ -194,6 +197,17 @@ source_set("flutter_shell_native_src") { "EGL", "GLESv2", ] + if (target_cpu == "arm") { + libs += [ "//flutter/third_party/updater/target/armv7-linux-androideabi/release/libupdater.a" ] + } else if (target_cpu == "arm64") { + libs += [ "//flutter/third_party/updater/target/aarch64-linux-android/release/libupdater.a" ] + } else if (target_cpu == "x64") { + libs += [ "//flutter/third_party/updater/target/x86_64-linux-android/release/libupdater.a" ] + } else if (target_cpu == "x86") { + libs += [ "//flutter/third_party/updater/target/i686-linux-android/release/libupdater.a" ] + } else { + assert(false, "Unsupported target_cpu") + } } action("gen_android_build_config_java") { @@ -818,8 +832,10 @@ if (target_cpu != "x86") { } } -if (host_os == "linux" && - (target_cpu == "x64" || target_cpu == "arm64" || target_cpu == "riscv64")) { +# Build analyze_snapshot for 64-bit target CPUs on linux. +# Or always targeting arm64 for Shorebird builds. +if ((host_os == "linux" && (target_cpu == "x64" || target_cpu == "riscv64")) || + target_cpu == "arm64") { zip_bundle("analyze_snapshot") { deps = [ "$dart_src/runtime/bin:analyze_snapshot($host_toolchain)" ] diff --git a/engine/src/flutter/shell/platform/android/android_exports.lst b/engine/src/flutter/shell/platform/android/android_exports.lst index 198bff773dd74..9b924e7738cd4 100644 --- a/engine/src/flutter/shell/platform/android/android_exports.lst +++ b/engine/src/flutter/shell/platform/android/android_exports.lst @@ -11,6 +11,18 @@ _binary_icudtl_dat_size; InternalFlutterGpu*; kInternalFlutterGpu*; + shorebird_init; + shorebird_active_path; + shorebird_active_patch_number; + shorebird_free_string; + shorebird_free_update_result; + shorebird_check_for_downloadable_update; + shorebird_check_for_update; + shorebird_update; + shorebird_update_with_result; + shorebird_next_boot_patch_number; + shorebird_current_boot_patch_number; + shorebird_validate_next_boot_patch; local: *; }; diff --git a/engine/src/flutter/shell/platform/android/flutter_main.cc b/engine/src/flutter/shell/platform/android/flutter_main.cc index d14ac40029645..d881ca39efd3f 100644 --- a/engine/src/flutter/shell/platform/android/flutter_main.cc +++ b/engine/src/flutter/shell/platform/android/flutter_main.cc @@ -20,6 +20,7 @@ #include "flutter/fml/platform/android/paths_android.h" #include "flutter/lib/ui/plugins/callback_cache.h" #include "flutter/runtime/dart_vm.h" +#include "flutter/shell/common/shorebird/shorebird.h" #include "flutter/shell/common/switches.h" #include "flutter/shell/platform/android/android_context_vk_impeller.h" #include "flutter/shell/platform/android/android_rendering_selector.h" @@ -29,6 +30,8 @@ #include "impeller/toolkit/android/proc_table.h" #include "txt/platform.h" +#include "third_party/updater/library/include/updater.h" + namespace flutter { constexpr int kMinimumAndroidApiLevelForImpeller = 29; @@ -93,6 +96,9 @@ void FlutterMain::Init(JNIEnv* env, jstring kernelPath, jstring appStoragePath, jstring engineCachesPath, + jstring shorebirdYaml, + jstring version, + jstring versionCode, jlong initTimeMillis, jint api_level) { std::vector args; @@ -151,8 +157,18 @@ void FlutterMain::Init(JNIEnv* env, flutter::DartCallbackCache::SetCachePath( fml::jni::JavaStringToString(env, appStoragePath)); - fml::paths::InitializeAndroidCachesPath( - fml::jni::JavaStringToString(env, engineCachesPath)); + auto code_cache_path = fml::jni::JavaStringToString(env, engineCachesPath); + auto app_storage_path = fml::jni::JavaStringToString(env, appStoragePath); + fml::paths::InitializeAndroidCachesPath(code_cache_path); + +#if FLUTTER_RELEASE + std::string shorebird_yaml = fml::jni::JavaStringToString(env, shorebirdYaml); + std::string version_string = fml::jni::JavaStringToString(env, version); + std::string version_code_string = + fml::jni::JavaStringToString(env, versionCode); + ConfigureShorebird(code_cache_path, app_storage_path, settings, + shorebird_yaml, version_string, version_code_string); +#endif flutter::DartCallbackCache::LoadCacheFromDisk(); @@ -245,6 +261,7 @@ bool FlutterMain::Register(JNIEnv* env) { { .name = "nativeInit", .signature = "(Landroid/content/Context;[Ljava/lang/String;Ljava/" + "lang/String;Ljava/lang/String;Ljava/lang/String;Ljava/" "lang/String;Ljava/lang/String;Ljava/lang/String;JI)V", .fnPtr = reinterpret_cast(&Init), }, diff --git a/engine/src/flutter/shell/platform/android/flutter_main.h b/engine/src/flutter/shell/platform/android/flutter_main.h index dda959801bc32..cf03f889ec30e 100644 --- a/engine/src/flutter/shell/platform/android/flutter_main.h +++ b/engine/src/flutter/shell/platform/android/flutter_main.h @@ -44,6 +44,9 @@ class FlutterMain { jstring kernelPath, jstring appStoragePath, jstring engineCachesPath, + jstring shorebirdYaml, + jstring version, + jstring versionCode, jlong initTimeMillis, jint api_level); diff --git a/engine/src/flutter/shell/platform/android/io/flutter/embedding/engine/FlutterJNI.java b/engine/src/flutter/shell/platform/android/io/flutter/embedding/engine/FlutterJNI.java index 59dc281909838..8570bad6d8b95 100644 --- a/engine/src/flutter/shell/platform/android/io/flutter/embedding/engine/FlutterJNI.java +++ b/engine/src/flutter/shell/platform/android/io/flutter/embedding/engine/FlutterJNI.java @@ -8,6 +8,8 @@ import android.annotation.SuppressLint; import android.content.Context; +import android.content.pm.PackageInfo; +import android.content.pm.PackageManager; import android.content.res.AssetManager; import android.graphics.Bitmap; import android.graphics.SurfaceTexture; @@ -42,6 +44,10 @@ import io.flutter.view.AccessibilityBridge; import io.flutter.view.FlutterCallbackInformation; import io.flutter.view.TextureRegistry; +import java.io.BufferedReader; +import java.io.IOException; +import java.io.InputStream; +import java.io.InputStreamReader; import java.lang.ref.WeakReference; import java.nio.ByteBuffer; import java.util.ArrayList; @@ -178,6 +184,9 @@ private static native void nativeInit( @Nullable String bundlePath, @NonNull String appStoragePath, @NonNull String engineCachesPath, + @Nullable String shorebirdYaml, + @Nullable String version, + @Nullable String versionCode, long initTimeMillis, int apiLevel); @@ -206,8 +215,47 @@ public void init( Log.w(TAG, "FlutterJNI.init called more than once"); } + String version = null; + String versionCode = null; + try { + PackageInfo packageInfo = + context.getPackageManager().getPackageInfo(context.getPackageName(), 0); + version = packageInfo.versionName; + if (Build.VERSION.SDK_INT >= API_LEVELS.API_28) { + versionCode = String.valueOf(packageInfo.getLongVersionCode()); + } else { + versionCode = String.valueOf(packageInfo.versionCode); + } + } catch (PackageManager.NameNotFoundException e) { + Log.e(TAG, "Failed to read app version. Shorebird updater can't run.", e); + } + + String shorebirdYaml = null; + try { + InputStream yaml = context.getAssets().open("flutter_assets/shorebird.yaml"); + BufferedReader r = new BufferedReader(new InputStreamReader(yaml)); + StringBuilder total = new StringBuilder(); + for (String line; (line = r.readLine()) != null; ) { + total.append(line).append('\n'); + } + shorebirdYaml = total.toString(); + Log.d(TAG, "shorebird.yaml: " + shorebirdYaml); + } catch (IOException e) { + Log.e(TAG, "Failed to load shorebird.yaml", e); + Log.e(TAG, "Did you remember to include shorebird.yaml in your pubspec.yaml's assets?"); + } + FlutterJNI.nativeInit( - context, args, bundlePath, appStoragePath, engineCachesPath, initTimeMillis, apiLevel); + context, + args, + bundlePath, + appStoragePath, + engineCachesPath, + shorebirdYaml, + version, + versionCode, + initTimeMillis, + apiLevel); FlutterJNI.initCalled = true; } diff --git a/engine/src/flutter/shell/platform/darwin/ios/BUILD.gn b/engine/src/flutter/shell/platform/darwin/ios/BUILD.gn index 9e398942d2a15..c4548b572c8cd 100644 --- a/engine/src/flutter/shell/platform/darwin/ios/BUILD.gn +++ b/engine/src/flutter/shell/platform/darwin/ios/BUILD.gn @@ -88,6 +88,14 @@ source_set("flutter_framework_source") { "//build/config/ios:ios_application_extension", ] + if (target_cpu == "arm64") { + libs = [ "//flutter/third_party/updater/target/aarch64-apple-ios/release/libupdater.a" ] + } else if (target_cpu == "x64") { + libs = [ "//flutter/third_party/updater/target/x86_64-apple-ios/release/libupdater.a" ] + } else { + assert(false, "Unsupported target_cpu") + } + sources = [ "framework/Source/FlutterAppDelegate.mm", "framework/Source/FlutterAppDelegate_Internal.h", @@ -208,12 +216,14 @@ source_set("flutter_framework_source") { deps = [ ":ios_gpu_configuration", + "$dart_src/runtime/bin:elf_loader", "//flutter/common", "//flutter/common/graphics", "//flutter/fml", "//flutter/lib/ui", "//flutter/runtime", "//flutter/shell/common", + "//flutter/shell/common/shorebird", "//flutter/shell/platform/common:common_cpp_input", "//flutter/shell/platform/darwin/common", "//flutter/shell/platform/darwin/common:framework_common", diff --git a/engine/src/flutter/shell/platform/darwin/ios/framework/Source/FlutterDartProject.mm b/engine/src/flutter/shell/platform/darwin/ios/framework/Source/FlutterDartProject.mm index 9fb21e18a8475..b4d2c0e7018a6 100644 --- a/engine/src/flutter/shell/platform/darwin/ios/framework/Source/FlutterDartProject.mm +++ b/engine/src/flutter/shell/platform/darwin/ios/framework/Source/FlutterDartProject.mm @@ -13,6 +13,8 @@ #include "flutter/common/constants.h" #include "flutter/fml/build_config.h" +#include "flutter/fml/paths.h" +#include "flutter/shell/common/shorebird/shorebird.h" #include "flutter/shell/common/switches.h" #import "flutter/shell/platform/darwin/common/InternalFlutterSwiftCommon/InternalFlutterSwiftCommon.h" #include "flutter/shell/platform/darwin/common/command_line.h" @@ -101,10 +103,12 @@ static BOOL DoesHardwareSupportWideGamut() { } if (flutter::DartVM::IsRunningPrecompiledCode()) { + NSLog(@"SANITY CHECK: Running precompiled code."); if (hasExplicitBundle) { NSString* executablePath = bundle.executablePath; if ([[NSFileManager defaultManager] fileExistsAtPath:executablePath]) { settings.application_library_paths.push_back(executablePath.UTF8String); + NSLog(@"Using precompiled library from %@", executablePath); } } @@ -116,6 +120,7 @@ static BOOL DoesHardwareSupportWideGamut() { NSString* executablePath = [NSBundle bundleWithPath:libraryPath].executablePath; if (executablePath.length > 0) { settings.application_library_paths.push_back(executablePath.UTF8String); + NSLog(@"Using library from %@", libraryPath); } } } @@ -130,6 +135,7 @@ static BOOL DoesHardwareSupportWideGamut() { [NSBundle bundleWithPath:applicationFrameworkPath].executablePath; if (executablePath.length > 0) { settings.application_library_paths.push_back(executablePath.UTF8String); + NSLog(@"Using App.framework from %@", applicationFrameworkPath); } } } @@ -161,6 +167,33 @@ static BOOL DoesHardwareSupportWideGamut() { } } + NSString* assetsPath = [NSString stringWithUTF8String:settings.assets_path.c_str()]; + NSLog(@"ASSET PATH %@", assetsPath); + + // FIXME: This may not be the correct path (e.g., should it include the organization id?) + // See + // https://developer.apple.com/library/archive/documentation/FileManagement/Conceptual/FileSystemProgrammingGuide/FileSystemOverview/FileSystemOverview.html#//apple_ref/doc/uid/TP40010672-CH2-SW13 + // /private/var/mobile/Containers/Data/Application/264477BF-6E38-47C9-AAD9-532BB842F197/Library/Application + // Support/shorebird/shorebird_updater + std::string cache_path = + fml::paths::JoinPaths({getenv("HOME"), "Library/Application Support/shorebird"}); + NSURL* shorebirdYamlPath = [NSURL URLWithString:@"shorebird.yaml" + relativeToURL:[NSURL fileURLWithPath:assetsPath]]; + NSString* appVersion = [mainBundle objectForInfoDictionaryKey:@"CFBundleShortVersionString"]; + NSString* appBuildNumber = [mainBundle objectForInfoDictionaryKey:@"CFBundleVersion"]; + NSString* shorebirdYamlContents = [NSString stringWithContentsOfURL:shorebirdYamlPath + encoding:NSUTF8StringEncoding + error:nil]; + if (shorebirdYamlContents != nil) { + // Note: we intentionally pass cache_path twice. We provide two different directories + // to ConfigureShorebird because Android differentiates between data that persists + // between releases and data that does not. iOS does not make this distinction. + flutter::ConfigureShorebird(cache_path, cache_path, settings, shorebirdYamlContents.UTF8String, + appVersion.UTF8String, appBuildNumber.UTF8String); + } else { + NSLog(@"Failed to find shorebird.yaml, not starting updater."); + } + // Domain network configuration // Disabled in https://github.com/flutter/flutter/issues/72723. // Re-enable in https://github.com/flutter/flutter/issues/54448. diff --git a/engine/src/flutter/shell/platform/darwin/ios/framework/Source/FlutterViewController.mm b/engine/src/flutter/shell/platform/darwin/ios/framework/Source/FlutterViewController.mm index 27e30b1f3d615..96c643de7c114 100644 --- a/engine/src/flutter/shell/platform/darwin/ios/framework/Source/FlutterViewController.mm +++ b/engine/src/flutter/shell/platform/darwin/ios/framework/Source/FlutterViewController.mm @@ -40,6 +40,7 @@ #import "flutter/third_party/spring_animation/spring_animation.h" FLUTTER_ASSERT_ARC +#import static constexpr int kMicrosecondsPerSecond = 1000 * 1000; static constexpr CGFloat kScrollViewContentSize = 2.0; diff --git a/engine/src/flutter/shell/platform/darwin/macos/BUILD.gn b/engine/src/flutter/shell/platform/darwin/macos/BUILD.gn index ea069e2674648..d37ae587a8000 100644 --- a/engine/src/flutter/shell/platform/darwin/macos/BUILD.gn +++ b/engine/src/flutter/shell/platform/darwin/macos/BUILD.gn @@ -145,12 +145,14 @@ source_set("flutter_framework_source") { ":macos_gpu_configuration", "//flutter/flow", "//flutter/fml", + "//flutter/shell/common/shorebird", "//flutter/shell/platform/common:common_cpp_accessibility", "//flutter/shell/platform/common:common_cpp_core", "//flutter/shell/platform/common:common_cpp_enums", "//flutter/shell/platform/common:common_cpp_input", "//flutter/shell/platform/common:common_cpp_isolate_scope", "//flutter/shell/platform/common:common_cpp_switches", + "//flutter/shell/platform/darwin/common", "//flutter/shell/platform/darwin/common:availability_version_check", "//flutter/shell/platform/darwin/common:framework_common", "//flutter/shell/platform/darwin/graphics", diff --git a/engine/src/flutter/shell/platform/darwin/macos/framework/Source/FlutterEngine.mm b/engine/src/flutter/shell/platform/darwin/macos/framework/Source/FlutterEngine.mm index 1f9ffee62b2ca..d6d842c17628b 100644 --- a/engine/src/flutter/shell/platform/darwin/macos/framework/Source/FlutterEngine.mm +++ b/engine/src/flutter/shell/platform/darwin/macos/framework/Source/FlutterEngine.mm @@ -12,6 +12,8 @@ #include "flutter/common/constants.h" #include "flutter/fml/logging.h" +#include "flutter/fml/paths.h" +#include "flutter/shell/common/shorebird/shorebird.h" #include "flutter/shell/platform/common/app_lifecycle_state.h" #include "flutter/shell/platform/common/engine_switches.h" #include "flutter/shell/platform/embedder/embedder.h" @@ -657,6 +659,40 @@ - (void)onFocusChangeRequest:(const FlutterViewFocusChangeRequest*)request { } } +- (BOOL)configureShorebird:(NSString**)patchPath { + NSLog(@"[shorebird] setting up non-linker shorebird"); + NSString* bundlePath = + [[NSBundle bundleWithURL:[NSBundle.mainBundle.privateFrameworksURL + URLByAppendingPathComponent:@"App.framework"]] bundlePath]; + bundlePath = [bundlePath stringByAppendingString:@"/App"]; + NSString* assetsPath = _project.assetsPath; + NSURL* shorebirdYamlPath = [NSURL URLWithString:@"shorebird.yaml" + relativeToURL:[NSURL fileURLWithPath:assetsPath]]; + NSString* shorebirdYamlContents = [NSString stringWithContentsOfURL:shorebirdYamlPath + encoding:NSUTF8StringEncoding + error:nil]; + NSString* appVersion = + [NSBundle.mainBundle objectForInfoDictionaryKey:@"CFBundleShortVersionString"]; + NSString* appBuildNumber = [NSBundle.mainBundle objectForInfoDictionaryKey:@"CFBundleVersion"]; + std::string cache_path = + fml::paths::JoinPaths({getenv("HOME"), "Library", "Application Support", "shorebird"}); + flutter::ReleaseVersion release_version = {appVersion.UTF8String, appBuildNumber.UTF8String}; + flutter::ShorebirdConfigArgs shorebird_args(cache_path, cache_path, bundlePath.UTF8String, + shorebirdYamlContents.UTF8String, release_version); + NSLog(@"[shorebird] calling ConfigureShorebird"); + std::string patch_path; + auto res = flutter::ConfigureShorebird(shorebird_args, patch_path); + if (!res) { + NSLog(@"[shorebird] ConfigureShorebird failed"); + return NO; + } + + NSLog(@"[shorebird] ConfigureShorebird success!"); + *patchPath = [NSString stringWithUTF8String:patch_path.c_str()]; + NSLog(@"[shorebird] patchPath: %@", *patchPath); + return YES; +} + - (BOOL)runWithEntrypoint:(NSString*)entrypoint { if (self.running) { return NO; @@ -779,7 +815,19 @@ - (BOOL)runWithEntrypoint:(NSString*)entrypoint { }; flutterArguments.custom_task_runners = &custom_task_runners; - [self loadAOTData:_project.assetsPath]; + NSString* elfPath; + BOOL configureShorebirdRes = [self configureShorebird:&elfPath]; + if (!configureShorebirdRes) { + // No patch exists, or we failed to configure shorebird. This is a fallback. + // Upstream, this code lives in -(void)loadAOTData:. + // + // This is the location where the test fixture places the snapshot file. + // For applications built by Flutter tool, this is in "App.framework". + elfPath = [NSString pathWithComponents:@[ _project.assetsPath, @"app_elf_snapshot.so" ]]; + } + + [self loadAOTData:elfPath]; + if (_aotData) { flutterArguments.aot_data = _aotData; } @@ -803,6 +851,7 @@ - (BOOL)runWithEntrypoint:(NSString*)entrypoint { }; FlutterRendererConfig rendererConfig = [_renderer createRendererConfig]; + FlutterEngineResult result = _embedderAPI.Initialize( FLUTTER_ENGINE_VERSION, &rendererConfig, &flutterArguments, (__bridge void*)(self), &_engine); if (result != kSuccess) { @@ -832,7 +881,7 @@ - (BOOL)runWithEntrypoint:(NSString*)entrypoint { return YES; } -- (void)loadAOTData:(NSString*)assetsDir { +- (void)loadAOTData:(NSString*)elfPath { if (!_embedderAPI.RunsAOTCompiledDartCode()) { return; } @@ -840,11 +889,8 @@ - (void)loadAOTData:(NSString*)assetsDir { BOOL isDirOut = false; // required for NSFileManager fileExistsAtPath. NSFileManager* fileManager = [NSFileManager defaultManager]; - // This is the location where the test fixture places the snapshot file. - // For applications built by Flutter tool, this is in "App.framework". - NSString* elfPath = [NSString pathWithComponents:@[ assetsDir, @"app_elf_snapshot.so" ]]; - if (![fileManager fileExistsAtPath:elfPath isDirectory:&isDirOut]) { + FML_LOG(INFO) << "in loadAOTData, elfPath does not exist: " << elfPath.UTF8String; return; } diff --git a/engine/src/flutter/shell/platform/linux/BUILD.gn b/engine/src/flutter/shell/platform/linux/BUILD.gn index 66797b3288330..34daffe32637f 100644 --- a/engine/src/flutter/shell/platform/linux/BUILD.gn +++ b/engine/src/flutter/shell/platform/linux/BUILD.gn @@ -158,6 +158,7 @@ source_set("flutter_linux_sources") { "fl_settings_channel.cc", "fl_settings_handler.cc", "fl_settings_portal.cc", + "fl_shorebird.cc", "fl_socket_accessible.cc", "fl_standard_message_codec.cc", "fl_standard_method_codec.cc", @@ -187,12 +188,14 @@ source_set("flutter_linux_sources") { deps = [ "$dart_src/runtime:dart_api", "//flutter/fml", + "//flutter/shell/common/shorebird", "//flutter/shell/platform/common:common_cpp_enums", "//flutter/shell/platform/common:common_cpp_input", "//flutter/shell/platform/common:common_cpp_isolate_scope", "//flutter/shell/platform/common:common_cpp_switches", "//flutter/shell/platform/embedder:embedder_headers", "//flutter/third_party/rapidjson", + "//flutter/third_party/tonic", ] } diff --git a/engine/src/flutter/shell/platform/linux/fl_engine.cc b/engine/src/flutter/shell/platform/linux/fl_engine.cc index f88d08cb05f47..f5671a17f3b7f 100644 --- a/engine/src/flutter/shell/platform/linux/fl_engine.cc +++ b/engine/src/flutter/shell/platform/linux/fl_engine.cc @@ -10,6 +10,7 @@ #include #include "flutter/common/constants.h" +#include "flutter/fml/logging.h" #include "flutter/shell/platform/common/engine_switches.h" #include "flutter/shell/platform/embedder/embedder.h" #include "flutter/shell/platform/linux/fl_accessibility_handler.h" @@ -23,6 +24,7 @@ #include "flutter/shell/platform/linux/fl_platform_handler.h" #include "flutter/shell/platform/linux/fl_plugin_registrar_private.h" #include "flutter/shell/platform/linux/fl_settings_handler.h" +#include "flutter/shell/platform/linux/fl_shorebird.h" #include "flutter/shell/platform/linux/fl_texture_gl_private.h" #include "flutter/shell/platform/linux/fl_texture_registrar_private.h" #include "flutter/shell/platform/linux/public/flutter_linux/fl_plugin_registry.h" @@ -830,6 +832,9 @@ gboolean fl_engine_start(FlEngine* self, GError** error) { g_autoptr(GPtrArray) command_line_args = g_ptr_array_new_with_free_func(g_free); + // FlutterProjectArgs expects a full argv, so when processing it for flags + // the first item is treated as the executable and ignored. Add a dummy + // value so that all switches are used. g_ptr_array_insert(command_line_args, 0, g_strdup("flutter")); for (const auto& env_switch : env_switches) { g_ptr_array_add(command_line_args, g_strdup(env_switch.c_str())); @@ -890,9 +895,22 @@ gboolean fl_engine_start(FlEngine* self, GError** error) { args.compositor = &compositor; if (self->embedder_api.RunsAOTCompiledDartCode()) { + // This struct contains raw C strings and needs to have its lifetime scoped + // to this block. FlutterEngineAOTDataSource source = {}; source.type = kFlutterEngineAOTDataSourceTypeElfPath; - source.elf_path = fl_dart_project_get_aot_library_path(self->project); + std::string patch_path; + auto setup_shorebird_result = + flutter::SetUpShorebird(args.assets_path, patch_path); + if (setup_shorebird_result) { + // If we have a patch installed, we replace the default AOT library path + // with the patch path here. + source.elf_path = patch_path.c_str(); + } else { + FML_LOG(ERROR) << "Failed to configure Shorebird."; + source.elf_path = fl_dart_project_get_aot_library_path(self->project); + } + if (self->embedder_api.CreateAOTData(&source, &self->aot_data) != kSuccess) { g_set_error(error, fl_engine_error_quark(), FL_ENGINE_ERROR_FAILED, diff --git a/engine/src/flutter/shell/platform/linux/fl_shorebird.cc b/engine/src/flutter/shell/platform/linux/fl_shorebird.cc new file mode 100644 index 0000000000000..eb913ad720c61 --- /dev/null +++ b/engine/src/flutter/shell/platform/linux/fl_shorebird.cc @@ -0,0 +1,56 @@ +#include "flutter/shell/platform/linux/fl_shorebird.h" + +#include +#include +#include + +#include "flutter/fml/file.h" +#include "flutter/fml/logging.h" +#include "flutter/fml/paths.h" +#include "flutter/shell/common/shorebird/shorebird.h" +#include "rapidjson/document.h" +#include "third_party/tonic/filesystem/filesystem/file.h" + +// Namespaced to avoid Google style warnings. +namespace flutter { + +gboolean SetUpShorebird(const char* assets_path, std::string& patch_path) { + auto shorebird_yaml_path = + fml::paths::JoinPaths({assets_path, "shorebird.yaml"}); + std::string shorebird_yaml_contents(""); + if (!filesystem::ReadFileToString(shorebird_yaml_path, + &shorebird_yaml_contents)) { + FML_LOG(ERROR) << "Failed to read shorebird.yaml."; + return false; + } + + std::string code_cache_path = + fml::paths::JoinPaths({g_get_home_dir(), ".shorebird_cache"}); + auto executable_location = fml::paths::GetExecutableDirectoryPath().second; + auto app_path = + fml::paths::JoinPaths({executable_location, "lib", "libapp.so"}); + auto version_json_path = fml::paths::JoinPaths({assets_path, "version.json"}); + std::ifstream input(version_json_path); + if (!input) { + return false; + } + std::string json_contents{std::istreambuf_iterator(input), + std::istreambuf_iterator()}; + + rapidjson::Document json_doc; + json_doc.Parse(json_contents.c_str()); + if (json_doc.HasParseError()) { + // Could not parse version file, aborting. + return false; + } + + const auto version_map = json_doc.GetObject(); + ReleaseVersion release_version{version_map["version"].GetString(), + version_map["build_number"].GetString()}; + + ShorebirdConfigArgs shorebird_args(code_cache_path, code_cache_path, app_path, + shorebird_yaml_contents, release_version); + return ConfigureShorebird(shorebird_args, patch_path); +} + +} // namespace flutter diff --git a/engine/src/flutter/shell/platform/linux/fl_shorebird.h b/engine/src/flutter/shell/platform/linux/fl_shorebird.h new file mode 100644 index 0000000000000..e38965776ac97 --- /dev/null +++ b/engine/src/flutter/shell/platform/linux/fl_shorebird.h @@ -0,0 +1,13 @@ +#ifndef FLUTTER_SHELL_PLATFORM_LINUX_FL_SHOREBIRD_H_ +#define FLUTTER_SHELL_PLATFORM_LINUX_FL_SHOREBIRD_H_ + +#include +#include + +namespace flutter { + +gboolean SetUpShorebird(const char* assets_path, std::string& patch_path); + +} // namespace flutter + +#endif // FLUTTER_SHELL_PLATFORM_LINUX_FL_SHOREBIRD_H_ diff --git a/engine/src/flutter/shell/platform/windows/BUILD.gn b/engine/src/flutter/shell/platform/windows/BUILD.gn index a6af78f4f44ac..18ca773c17f3e 100644 --- a/engine/src/flutter/shell/platform/windows/BUILD.gn +++ b/engine/src/flutter/shell/platform/windows/BUILD.gn @@ -178,6 +178,7 @@ source_set("flutter_windows_source") { ":flutter_windows_headers", "//flutter/fml", "//flutter/impeller/renderer/backend/gles", + "//flutter/shell/common/shorebird:shorebird", "//flutter/shell/geometry", "//flutter/shell/platform/common:common_cpp", "//flutter/shell/platform/common:common_cpp_input", @@ -193,6 +194,7 @@ source_set("flutter_windows_source") { "//flutter/third_party/angle:libEGL_static", "//flutter/third_party/angle:libGLESv2_static", "//flutter/third_party/rapidjson", + "//flutter/third_party/tonic", ] } @@ -204,6 +206,11 @@ copy("publish_headers_windows") { deps = [ "//flutter/shell/platform/common:publish_headers" ] } +copy("updater_exports_windows") { + sources = [ "flutter_windows.dll.def" ] + outputs = [ "$root_out_dir/{{source_file_part}}" ] +} + shared_library("flutter_windows") { deps = [ ":flutter_windows_source" ] @@ -319,6 +326,7 @@ group("windows") { deps = [ ":flutter_windows", ":publish_headers_windows", + ":updater_exports_windows", "//flutter/shell/platform/windows/client_wrapper:publish_wrapper_windows", ] diff --git a/engine/src/flutter/shell/platform/windows/flutter_project_bundle.h b/engine/src/flutter/shell/platform/windows/flutter_project_bundle.h index f504ea8555687..2a812e02234d3 100644 --- a/engine/src/flutter/shell/platform/windows/flutter_project_bundle.h +++ b/engine/src/flutter/shell/platform/windows/flutter_project_bundle.h @@ -68,6 +68,10 @@ class FlutterProjectBundle { // Sets engine switches. void SetSwitches(const std::vector& switches); + void SetAotLibraryPath(const std::filesystem::path& aot_library_path) { + aot_library_path_ = aot_library_path; + } + // Attempts to load AOT data for this bundle. The returned data must be // retained until any engine instance it is passed to has been shut down. // diff --git a/engine/src/flutter/shell/platform/windows/flutter_windows.dll.def b/engine/src/flutter/shell/platform/windows/flutter_windows.dll.def new file mode 100644 index 0000000000000..bcaa785977259 --- /dev/null +++ b/engine/src/flutter/shell/platform/windows/flutter_windows.dll.def @@ -0,0 +1,17 @@ +EXPORTS + shorebird_check_for_downloadable_update = shorebird_check_for_downloadable_update + shorebird_check_for_update = shorebird_check_for_update + shorebird_current_boot_patch_number = shorebird_current_boot_patch_number + shorebird_free_string = shorebird_free_string + shorebird_free_update_result = shorebird_free_update_result + shorebird_init = shorebird_init + shorebird_next_boot_patch_number = shorebird_next_boot_patch_number + shorebird_next_boot_patch_path = shorebird_next_boot_patch_path + shorebird_validate_next_boot_patch = shorebird_validate_next_boot_patch + shorebird_report_launch_failure = shorebird_report_launch_failure + shorebird_report_launch_start = shorebird_report_launch_start + shorebird_report_launch_success = shorebird_report_launch_success + shorebird_should_auto_update = shorebird_should_auto_update + shorebird_start_update_thread = shorebird_start_update_thread + shorebird_update = shorebird_update + shorebird_update_with_result = shorebird_update_with_result \ No newline at end of file diff --git a/engine/src/flutter/shell/platform/windows/flutter_windows_engine.cc b/engine/src/flutter/shell/platform/windows/flutter_windows_engine.cc index d9f797ca257ba..b48c62b76dbb1 100644 --- a/engine/src/flutter/shell/platform/windows/flutter_windows_engine.cc +++ b/engine/src/flutter/shell/platform/windows/flutter_windows_engine.cc @@ -5,15 +5,20 @@ #include "flutter/shell/platform/windows/flutter_windows_engine.h" #include +#include +#include +#include #include #include #include +#include #include "flutter/fml/logging.h" #include "flutter/fml/paths.h" #include "flutter/fml/platform/win/wstring_conversion.h" #include "flutter/fml/synchronization/waitable_event.h" +#include "flutter/shell/common/shorebird/shorebird.h" #include "flutter/shell/platform/common/client_wrapper/binary_messenger_impl.h" #include "flutter/shell/platform/common/client_wrapper/include/flutter/standard_message_codec.h" #include "flutter/shell/platform/common/path_utils.h" @@ -29,6 +34,7 @@ #include "flutter/shell/platform/windows/window_manager.h" #include "flutter/third_party/accessibility/ax/ax_node.h" #include "shell/platform/windows/flutter_project_bundle.h" +#include "third_party/tonic/filesystem/filesystem/file.h" // winbase.h defines GetCurrentTime as a macro. #undef GetCurrentTime @@ -282,13 +288,148 @@ bool FlutterWindowsEngine::Run() { return Run(""); } +int GetReleaseVersionAndBuildNumber(ReleaseVersion* release_version) { + char module_path[MAX_PATH]; + // Get the full path of the currently running executable. The return value is + // the size of the string that was copied to the buffer, with -1 indicating + // failure. + if (GetModuleFileNameA(NULL, module_path, MAX_PATH) == -1) { + return -1; + } + + // Get the size of the version information + DWORD handle = -1; + DWORD version_info_size = GetFileVersionInfoSizeA(module_path, &handle); + if (version_info_size == -1) { + return -1; + } + + // Allocate memory for version info + std::unique_ptr version_data(new char[version_info_size]); + if (!GetFileVersionInfoA(module_path, handle, version_info_size, + version_data.get())) { + return -1; + } + + // Adopted from + // https://learn.microsoft.com/en-us/windows/win32/api/winver/nf-winver-verqueryvaluea + // Get the translation table + struct LANGANDCODEPAGE { + WORD wLanguage; + WORD wCodePage; + }* lpTranslate; + + UINT cbTranslate = 0; + if (!VerQueryValueA(version_data.get(), "\\VarFileInfo\\Translation", + (LPVOID*)&lpTranslate, &cbTranslate)) { + FML_LOG(ERROR) << "Error: Unable to get translation info."; + return -1; + } + + // Construct the query string using the first translation found + char subBlock[64]; + sprintf_s(subBlock, "\\StringFileInfo\\%04x%04x\\ProductVersion", + lpTranslate[0].wLanguage, lpTranslate[0].wCodePage); + + LPSTR versionString = nullptr; + UINT size = 0; + if (!VerQueryValueA(version_data.get(), subBlock, (LPVOID*)&versionString, + &size)) { + return -1; + } + + if (!versionString) { + return -1; + } + + // The version string is in the format of "1.0.0+1", with the label ("+1") + // being optional. + auto version = std::string(versionString); + auto plusPos = version.find("+"); + if (plusPos != std::string::npos) { + auto semVer = version.substr(0, plusPos); + auto patch = version.substr(plusPos + 1, version.length()); + release_version->version = semVer; + release_version->build_number = patch; + } else { + release_version->version = version; + } + + return kSuccess; +} + +bool GetLocalAppDataPath(std::string& outPath) { + PWSTR path = nullptr; + HRESULT result = SHGetKnownFolderPath(FOLDERID_LocalAppData, 0, NULL, &path); + if (!SUCCEEDED(result)) { + return false; + } + + std::wstring widePath(path); + std::string localAppDataPath(widePath.begin(), widePath.end()); + // The calling process is responsible for freeing this resource + // https://learn.microsoft.com/en-us/windows/win32/api/shlobj_core/nf-shlobj_core-shgetknownfolderpath + CoTaskMemFree(path); + outPath = localAppDataPath; + return true; +} + +bool SetUpShorebird(std::string assets_path_string, std::string& patch_path) { + auto shorebird_yaml_path = + fml::paths::JoinPaths({assets_path_string, "shorebird.yaml"}); + std::string shorebird_yaml_contents(""); + if (!filesystem::ReadFileToString(shorebird_yaml_path, + &shorebird_yaml_contents)) { + FML_LOG(ERROR) << "Failed to read shorebird.yaml."; + return false; + } + + std::string code_cache_path; + if (!GetLocalAppDataPath(code_cache_path)) { + FML_LOG(ERROR) << "Failed to retrieve the local AppData directory."; + return false; + } + + auto executable_location = fml::paths::GetExecutableDirectoryPath().second; + auto app_path = + fml::paths::JoinPaths({executable_location, "data", "app.so"}); + ReleaseVersion release_version; + auto release_version_result = + GetReleaseVersionAndBuildNumber(&release_version); + if (release_version_result != kSuccess) { + FML_LOG(ERROR) + << "Failed to retrieve the release version and build number."; + return false; + } + + ShorebirdConfigArgs shorebird_args(code_cache_path, code_cache_path, app_path, + shorebird_yaml_contents, release_version); + return ConfigureShorebird(shorebird_args, patch_path); +} + bool FlutterWindowsEngine::Run(std::string_view entrypoint) { + std::string assets_path_string = project_->assets_path().u8string(); + std::string icu_path_string = project_->icu_path().u8string(); + if (!project_->HasValidPaths()) { FML_LOG(ERROR) << "Missing or unresolvable paths to assets."; return false; } std::string assets_path_string = fml::PathToUtf8(project_->assets_path()); std::string icu_path_string = fml::PathToUtf8(project_->icu_path()); + + std::string patch_path; + auto setup_shorebird_result = SetUpShorebird(assets_path_string, patch_path); + if (setup_shorebird_result) { + // If we have a patch installed, we replace the default AOT library path + // with the patch path here. + FML_LOG(INFO) << "Setting project patch path: " << patch_path; + project_->SetAotLibraryPath(patch_path); + } else { + FML_LOG(ERROR) << "Failed to configure Shorebird."; + } + + // This loads AOT data from the project_'s aot_library_path_. if (embedder_api_.RunsAOTCompiledDartCode()) { aot_data_ = project_->LoadAotData(embedder_api_); if (!aot_data_) { @@ -452,6 +593,15 @@ bool FlutterWindowsEngine::Run(std::string_view entrypoint) { host->root_isolate_create_callback_(); } }; + // Copied from shell\platform\darwin\macos\framework\Source\FlutterEngine.mm + // Writes log messages to stdout. + args.log_message_callback = [](const char* tag, const char* message, + void* user_data) { + if (tag && tag[0]) { + std::cout << tag << ": "; + } + std::cout << message << std::endl; + }; args.channel_update_callback = [](const FlutterChannelUpdate* update, void* user_data) { auto host = static_cast(user_data); diff --git a/engine/src/flutter/sky/tools/create_ios_framework.py b/engine/src/flutter/sky/tools/create_ios_framework.py index d87e0f91bff49..0ead267e9609e 100644 --- a/engine/src/flutter/sky/tools/create_ios_framework.py +++ b/engine/src/flutter/sky/tools/create_ios_framework.py @@ -19,7 +19,8 @@ def main(): parser = argparse.ArgumentParser( description=( 'Creates Flutter.framework, Flutter.xcframework and ' - 'copies architecture-dependent gen_snapshot binaries to output dir' + 'copies architecture-dependent analyze_snapshot and gen_snapshot ' + 'binaries to output dir' ) ) @@ -89,13 +90,17 @@ def main(): '%s_extension_safe' % simulator_x64_out_dir, '%s_extension_safe' % simulator_arm64_out_dir ) - # Copy gen_snapshot binary to destination directory. + # Copy analyze_snapshot and gen_snapshot binaries to destination directory. if arm64_out_dir: gen_snapshot = os.path.join(arm64_out_dir, 'universal', 'gen_snapshot_arm64') + analyze_snapshot = os.path.join(arm64_out_dir, 'analyze_snapshot_arm64') sky_utils.copy_binary(gen_snapshot, os.path.join(dst, 'gen_snapshot_arm64')) + sky_utils.copy_binary(analyze_snapshot, os.path.join(dst, 'analyze_snapshot_arm64')) if x64_out_dir: gen_snapshot = os.path.join(x64_out_dir, 'universal', 'gen_snapshot_x64') + analyze_snapshot = os.path.join(x64_out_dir, 'analyze_snapshot_x64') sky_utils.copy_binary(gen_snapshot, os.path.join(dst, 'gen_snapshot_x64')) + sky_utils.copy_binary(analyze_snapshot, os.path.join(dst, 'analyze_snapshot_x64')) zip_archive(dst, args) return 0 @@ -177,7 +182,7 @@ def zip_archive(dst, args): # See: https://github.com/flutter/flutter/blob/62382c7b83a16b3f48dc06c19a47f6b8667005a5/dev/bots/suite_runners/run_verify_binaries_codesigned_tests.dart#L82-L130 # Binaries that must be codesigned and require entitlements for particular APIs. - with_entitlements = ['gen_snapshot_arm64'] + with_entitlements = ['analyze_snapshot_arm64', 'gen_snapshot_arm64'] with_entitlements_file = os.path.join(dst, 'entitlements.txt') sky_utils.write_codesign_config(with_entitlements_file, with_entitlements) @@ -211,6 +216,7 @@ def zip_archive(dst, args): # pylint: enable=line-too-long zip_contents = [ + 'analyze_snapshot_arm64', 'gen_snapshot_arm64', 'Flutter.xcframework', 'entitlements.txt', diff --git a/engine/src/flutter/testing/run_tests.py b/engine/src/flutter/testing/run_tests.py index 32dd2ba23a047..388479f61f107 100755 --- a/engine/src/flutter/testing/run_tests.py +++ b/engine/src/flutter/testing/run_tests.py @@ -485,6 +485,7 @@ def make_test( make_test('platform_view_android_delegate_unittests'), # https://github.com/flutter/flutter/issues/36295 make_test('shell_unittests'), + make_test('shorebird_unittests'), ] if is_windows(): diff --git a/packages/flutter_tools/gradle/build.gradle.kts b/packages/flutter_tools/gradle/build.gradle.kts index dd7341f6c0ee5..2147d6b2da444 100644 --- a/packages/flutter_tools/gradle/build.gradle.kts +++ b/packages/flutter_tools/gradle/build.gradle.kts @@ -67,6 +67,7 @@ dependencies { // * ndkVersion in FlutterExtension in packages/flutter_tools/gradle/src/main/kotlin/FlutterExtension.kt compileOnly("com.android.tools.build:gradle:8.11.1") + implementation("org.yaml:snakeyaml:2.0") testImplementation(kotlin("test")) testImplementation("com.android.tools.build:gradle:8.11.1") testImplementation("org.mockito:mockito-core:5.8.0") diff --git a/packages/flutter_tools/lib/src/base/build.dart b/packages/flutter_tools/lib/src/base/build.dart index 8c724e68b76d3..db63246b9f29b 100644 --- a/packages/flutter_tools/lib/src/base/build.dart +++ b/packages/flutter_tools/lib/src/base/build.dart @@ -8,7 +8,6 @@ import '../artifacts.dart'; import '../build_info.dart'; import '../darwin/darwin.dart'; import '../macos/xcode.dart'; - import 'file_system.dart'; import 'logger.dart'; import 'process.dart'; @@ -130,7 +129,28 @@ class AOTSnapshotter { final Directory outputDir = _fileSystem.directory(outputPath); outputDir.createSync(recursive: true); - final genSnapshotArgs = ['--deterministic']; + // Currently we only use the linker on iOS, but we will eventually split out + // the concept of "optimizes patch snapshot" from "uses linker" and probably + // only uses the linker on iOS, but optimize patch snapshots everywhere. + // TODO(eseidel): TargetPlatform.darwin doesn't use the linker. + bool usesLinker = (platform == TargetPlatform.ios || platform == TargetPlatform.darwin); + final dumpLinkInfoArgs = [ + // Shorebird dumps the class table information during snapshot compilation which is later used during linking. + '--print_class_table_link_debug_info_to=${_fileSystem.path.join(outputDir.parent.path, 'App.class_table.json')}', + '--print_class_table_link_info_to=${_fileSystem.path.join(outputDir.parent.path, 'App.ct.link')}', + '--print_field_table_link_debug_info_to=${_fileSystem.path.join(outputDir.parent.path, 'App.field_table.json')}', + '--print_field_table_link_info_to=${_fileSystem.path.join(outputDir.parent.path, 'App.ft.link')}', + '--print_dispatch_table_link_debug_info_to=${_fileSystem.path.join(outputDir.parent.path, 'App.dispatch_table.json')}', + '--print_dispatch_table_link_info_to=${_fileSystem.path.join(outputDir.parent.path, 'App.dt.link')}', + ]; + + final genSnapshotArgs = [ + // Shorebird uses --deterministic to improve snapshot stability and increase linking. + '--deterministic', + // Only save LinkInfo if we're using the linker. + if (usesLinker) + ...dumpLinkInfoArgs, + ]; final bool targetingApplePlatform = platform == TargetPlatform.ios || platform == TargetPlatform.darwin; diff --git a/packages/flutter_tools/lib/src/build_system/targets/assets.dart b/packages/flutter_tools/lib/src/build_system/targets/assets.dart index dcbe48c2a6015..4c7eb44cd87e6 100644 --- a/packages/flutter_tools/lib/src/build_system/targets/assets.dart +++ b/packages/flutter_tools/lib/src/build_system/targets/assets.dart @@ -13,6 +13,7 @@ import '../../dart/package_map.dart'; import '../../devfs.dart'; import '../../flutter_manifest.dart'; import '../../isolated/native_assets/dart_hook_result.dart'; +import '../../shorebird/shorebird_yaml.dart'; import '../build_system.dart'; import '../depfile.dart'; import '../exceptions.dart'; @@ -188,6 +189,19 @@ Future copyAssets( } if (doCopy) { await (content.file as File).copy(file.path); + if (file.basename == 'shorebird.yaml') { + try { + updateShorebirdYaml( + environment.defines[kFlavor], + file.path, + environment: globals.platform.environment, + ); + } on Exception catch (error) { + throw Exception( + 'Failed to generate shorebird configuration. Error: $error', + ); + } + } } } else { await file.writeAsBytes(await entry.value.content.contentsAsBytes()); diff --git a/packages/flutter_tools/lib/src/build_system/targets/common.dart b/packages/flutter_tools/lib/src/build_system/targets/common.dart index 692bc456838cb..3c26b8548d684 100644 --- a/packages/flutter_tools/lib/src/build_system/targets/common.dart +++ b/packages/flutter_tools/lib/src/build_system/targets/common.dart @@ -530,3 +530,38 @@ abstract final class Lipo { } } } + +/// For managing the supplementary linking files for Shorebird. +abstract final class LinkSupplement { + static Future create( + Environment environment, { + required String inputBuildDir, + required String outputBuildDir, + }) async { + // If the shorebird directory exists, delete it first. + final Directory shorebirdDir = environment.fileSystem.directory( + environment.fileSystem.path.join(outputBuildDir, 'shorebird'), + ); + if (shorebirdDir.existsSync()) { + shorebirdDir.deleteSync(recursive: true); + } + + void maybeCopy(String name) { + final File file = environment.fileSystem.file( + environment.fileSystem.path.join(inputBuildDir, name), + ); + if (file.existsSync()) { + file.copySync(environment.fileSystem.path.join(shorebirdDir.path, name)); + } + } + + // Copy the link information (generated by gen_snapshot) + // into the shorebird directory. + maybeCopy('App.ct.link'); + maybeCopy('App.class_table.json'); + maybeCopy('App.dt.link'); + maybeCopy('App.dispatch_table.json'); + maybeCopy('App.ft.link'); + maybeCopy('App.field_table.json'); + } +} diff --git a/packages/flutter_tools/lib/src/build_system/targets/ios.dart b/packages/flutter_tools/lib/src/build_system/targets/ios.dart index 82a470f2b721e..53a063442c926 100644 --- a/packages/flutter_tools/lib/src/build_system/targets/ios.dart +++ b/packages/flutter_tools/lib/src/build_system/targets/ios.dart @@ -143,6 +143,12 @@ abstract class AotAssemblyBase extends Target { // Don't fail if the dSYM wasn't created (i.e. during a debug build). skipMissingInputs: true, ); + + await LinkSupplement.create( + environment, + inputBuildDir: buildOutputPath, + outputBuildDir: getIosBuildDirectory(), + ); } } diff --git a/packages/flutter_tools/lib/src/build_system/targets/macos.dart b/packages/flutter_tools/lib/src/build_system/targets/macos.dart index c0f2923a5938e..bb4081cdcea3a 100644 --- a/packages/flutter_tools/lib/src/build_system/targets/macos.dart +++ b/packages/flutter_tools/lib/src/build_system/targets/macos.dart @@ -353,6 +353,12 @@ class CompileMacOSFramework extends Target { // Don't fail if the dSYM wasn't created (i.e. during a debug build). skipMissingInputs: true, ); + + await LinkSupplement.create( + environment, + inputBuildDir: buildOutputPath, + outputBuildDir: getMacOSBuildDirectory(), + ); } @override diff --git a/packages/flutter_tools/lib/src/cache.dart b/packages/flutter_tools/lib/src/cache.dart index ec17a559048ae..85db533f39c1f 100644 --- a/packages/flutter_tools/lib/src/cache.dart +++ b/packages/flutter_tools/lib/src/cache.dart @@ -40,6 +40,7 @@ import 'base/utils.dart' show getElapsedAsSeconds, getSizeAsPlatformMB; import 'convert.dart'; import 'features.dart'; +const kShorebirdStorageUrl = 'https://download.shorebird.dev'; const kFlutterRootEnvironmentVariableName = 'FLUTTER_ROOT'; // should point to //flutter/ (root of flutter/flutter repo) const kFlutterEngineEnvironmentVariableName = @@ -546,6 +547,10 @@ class Cache { ? 'https://storage.googleapis.com' : 'https://storage.googleapis.com/$storageRealm'; } + // Shorebird's artifact proxy is a trusted source. + if (overrideUrl == kShorebirdStorageUrl) { + return overrideUrl; + } // verify that this is a valid URI. overrideUrl = storageRealm.isEmpty ? overrideUrl : '$overrideUrl/$storageRealm'; try { diff --git a/packages/flutter_tools/lib/src/ios/application_package.dart b/packages/flutter_tools/lib/src/ios/application_package.dart index f1c8f44849548..e8613d4d7a8bd 100644 --- a/packages/flutter_tools/lib/src/ios/application_package.dart +++ b/packages/flutter_tools/lib/src/ios/application_package.dart @@ -130,6 +130,18 @@ class BuildableIOSApp extends IOSApp { @override String? get name => _appProductName; + String get shorebirdYamlPath => + globals.fs.path.join( + archiveBundleOutputPath, + 'Products', + 'Applications', + _appProductName != null ? '$_appProductName.app' : 'Runner.app', + 'Frameworks', + 'App.framework', + 'flutter_assets', + 'shorebird.yaml', + ); + @override String get simulatorBundlePath => _buildAppPath(XcodeSdk.IPhoneSimulator.platformName); diff --git a/packages/flutter_tools/lib/src/shorebird/shorebird_yaml.dart b/packages/flutter_tools/lib/src/shorebird/shorebird_yaml.dart new file mode 100644 index 0000000000000..9e40f392b7eb8 --- /dev/null +++ b/packages/flutter_tools/lib/src/shorebird/shorebird_yaml.dart @@ -0,0 +1,64 @@ +// Copyright 2024 The Flutter Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +import 'package:yaml/yaml.dart'; +import 'package:yaml_edit/yaml_edit.dart'; + +import '../base/file_system.dart'; +import '../globals.dart' as globals; + +void updateShorebirdYaml(String? flavor, String shorebirdYamlPath, {required Map environment}) { + final File shorebirdYaml = globals.fs.file(shorebirdYamlPath); + if (!shorebirdYaml.existsSync()) { + throw Exception('shorebird.yaml not found at $shorebirdYamlPath'); + } + final YamlDocument input = loadYamlDocument(shorebirdYaml.readAsStringSync()); + final YamlMap yamlMap = input.contents as YamlMap; + final Map compiled = compileShorebirdYaml(yamlMap, flavor: flavor, environment: environment); + // Currently we write out over the same yaml file, we should fix this to + // write to a new .json file instead and avoid naming confusion between the + // input and compiled files. + final YamlEditor yamlEditor = YamlEditor(''); + yamlEditor.update([], compiled); + shorebirdYaml.writeAsStringSync(yamlEditor.toString(), flush: true); +} + +String appIdForFlavor(YamlMap yamlMap, {required String? flavor}) { + if (flavor == null || flavor.isEmpty) { + final String? defaultAppId = yamlMap['app_id'] as String?; + if (defaultAppId == null || defaultAppId.isEmpty) { + throw Exception('Cannot find "app_id" in shorebird.yaml'); + } + return defaultAppId; + } + + final YamlMap? yamlFlavors = yamlMap['flavors'] as YamlMap?; + if (yamlFlavors == null) { + throw Exception('Cannot find "flavors" in shorebird.yaml.'); + } + final String? flavorAppId = yamlFlavors[flavor] as String?; + if (flavorAppId == null || flavorAppId.isEmpty) { + throw Exception('Cannot find "app_id" for $flavor in shorebird.yaml'); + } + return flavorAppId; +} + +Map compileShorebirdYaml(YamlMap yamlMap, {required String? flavor, required Map environment}) { + final String appId = appIdForFlavor(yamlMap, flavor: flavor); + final Map compiled = { + 'app_id': appId, + }; + void copyIfSet(String key) { + if (yamlMap[key] != null) { + compiled[key] = yamlMap[key]; + } + } + copyIfSet('base_url'); + copyIfSet('auto_update'); + final String? shorebirdPublicKeyEnvVar = environment['SHOREBIRD_PUBLIC_KEY']; + if (shorebirdPublicKeyEnvVar != null) { + compiled['patch_public_key'] = shorebirdPublicKeyEnvVar; + } + return compiled; +} diff --git a/packages/flutter_tools/lib/src/version.dart b/packages/flutter_tools/lib/src/version.dart index 16d1c2544172c..6a654f5c642f9 100644 --- a/packages/flutter_tools/lib/src/version.dart +++ b/packages/flutter_tools/lib/src/version.dart @@ -1086,6 +1086,19 @@ class GitTagVersion { } } + // Check if running on a Shorebird release branch. + final String shorebirdFlutterReleases = _runGit( + 'git for-each-ref --contains $gitRef --format %(refname:short) refs/remotes/origin/flutter_release/*', + processUtils, + workingDirectory, + ).trim(); + final String? shorebirdFlutterVersion = LineSplitter.split( + shorebirdFlutterReleases, + ).map((e) => e.replaceFirst('origin/flutter_release/', '')).toList().firstOrNull; + if (shorebirdFlutterVersion != null) { + return parse(shorebirdFlutterVersion); + } + // If we don't exist in a tag, use git to find the latest tag. return _useNewestTagAndCommitsPastFallback( git: git, diff --git a/packages/flutter_tools/lib/src/windows/build_windows.dart b/packages/flutter_tools/lib/src/windows/build_windows.dart index 70ae440fc1b51..9bacc583560d4 100644 --- a/packages/flutter_tools/lib/src/windows/build_windows.dart +++ b/packages/flutter_tools/lib/src/windows/build_windows.dart @@ -21,6 +21,7 @@ import '../flutter_plugins.dart'; import '../globals.dart' as globals; import '../migrations/cmake_custom_command_migration.dart'; import '../migrations/cmake_native_assets_migration.dart'; +import '../shorebird/shorebird_yaml.dart'; import 'migrations/build_architecture_migration.dart'; import 'migrations/show_window_migration.dart'; import 'migrations/version_migration.dart'; diff --git a/packages/flutter_tools/test/commands.shard/hermetic/build_macos_test.dart b/packages/flutter_tools/test/commands.shard/hermetic/build_macos_test.dart index 561b2a9c6d174..11c5f6125ac09 100644 --- a/packages/flutter_tools/test/commands.shard/hermetic/build_macos_test.dart +++ b/packages/flutter_tools/test/commands.shard/hermetic/build_macos_test.dart @@ -98,6 +98,14 @@ final macosPlatformCustomEnv = FakePlatform( environment: {'FLUTTER_ROOT': '/', 'HOME': '/'}, ); +final Platform macosPlatformWithShorebirdPublicKey = FakePlatform( + operatingSystem: 'macos', + environment: { + 'FLUTTER_ROOT': '/', + 'HOME': '/', + 'SHOREBIRD_PUBLIC_KEY': 'my_public_key', + } +); final Platform notMacosPlatform = FakePlatform(environment: {'FLUTTER_ROOT': '/'}); void main() { diff --git a/packages/flutter_tools/test/commands.shard/hermetic/build_windows_test.dart b/packages/flutter_tools/test/commands.shard/hermetic/build_windows_test.dart index 11c574670713a..25da553b819e3 100644 --- a/packages/flutter_tools/test/commands.shard/hermetic/build_windows_test.dart +++ b/packages/flutter_tools/test/commands.shard/hermetic/build_windows_test.dart @@ -38,6 +38,13 @@ final Platform windowsPlatform = FakePlatform( 'USERPROFILE': '/', }, ); +final Platform windowsPlatformWithPublicKey = + FakePlatform(operatingSystem: 'windows', environment: { + 'PROGRAMFILES(X86)': r'C:\Program Files (x86)\', + 'FLUTTER_ROOT': flutterRoot, + 'USERPROFILE': '/', + 'SHOREBIRD_PUBLIC_KEY': 'my_public_key', +}); final Platform notWindowsPlatform = FakePlatform( environment: {'FLUTTER_ROOT': flutterRoot}, ); @@ -115,7 +122,9 @@ void main() { ...['--target', 'INSTALL'], if (verbose) '--verbose', ], - environment: {if (verbose) 'VERBOSE_SCRIPT_LOGGING': 'true'}, + environment: { + if (verbose) 'VERBOSE_SCRIPT_LOGGING': 'true' + }, onRun: onRun, stdout: stdout, ); @@ -131,7 +140,8 @@ void main() { setUpMockProjectFilesForBuild(); expect( - createTestCommandRunner(command).run(const ['windows', '--no-pub']), + createTestCommandRunner(command) + .run(const ['windows', '--no-pub']), throwsToolExit(), ); }, @@ -154,10 +164,10 @@ void main() { setUpMockCoreProjectFiles(); expect( - createTestCommandRunner(command).run(const ['windows', '--no-pub']), + createTestCommandRunner(command) + .run(const ['windows', '--no-pub']), throwsToolExit( - message: - 'No Windows desktop project configured. See ' + message: 'No Windows desktop project configured. See ' 'https://flutter.dev/to/add-desktop-support ' 'to learn about adding Windows support to a project.', ), @@ -182,8 +192,10 @@ void main() { setUpMockProjectFilesForBuild(); expect( - createTestCommandRunner(command).run(const ['windows', '--no-pub']), - throwsToolExit(message: '"build windows" only supported on Windows hosts.'), + createTestCommandRunner(command) + .run(const ['windows', '--no-pub']), + throwsToolExit( + message: '"build windows" only supported on Windows hosts.'), ); }, overrides: { @@ -205,7 +217,8 @@ void main() { setUpMockProjectFilesForBuild(); expect( - createTestCommandRunner(command).run(const ['windows', '--no-pub']), + createTestCommandRunner(command) + .run(const ['windows', '--no-pub']), throwsToolExit( message: '"build windows" is not currently supported. To enable, run "flutter config --enable-windows-desktop".', @@ -235,7 +248,8 @@ void main() { buildCommand('Release', stdout: 'STDOUT STUFF'), ]); - await createTestCommandRunner(command).run(const ['windows', '--no-pub']); + await createTestCommandRunner(command) + .run(const ['windows', '--no-pub']); expect(testLogger.statusText, isNot(contains('STDOUT STUFF'))); expect(testLogger.traceText, contains('STDOUT STUFF')); }, @@ -262,7 +276,8 @@ void main() { buildCommand('Release'), ]); - await createTestCommandRunner(command).run(const ['windows', '--no-pub']); + await createTestCommandRunner(command) + .run(const ['windows', '--no-pub']); expect( analyticsTimingEventExists( @@ -332,7 +347,8 @@ C:\foo\windows\x64\runner\main.cpp(17,1): error C2065: 'Baz': undeclared identif buildCommand('Release', stdout: stdout), ]); - await createTestCommandRunner(command).run(const ['windows', '--no-pub']); + await createTestCommandRunner(command) + .run(const ['windows', '--no-pub']); // Just the warnings and errors should be surfaced. expect(testLogger.errorText, r''' C:\foo\windows\x64\runner\main.cpp(18): error C2220: the following warning is treated as an error [C:\foo\build\windows\x64\runner\test.vcxproj] @@ -365,7 +381,8 @@ C:\foo\windows\x64\runner\main.cpp(17,1): error C2065: 'Baz': undeclared identif buildCommand('Release', verbose: true, stdout: 'STDOUT STUFF'), ]); - await createTestCommandRunner(command).run(const ['windows', '--no-pub', '-v']); + await createTestCommandRunner(command) + .run(const ['windows', '--no-pub', '-v']); expect(testLogger.statusText, contains('STDOUT STUFF')); expect(testLogger.traceText, isNot(contains('STDOUT STUFF'))); }, @@ -485,7 +502,8 @@ if %errorlevel% neq 0 goto :VCEnd assembleProject.createSync(recursive: true); assembleProject.writeAsStringSync(fakeBadProjectContent); - await createTestCommandRunner(command).run(const ['windows', '--no-pub']); + await createTestCommandRunner(command) + .run(const ['windows', '--no-pub']); final List projectLines = assembleProject.readAsLinesSync(); @@ -645,7 +663,8 @@ if %errorlevel% neq 0 goto :VCEnd await createTestCommandRunner( command, ).run(const ['windows', '--release', '--no-pub']); - expect(testLogger.statusText, contains(r'โœ“ Built build\windows\x64\runner\Release')); + expect(testLogger.statusText, + contains(r'โœ“ Built build\windows\x64\runner\Release')); }, overrides: { FileSystem: () => fileSystem, @@ -702,7 +721,8 @@ if %errorlevel% neq 0 goto :VCEnd buildCommand('Release'), ]); - await createTestCommandRunner(command).run(const ['windows', '--no-pub']); + await createTestCommandRunner(command) + .run(const ['windows', '--no-pub']); final File cmakeConfig = fileSystem.currentDirectory .childDirectory('windows') @@ -750,7 +770,12 @@ if %errorlevel% neq 0 goto :VCEnd await createTestCommandRunner( command, - ).run(const ['windows', '--no-pub', '--build-name=1.2.3', '--build-number=4']); + ).run(const [ + 'windows', + '--no-pub', + '--build-name=1.2.3', + '--build-number=4' + ]); final File cmakeConfig = fileSystem.currentDirectory .childDirectory('windows') @@ -906,7 +931,12 @@ if %errorlevel% neq 0 goto :VCEnd await createTestCommandRunner( command, - ).run(const ['windows', '--no-pub', '--build-name=1.2.3', '--build-number=4']); + ).run(const [ + 'windows', + '--no-pub', + '--build-name=1.2.3', + '--build-number=4' + ]); final File cmakeConfig = fileSystem.currentDirectory .childDirectory('windows') @@ -954,7 +984,12 @@ if %errorlevel% neq 0 goto :VCEnd await createTestCommandRunner( command, - ).run(const ['windows', '--no-pub', '--build-name=1.2.3', '--build-number=hello']); + ).run(const [ + 'windows', + '--no-pub', + '--build-name=1.2.3', + '--build-number=hello' + ]); final File cmakeConfig = fileSystem.currentDirectory .childDirectory('windows') @@ -1011,7 +1046,12 @@ if %errorlevel% neq 0 goto :VCEnd await createTestCommandRunner( command, - ).run(const ['windows', '--no-pub', '--build-name=1.2.3', '--build-number=4.5']); + ).run(const [ + 'windows', + '--no-pub', + '--build-name=1.2.3', + '--build-number=4.5' + ]); final File cmakeConfig = fileSystem.currentDirectory .childDirectory('windows') @@ -1131,14 +1171,16 @@ if %errorlevel% neq 0 goto :VCEnd contains('A summary of your Windows bundle analysis can be found at'), ); expect(testLogger.statusText, contains('dart devtools --appSizeBase=')); - expect(fakeAnalytics.sentEvents, contains(Event.codeSizeAnalysis(platform: 'windows'))); + expect(fakeAnalytics.sentEvents, + contains(Event.codeSizeAnalysis(platform: 'windows'))); }, overrides: { FeatureFlags: () => TestFeatureFlags(isWindowsEnabled: true), FileSystem: () => fileSystem, ProcessManager: () => processManager, Platform: () => windowsPlatform, - FileSystemUtils: () => FileSystemUtils(fileSystem: fileSystem, platform: windowsPlatform), + FileSystemUtils: () => + FileSystemUtils(fileSystem: fileSystem, platform: windowsPlatform), Analytics: () => fakeAnalytics, }, ); @@ -1155,12 +1197,14 @@ if %errorlevel% neq 0 goto :VCEnd logger: BufferLogger.test(), operatingSystemUtils: FakeOperatingSystemUtils(), )..visualStudioOverride = fakeVisualStudio; - fileSystem.currentDirectory = fileSystem.directory("test_'path")..createSync(); + fileSystem.currentDirectory = fileSystem.directory("test_'path") + ..createSync(); final String absPath = fileSystem.currentDirectory.absolute.path; setUpMockCoreProjectFiles(); expect( - createTestCommandRunner(command).run(const ['windows', '--no-pub']), + createTestCommandRunner(command) + .run(const ['windows', '--no-pub']), throwsToolExit( message: 'Path $absPath contains invalid characters in "\'#!\$^&*=|,;<>?". ' @@ -1199,7 +1243,8 @@ No file or variants found for asset: images/a_dot_burr.jpeg. buildCommand('Release', stdout: stdout), ]); - await createTestCommandRunner(command).run(const ['windows', '--no-pub']); + await createTestCommandRunner(command) + .run(const ['windows', '--no-pub']); // Just the warnings and errors should be surfaced. expect(testLogger.errorText, r''' Error detected in pubspec.yaml: @@ -1213,6 +1258,39 @@ No file or variants found for asset: images/a_dot_burr.jpeg. FeatureFlags: () => TestFeatureFlags(isWindowsEnabled: true), }, ); + + testUsingContext( + 'shorebird.yaml is updated when SHOREBIRD_PUBLIC_KEY env var is set', + () async { + final FakeVisualStudio fakeVisualStudio = FakeVisualStudio(); + final BuildWindowsCommand command = BuildWindowsCommand( + logger: BufferLogger.test(), + operatingSystemUtils: FakeOperatingSystemUtils()) + ..visualStudioOverride = fakeVisualStudio; + setUpMockProjectFilesForBuild(); + final File shorebirdYamlFile = fileSystem.file( + r'build\windows\x64\runner\Release\data\flutter_assets\shorebird.yaml', + ) + ..createSync(recursive: true) + ..writeAsStringSync('app_id: my-app-id'); + + processManager = FakeProcessManager.list([ + cmakeGenerationCommand(), + buildCommand('Release'), + ]); + + await createTestCommandRunner(command) + .run(const ['windows', '--release', '--no-pub']); + + final String updatedYaml = shorebirdYamlFile.readAsStringSync(); + expect(updatedYaml, contains('app_id: my-app-id')); + expect(updatedYaml, contains('patch_public_key: my_public_key')); + }, overrides: { + FileSystem: () => fileSystem, + ProcessManager: () => processManager, + Platform: () => windowsPlatformWithPublicKey, + FeatureFlags: () => TestFeatureFlags(isWindowsEnabled: true), + }); } class FakeVisualStudio extends Fake implements VisualStudio { diff --git a/packages/flutter_tools/test/general.shard/build_system/targets/macos_test.dart b/packages/flutter_tools/test/general.shard/build_system/targets/macos_test.dart index a580bb35f82e6..ec6ea8dfc47c3 100644 --- a/packages/flutter_tools/test/general.shard/build_system/targets/macos_test.dart +++ b/packages/flutter_tools/test/general.shard/build_system/targets/macos_test.dart @@ -145,7 +145,14 @@ void main() { ); lipoExtractX86_64Command = FakeCommand( - command: ['lipo', '-output', binary.path, '-extract', 'x86_64', binary.path], + command: [ + 'lipo', + '-output', + binary.path, + '-extract', + 'x86_64', + binary.path + ], ); }); @@ -232,7 +239,8 @@ void main() { isException.having( (Exception exception) => exception.toString(), 'description', - contains('FlutterMacOS.framework/Versions/A/FlutterMacOS does not exist, cannot thin'), + contains( + 'FlutterMacOS.framework/Versions/A/FlutterMacOS does not exist, cannot thin'), ), ), ); @@ -285,7 +293,8 @@ void main() { expect( logger.traceText, - contains('Skipping lipo for non-fat file /FlutterMacOS.framework/Versions/A/FlutterMacOS'), + contains( + 'Skipping lipo for non-fat file /FlutterMacOS.framework/Versions/A/FlutterMacOS'), ); }); @@ -313,7 +322,8 @@ void main() { lipoVerifyX86_64Command, ]); - await const ReleaseUnpackMacOS().build(environment..defines[kBuildMode] = 'release'); + await const ReleaseUnpackMacOS() + .build(environment..defines[kBuildMode] = 'release'); expect(processManager, hasNoRemainingExpectations); }, @@ -335,7 +345,8 @@ void main() { copyFrameworkDsymCommand, ]); - await const ReleaseUnpackMacOS().build(environment..defines[kBuildMode] = 'release'); + await const ReleaseUnpackMacOS() + .build(environment..defines[kBuildMode] = 'release'); expect(processManager, hasNoRemainingExpectations); }, @@ -372,7 +383,8 @@ void main() { ]); await expectLater( - const ReleaseUnpackMacOS().build(environment..defines[kBuildMode] = 'release'), + const ReleaseUnpackMacOS() + .build(environment..defines[kBuildMode] = 'release'), throwsA( isException.having( (Exception exception) => exception.toString(), @@ -393,15 +405,19 @@ void main() { () async { fileSystem .directory( - artifacts.getArtifactPath(Artifact.flutterMacOSFramework, mode: BuildMode.debug), + artifacts.getArtifactPath(Artifact.flutterMacOSFramework, + mode: BuildMode.debug), ) .createSync(); - final String inputKernel = fileSystem.path.join(environment.buildDir.path, 'app.dill'); + final String inputKernel = + fileSystem.path.join(environment.buildDir.path, 'app.dill'); fileSystem.file(inputKernel) ..createSync(recursive: true) ..writeAsStringSync('testing'); - expect(() async => const DebugMacOSBundleFlutterAssets().build(environment), throwsException); + expect( + () async => const DebugMacOSBundleFlutterAssets().build(environment), + throwsException); }, overrides: { FileSystem: () => fileSystem, @@ -414,7 +430,8 @@ void main() { () async { fileSystem .directory( - artifacts.getArtifactPath(Artifact.flutterMacOSFramework, mode: BuildMode.debug), + artifacts.getArtifactPath(Artifact.flutterMacOSFramework, + mode: BuildMode.debug), ) .createSync(); fileSystem @@ -447,20 +464,25 @@ void main() { expect( fileSystem - .file('App.framework/Versions/A/Resources/flutter_assets/kernel_blob.bin') + .file( + 'App.framework/Versions/A/Resources/flutter_assets/kernel_blob.bin') .readAsStringSync(), 'testing', ); expect( - fileSystem.file('App.framework/Versions/A/Resources/Info.plist').readAsStringSync(), + fileSystem + .file('App.framework/Versions/A/Resources/Info.plist') + .readAsStringSync(), contains('io.flutter.flutter.app'), ); expect( - fileSystem.file('App.framework/Versions/A/Resources/flutter_assets/vm_snapshot_data'), + fileSystem.file( + 'App.framework/Versions/A/Resources/flutter_assets/vm_snapshot_data'), exists, ); expect( - fileSystem.file('App.framework/Versions/A/Resources/flutter_assets/isolate_snapshot_data'), + fileSystem.file( + 'App.framework/Versions/A/Resources/flutter_assets/isolate_snapshot_data'), exists, ); }, @@ -563,23 +585,30 @@ void main() { fileSystem .file('bin/cache/artifacts/engine/darwin-x64/isolate_snapshot.bin') .createSync(recursive: true); - fileSystem.file('${environment.buildDir.path}/App.framework/App').createSync(recursive: true); - fileSystem.file('${environment.buildDir.path}/native_assets.json').createSync(); + fileSystem + .file('${environment.buildDir.path}/App.framework/App') + .createSync(recursive: true); + fileSystem + .file('${environment.buildDir.path}/native_assets.json') + .createSync(); await const ProfileMacOSBundleFlutterAssets().build( environment..defines[kBuildMode] = 'profile', ); expect( - fileSystem.file('App.framework/Versions/A/Resources/flutter_assets/kernel_blob.bin'), + fileSystem.file( + 'App.framework/Versions/A/Resources/flutter_assets/kernel_blob.bin'), isNot(exists), ); expect( - fileSystem.file('App.framework/Versions/A/Resources/flutter_assets/vm_snapshot_data'), + fileSystem.file( + 'App.framework/Versions/A/Resources/flutter_assets/vm_snapshot_data'), isNot(exists), ); expect( - fileSystem.file('App.framework/Versions/A/Resources/flutter_assets/isolate_snapshot_data'), + fileSystem.file( + 'App.framework/Versions/A/Resources/flutter_assets/isolate_snapshot_data'), isNot(exists), ); }, @@ -598,17 +627,23 @@ void main() { fileSystem .file('bin/cache/artifacts/engine/darwin-x64/isolate_snapshot.bin') .createSync(recursive: true); - fileSystem.file('${environment.buildDir.path}/App.framework/App').createSync(recursive: true); fileSystem - .file('${environment.buildDir.path}/App.framework.dSYM/Contents/Resources/DWARF/App') + .file('${environment.buildDir.path}/App.framework/App') + .createSync(recursive: true); + fileSystem + .file( + '${environment.buildDir.path}/App.framework.dSYM/Contents/Resources/DWARF/App') .createSync(recursive: true); - fileSystem.file('${environment.buildDir.path}/native_assets.json').createSync(); + fileSystem + .file('${environment.buildDir.path}/native_assets.json') + .createSync(); await const ReleaseMacOSBundleFlutterAssets().build( environment..defines[kBuildMode] = 'release', ); - expect(fileSystem.file('App.framework.dSYM/Contents/Resources/DWARF/App'), exists); + expect(fileSystem.file('App.framework.dSYM/Contents/Resources/DWARF/App'), + exists); }, overrides: { FileSystem: () => fileSystem, @@ -625,17 +660,20 @@ void main() { fileSystem .file('bin/cache/artifacts/engine/darwin-x64/isolate_snapshot.bin') .createSync(recursive: true); - final File inputFramework = - fileSystem.file(fileSystem.path.join(environment.buildDir.path, 'App.framework', 'App')) - ..createSync(recursive: true) - ..writeAsStringSync('ABC'); - fileSystem.file(environment.buildDir.childFile('native_assets.json')).createSync(); + final File inputFramework = fileSystem.file(fileSystem.path + .join(environment.buildDir.path, 'App.framework', 'App')) + ..createSync(recursive: true) + ..writeAsStringSync('ABC'); + fileSystem + .file(environment.buildDir.childFile('native_assets.json')) + .createSync(); await const ProfileMacOSBundleFlutterAssets().build( environment..defines[kBuildMode] = 'profile', ); final File outputFramework = fileSystem.file( - fileSystem.path.join(environment.outputDir.path, 'App.framework', 'App'), + fileSystem.path + .join(environment.outputDir.path, 'App.framework', 'App'), ); expect(outputFramework.readAsStringSync(), 'ABC'); @@ -666,9 +704,12 @@ void main() { .file('bin/cache/artifacts/engine/darwin-x64/isolate_snapshot.bin') .createSync(recursive: true); fileSystem - .file(fileSystem.path.join(environment.buildDir.path, 'App.framework', 'App')) + .file(fileSystem.path + .join(environment.buildDir.path, 'App.framework', 'App')) .createSync(recursive: true); - fileSystem.file(environment.buildDir.childFile('native_assets.json')).createSync(); + fileSystem + .file(environment.buildDir.childFile('native_assets.json')) + .createSync(); await const ReleaseMacOSBundleFlutterAssets().build(environment); expect( @@ -702,7 +743,8 @@ void main() { expect( fakeAnalytics.sentEvents, contains( - Event.appleUsageEvent(workflow: 'assemble', parameter: 'macos-archive', result: 'fail'), + Event.appleUsageEvent( + workflow: 'assemble', parameter: 'macos-archive', result: 'fail'), ), ); }, @@ -739,7 +781,10 @@ void main() { '-install_name', '@rpath/App.framework/App', '-o', - environment.buildDir.childDirectory('App.framework').childFile('App').path, + environment.buildDir + .childDirectory('App.framework') + .childFile('App') + .path, ], ), ); @@ -753,6 +798,54 @@ void main() { }, ); + testUsingContext( + 'ReleaseMacOSBundleFlutterAssets updates shorebird.yaml if present', + () async { + environment.defines[kBuildMode] = 'release'; + environment.defines[kXcodeAction] = 'install'; + environment.defines[kFlavor] = 'internal'; + + fileSystem + .file('bin/cache/artifacts/engine/darwin-x64/vm_isolate_snapshot.bin') + .createSync(recursive: true); + fileSystem + .file('bin/cache/artifacts/engine/darwin-x64/isolate_snapshot.bin') + .createSync(recursive: true); + fileSystem + .file(fileSystem.path + .join(environment.buildDir.path, 'App.framework', 'App')) + .createSync(recursive: true); + final String shorebirdYamlPath = fileSystem.path.join( + environment.buildDir.path, + 'App.framework', + 'Versions', + 'A', + 'Resources', + 'flutter_assets', + 'shorebird.yaml', + ); + fileSystem.file(fileSystem.path + .join(environment.buildDir.path, 'App.framework', 'App')) + ..createSync(recursive: true) + ..writeAsStringSync(''' +# Some other text that should be removed +app_id: base-app-id +flavors: + internal: internal-app-id + stable: stable-app-id +'''); + + await const ReleaseMacOSBundleFlutterAssets().build(environment); + + expect(fileSystem.file(shorebirdYamlPath).readAsStringSync(), + 'app_id: internal-app-id'); + }, + overrides: { + FileSystem: () => fileSystem, + ProcessManager: () => processManager, + }, + ); + testUsingContext( 'DebugMacOSFramework creates universal binary', () async { @@ -782,7 +875,10 @@ void main() { '-install_name', '@rpath/App.framework/App', '-o', - environment.buildDir.childDirectory('App.framework').childFile('App').path, + environment.buildDir + .childDirectory('App.framework') + .childFile('App') + .path, ], ), ); @@ -889,14 +985,18 @@ void main() { command: [ 'lipo', environment.buildDir - .childFile('arm64/App.framework.dSYM/Contents/Resources/DWARF/App') + .childFile( + 'arm64/App.framework.dSYM/Contents/Resources/DWARF/App') .path, environment.buildDir - .childFile('x86_64/App.framework.dSYM/Contents/Resources/DWARF/App') + .childFile( + 'x86_64/App.framework.dSYM/Contents/Resources/DWARF/App') .path, '-create', '-output', - environment.buildDir.childFile('App.framework.dSYM/Contents/Resources/DWARF/App').path, + environment.buildDir + .childFile('App.framework.dSYM/Contents/Resources/DWARF/App') + .path, ], ), ]); diff --git a/packages/flutter_tools/test/general.shard/shorebird/shorebird_yaml_test.dart b/packages/flutter_tools/test/general.shard/shorebird/shorebird_yaml_test.dart new file mode 100644 index 0000000000000..602de37681555 --- /dev/null +++ b/packages/flutter_tools/test/general.shard/shorebird/shorebird_yaml_test.dart @@ -0,0 +1,101 @@ +// Copyright 2024 The Flutter Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +import 'dart:io'; + +import 'package:flutter_tools/src/shorebird/shorebird_yaml.dart'; +import 'package:test/test.dart'; +import 'package:yaml/yaml.dart'; + +void main() { + group('ShorebirdYaml', () { + test('yaml ignores comments', () { + const String yamlContents = ''' +# This file is used to configure the Shorebird updater used by your app. +app_id: 6160a7d8-cc18-4928-1233-05b51c0bb02c + +# auto_update controls if Shorebird should automatically update in the background on launch. +auto_update: false +'''; + final YamlDocument input = loadYamlDocument(yamlContents); + final YamlMap yamlMap = input.contents as YamlMap; + final Map compiled = + compileShorebirdYaml(yamlMap, flavor: null, environment: {}); + expect(compiled, { + 'app_id': '6160a7d8-cc18-4928-1233-05b51c0bb02c', + 'auto_update': false, + }); + }); + test('flavors', () { + // These are invalid app_ids but make for easy testing. + const String yamlContents = ''' +app_id: 1-a +auto_update: false +flavors: + foo: 2-a + bar: 3-a +'''; + final YamlDocument input = loadYamlDocument(yamlContents); + final YamlMap yamlMap = input.contents as YamlMap; + expect(appIdForFlavor(yamlMap, flavor: null), '1-a'); + expect(appIdForFlavor(yamlMap, flavor: 'foo'), '2-a'); + expect(appIdForFlavor(yamlMap, flavor: 'bar'), '3-a'); + expect(() => appIdForFlavor(yamlMap, flavor: 'unknown'), throwsException); + }); + test('all values', () { + // These are invalid app_ids but make for easy testing. + const String yamlContents = ''' +app_id: 1-a +auto_update: false +flavors: + foo: 2-a + bar: 3-a +base_url: https://example.com +'''; + final YamlDocument input = loadYamlDocument(yamlContents); + final YamlMap yamlMap = input.contents as YamlMap; + final Map compiled1 = + compileShorebirdYaml(yamlMap, flavor: null, environment: {}); + expect(compiled1, { + 'app_id': '1-a', + 'auto_update': false, + 'base_url': 'https://example.com', + }); + final Map compiled2 = + compileShorebirdYaml(yamlMap, flavor: 'foo', environment: {'SHOREBIRD_PUBLIC_KEY': '4-a'}); + expect(compiled2, { + 'app_id': '2-a', + 'auto_update': false, + 'base_url': 'https://example.com', + 'patch_public_key': '4-a', + }); + }); + test('edit in place', () { + const String yamlContents = ''' +app_id: 1-a +auto_update: false +flavors: + foo: 2-a + bar: 3-a +base_url: https://example.com +'''; + // Make a temporary file to test editing in place. + final Directory tempDir = Directory.systemTemp.createTempSync('shorebird_yaml_test.'); + final File tempFile = File('${tempDir.path}/shorebird.yaml'); + tempFile.writeAsStringSync(yamlContents); + updateShorebirdYaml( + 'foo', + tempFile.path, + environment: {'SHOREBIRD_PUBLIC_KEY': '4-a'}, + ); + final String updatedContents = tempFile.readAsStringSync(); + // Order is not guaranteed, so parse as YAML to compare. + final YamlDocument updated = loadYamlDocument(updatedContents); + final YamlMap yamlMap = updated.contents as YamlMap; + expect(yamlMap['app_id'], '2-a'); + expect(yamlMap['auto_update'], false); + expect(yamlMap['base_url'], 'https://example.com'); + }); + }); +} diff --git a/packages/shorebird_tests/.gitignore b/packages/shorebird_tests/.gitignore new file mode 100644 index 0000000000000..3a85790408401 --- /dev/null +++ b/packages/shorebird_tests/.gitignore @@ -0,0 +1,3 @@ +# https://dart.dev/guides/libraries/private-files +# Created by `dart pub` +.dart_tool/ diff --git a/packages/shorebird_tests/README.md b/packages/shorebird_tests/README.md new file mode 100644 index 0000000000000..b87fb24a480da --- /dev/null +++ b/packages/shorebird_tests/README.md @@ -0,0 +1,2 @@ +A dart project that includes tests that perform asserts in the modifications +made on the Flutter framework by the Shorebird team. diff --git a/packages/shorebird_tests/analysis_options.yaml b/packages/shorebird_tests/analysis_options.yaml new file mode 100644 index 0000000000000..a767d79d7f4b1 --- /dev/null +++ b/packages/shorebird_tests/analysis_options.yaml @@ -0,0 +1,2 @@ +# This file configures the static analysis results for your project (errors, +include: package:lints/recommended.yaml diff --git a/packages/shorebird_tests/pubspec.yaml b/packages/shorebird_tests/pubspec.yaml new file mode 100644 index 0000000000000..d0fba0c1fa95d --- /dev/null +++ b/packages/shorebird_tests/pubspec.yaml @@ -0,0 +1,16 @@ +name: shorebird_tests +description: Shorebird's Flutter customizations tests +version: 1.0.0 + +environment: + sdk: ^3.3.4 + +dependencies: + archive: ^3.5.1 + path: ^1.9.0 + yaml: ^3.1.2 + +dev_dependencies: + lints: ^3.0.0 + meta: ^1.15.0 + test: ^1.24.0 diff --git a/packages/shorebird_tests/test/android_test.dart b/packages/shorebird_tests/test/android_test.dart new file mode 100644 index 0000000000000..b24cce0ee9bf6 --- /dev/null +++ b/packages/shorebird_tests/test/android_test.dart @@ -0,0 +1,92 @@ +import 'package:test/test.dart'; + +import 'shorebird_tests.dart'; + +void main() { + group('shorebird android projects', () { + testWithShorebirdProject('can build an apk', (projectDirectory) async { + await projectDirectory.runFlutterBuildApk(); + + expect(projectDirectory.apkFile().existsSync(), isTrue); + expect(projectDirectory.shorebirdFile.existsSync(), isTrue); + expect(projectDirectory.getGeneratedAndroidShorebirdYaml(), completes); + }); + + group('when passing the public key through the environment variable', () { + testWithShorebirdProject( + 'adds the public key on top of the original file', + (projectDirectory) async { + final originalYaml = projectDirectory.shorebirdYaml; + + const base64PublicKey = 'public_123'; + await projectDirectory.runFlutterBuildApk( + environment: { + 'SHOREBIRD_PUBLIC_KEY': base64PublicKey, + }, + ); + + final generatedYaml = + await projectDirectory.getGeneratedAndroidShorebirdYaml(); + + expect( + generatedYaml.keys, + containsAll(originalYaml.keys), + ); + + expect( + generatedYaml['patch_public_key'], + equals(base64PublicKey), + ); + }, + ); + }); + + group('when building with a flavor', () { + testWithShorebirdProject( + 'correctly changes the app id', + (projectDirectory) async { + await projectDirectory.addProjectFlavors(); + projectDirectory.addShorebirdFlavors(); + + await projectDirectory.runFlutterBuildApk(flavor: 'internal'); + + final generatedYaml = + await projectDirectory.getGeneratedAndroidShorebirdYaml( + flavor: 'internal', + ); + + expect(generatedYaml['app_id'], equals('internal_123')); + }, + ); + + group('when public key passed through environment variable', () { + testWithShorebirdProject( + 'correctly changes the app id and adds the public key', + (projectDirectory) async { + const base64PublicKey = 'public_123'; + await projectDirectory.addProjectFlavors(); + projectDirectory.addShorebirdFlavors(); + + await projectDirectory.runFlutterBuildApk( + flavor: 'internal', + environment: { + 'SHOREBIRD_PUBLIC_KEY': base64PublicKey, + }, + ); + + final generatedYaml = + await projectDirectory.getGeneratedAndroidShorebirdYaml( + flavor: 'internal', + ); + + expect(generatedYaml['app_id'], equals('internal_123')); + expect( + generatedYaml['patch_public_key'], + equals(base64PublicKey), + ); + }, + ); + }); + }); + }); +} diff --git a/packages/shorebird_tests/test/base_test.dart b/packages/shorebird_tests/test/base_test.dart new file mode 100644 index 0000000000000..14dba7ca6cc85 --- /dev/null +++ b/packages/shorebird_tests/test/base_test.dart @@ -0,0 +1,15 @@ +import 'package:test/test.dart'; + +import 'shorebird_tests.dart'; + +void main() { + group('shorebird helpers', () { + testWithShorebirdProject('can build a base project', + (projectDirectory) async { + expect(projectDirectory.existsSync(), isTrue); + + expect(projectDirectory.pubspecFile.existsSync(), isTrue); + expect(projectDirectory.shorebirdFile.existsSync(), isTrue); + }); + }); +} diff --git a/packages/shorebird_tests/test/ios_test.dart b/packages/shorebird_tests/test/ios_test.dart new file mode 100644 index 0000000000000..3fe6e1652c3e5 --- /dev/null +++ b/packages/shorebird_tests/test/ios_test.dart @@ -0,0 +1,91 @@ +import 'package:test/test.dart'; + +import 'shorebird_tests.dart'; + +void main() { + group( + 'shorebird ios projects', + () { + testWithShorebirdProject('can build', (projectDirectory) async { + await projectDirectory.runFlutterBuildIos(); + + expect(projectDirectory.iosArchiveFile().existsSync(), isTrue); + expect(projectDirectory.getGeneratedIosShorebirdYaml(), completes); + }); + + group('when passing the public key through the environment variable', () { + testWithShorebirdProject( + 'adds the public key on top of the original file', + (projectDirectory) async { + final originalYaml = projectDirectory.shorebirdYaml; + + const base64PublicKey = 'public_123'; + await projectDirectory.runFlutterBuildIos( + environment: { + 'SHOREBIRD_PUBLIC_KEY': base64PublicKey, + }, + ); + + final generatedYaml = + await projectDirectory.getGeneratedIosShorebirdYaml(); + + expect( + generatedYaml.keys, + containsAll(originalYaml.keys), + ); + + expect( + generatedYaml['patch_public_key'], + equals(base64PublicKey), + ); + }, + ); + }); + + group('when building with a flavor', () { + testWithShorebirdProject( + 'correctly changes the app id', + (projectDirectory) async { + await projectDirectory.addProjectFlavors(); + projectDirectory.addShorebirdFlavors(); + + await projectDirectory.runFlutterBuildIos(flavor: 'internal'); + + final generatedYaml = + await projectDirectory.getGeneratedIosShorebirdYaml(); + + expect(generatedYaml['app_id'], equals('internal_123')); + }, + ); + + group('when public key passed through environment variable', () { + testWithShorebirdProject( + 'correctly changes the app id and adds the public key', + (projectDirectory) async { + const base64PublicKey = 'public_123'; + await projectDirectory.addProjectFlavors(); + projectDirectory.addShorebirdFlavors(); + + await projectDirectory.runFlutterBuildIos( + flavor: 'internal', + environment: { + 'SHOREBIRD_PUBLIC_KEY': base64PublicKey, + }, + ); + + final generatedYaml = + await projectDirectory.getGeneratedIosShorebirdYaml(); + + expect(generatedYaml['app_id'], equals('internal_123')); + expect( + generatedYaml['patch_public_key'], + equals(base64PublicKey), + ); + }, + ); + }); + }); + }, + testOn: 'mac-os', + ); +} diff --git a/packages/shorebird_tests/test/shorebird_tests.dart b/packages/shorebird_tests/test/shorebird_tests.dart new file mode 100644 index 0000000000000..4bead2de4503e --- /dev/null +++ b/packages/shorebird_tests/test/shorebird_tests.dart @@ -0,0 +1,281 @@ +import 'dart:async'; +import 'dart:io'; + +import 'package:archive/archive_io.dart'; +import 'package:path/path.dart' as path; + +import 'package:meta/meta.dart'; +import 'package:test/test.dart'; +import 'package:yaml/yaml.dart'; + +/// This will be the path to the flutter binary housed in this flutter repository. +/// +/// Which since we are running the tests from this inner package , we need to go up two directories +/// in order to find the flutter binary in the bin folder. +File get _flutterBinaryFile => File( + path.join( + Directory.current.path, + '..', + '..', + 'bin', + 'flutter${Platform.isWindows ? '.bat' : ''}', + ), + ); + +/// Runs a flutter command using the correct binary ([_flutterBinaryFile]) with the given arguments. +Future _runFlutterCommand( + List arguments, { + required Directory workingDirectory, + Map? environment, +}) { + return Process.run( + _flutterBinaryFile.absolute.path, + arguments, + workingDirectory: workingDirectory.path, + environment: { + 'FLUTTER_STORAGE_BASE_URL': 'https://download.shorebird.dev', + if (environment != null) ...environment, + }, + ); +} + +Future _createFlutterProject(Directory projectDirectory) async { + final result = await _runFlutterCommand( + ['create', '--empty', '.'], + workingDirectory: projectDirectory, + ); + if (result.exitCode != 0) { + throw Exception('Failed to create Flutter project: ${result.stderr}'); + } +} + +@isTest +Future testWithShorebirdProject(String name, + FutureOr Function(Directory projectDirectory) testFn) async { + test( + name, + () async { + final parentDirectory = Directory.systemTemp.createTempSync(); + final projectDirectory = Directory( + path.join( + parentDirectory.path, + 'shorebird_test', + ), + )..createSync(); + + try { + await _createFlutterProject(projectDirectory); + + projectDirectory.pubspecFile.writeAsStringSync(''' +${projectDirectory.pubspecFile.readAsStringSync()} + assets: + - shorebird.yaml +'''); + + File( + path.join( + projectDirectory.path, + 'shorebird.yaml', + ), + ).writeAsStringSync(''' +app_id: "123" +'''); + + await testFn(projectDirectory); + } finally { + projectDirectory.deleteSync(recursive: true); + } + }, + timeout: Timeout( + // These tests usually run flutter create, flutter build, etc, which can take a while, + // specially in CI, so setting from the default of 30 seconds to 6 minutes. + Duration(minutes: 6), + ), + ); +} + +extension ShorebirdProjectDirectoryOnDirectory on Directory { + File get pubspecFile => File( + path.join(this.path, 'pubspec.yaml'), + ); + + File get shorebirdFile => File( + path.join(this.path, 'shorebird.yaml'), + ); + + YamlMap get shorebirdYaml => + loadYaml(shorebirdFile.readAsStringSync()) as YamlMap; + + File get appGradleFile => File( + path.join(this.path, 'android', 'app', 'build.gradle'), + ); + + Future addPubDependency(String name, {bool dev = false}) async { + final result = await _runFlutterCommand( + ['pub', 'add', if (dev) '--dev', name], + workingDirectory: this, + ); + if (result.exitCode != 0) { + throw Exception( + 'Failed to run `flutter pub add $name`: ${result.stderr}'); + } + } + + Future addProjectFlavors() async { + await addPubDependency( + // TODO(felangel): revert to using published version once 3.29.0 support is released. + // https://github.com/AngeloAvv/flutter_flavorizr/pull/291 + 'dev:flutter_flavorizr:{"git":{"url":"https://github.com/wjlee611/flutter_flavorizr.git","ref":"chore/temp-migrate-3-29","path":"."}}', + ); + + await File( + path.join( + this.path, + 'flavorizr.yaml', + ), + ).writeAsString(''' +flavors: + playStore: + app: + name: "App" + + android: + applicationId: "com.example.shorebird_test" + ios: + bundleId: "com.example.shorebird_test" + internal: + app: + name: "App (Internal)" + + android: + applicationId: "com.example.shorebird_test.internal" + ios: + bundleId: "com.example.shorebird_test.internal" + global: + app: + name: "App (Global)" + + android: + applicationId: "com.example.shorebird_test.global" + ios: + bundleId: "com.example.shorebird_test.global" +'''); + + final result = await _runFlutterCommand( + ['pub', 'run', 'flutter_flavorizr'], + workingDirectory: this, + ); + if (result.exitCode != 0) { + throw Exception( + 'Failed to run `flutter pub run flutter_flavorizr`: ${result.stderr}'); + } + } + + void addShorebirdFlavors() { + const flavors = ''' +flavors: + global: global_123 + internal: internal_123 + playStore: playStore_123 +'''; + + final currentShorebirdContent = shorebirdFile.readAsStringSync(); + shorebirdFile.writeAsStringSync( + ''' +$currentShorebirdContent +$flavors +''', + ); + } + + Future runFlutterBuildApk({ + String? flavor, + Map? environment, + }) async { + final result = await _runFlutterCommand( + [ + 'build', + 'apk', + if (flavor != null) '--flavor=$flavor', + ], + workingDirectory: this, + environment: environment, + ); + if (result.exitCode != 0) { + throw Exception('Failed to run `flutter build apk`: ${result.stderr}'); + } + } + + Future runFlutterBuildIos({ + Map? environment, + String? flavor, + }) async { + final result = await _runFlutterCommand( + // The projects used to test are generated on spot, to make it simpler we don't + // configure any apple accounts on it, so we skip code signing here. + ['build', 'ipa', '--no-codesign', if (flavor != null) '--flavor=$flavor'], + workingDirectory: this, + environment: environment, + ); + + if (result.exitCode != 0) { + throw Exception('Failed to run `flutter build ios`: ${result.stderr}'); + } + } + + File apkFile({String? flavor}) => File( + path.join( + this.path, + 'build', + 'app', + 'outputs', + 'flutter-apk', + 'app-${flavor != null ? '$flavor-' : ''}release.apk', + ), + ); + + Directory iosArchiveFile() => Directory( + path.join( + this.path, + 'build', + 'ios', + 'archive', + 'Runner.xcarchive', + ), + ); + + Future getGeneratedAndroidShorebirdYaml({String? flavor}) async { + final decodedBytes = + ZipDecoder().decodeBytes(apkFile(flavor: flavor).readAsBytesSync()); + + await extractArchiveToDisk( + decodedBytes, path.join(this.path, 'apk-extracted')); + + final yamlString = File( + path.join( + this.path, + 'apk-extracted', + 'assets', + 'flutter_assets', + 'shorebird.yaml', + ), + ).readAsStringSync(); + return loadYaml(yamlString) as YamlMap; + } + + Future getGeneratedIosShorebirdYaml() async { + final yamlString = File( + path.join( + iosArchiveFile().path, + 'Products', + 'Applications', + 'Runner.app', + 'Frameworks', + 'App.framework', + 'flutter_assets', + 'shorebird.yaml', + ), + ).readAsStringSync(); + return loadYaml(yamlString) as YamlMap; + } +} From a1d51639031f4bce86e64d29be0eb4b4ca0caa15 Mon Sep 17 00:00:00 2001 From: Eric Seidel Date: Fri, 14 Nov 2025 15:04:34 -0800 Subject: [PATCH 02/95] chore: run et format --- engine/src/flutter/shell/platform/windows/BUILD.gn | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/engine/src/flutter/shell/platform/windows/BUILD.gn b/engine/src/flutter/shell/platform/windows/BUILD.gn index 18ca773c17f3e..c809f7cd0f0fe 100644 --- a/engine/src/flutter/shell/platform/windows/BUILD.gn +++ b/engine/src/flutter/shell/platform/windows/BUILD.gn @@ -178,7 +178,7 @@ source_set("flutter_windows_source") { ":flutter_windows_headers", "//flutter/fml", "//flutter/impeller/renderer/backend/gles", - "//flutter/shell/common/shorebird:shorebird", + "//flutter/shell/common/shorebird", "//flutter/shell/geometry", "//flutter/shell/platform/common:common_cpp", "//flutter/shell/platform/common:common_cpp_input", From b2ca1fb85b44a96542f6572fec74e501ee7bbc3c Mon Sep 17 00:00:00 2001 From: Eric Seidel Date: Fri, 14 Nov 2025 16:06:55 -0800 Subject: [PATCH 03/95] fix: update name to application_library_paths --- .../flutter/shell/common/shorebird/shorebird.cc | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/engine/src/flutter/shell/common/shorebird/shorebird.cc b/engine/src/flutter/shell/common/shorebird/shorebird.cc index a2b2f382820a1..ec2bc6bf3c9cd 100644 --- a/engine/src/flutter/shell/common/shorebird/shorebird.cc +++ b/engine/src/flutter/shell/common/shorebird/shorebird.cc @@ -144,7 +144,7 @@ bool ConfigureShorebird(const ShorebirdConfigArgs& args, // https://stackoverflow.com/questions/26032039/convert-vectorstring-into-char-c std::vector c_paths{}; c_paths.push_back(args.release_app_library_path.c_str()); - // Do not modify application_library_path or c_strings will invalidate. + // Do not modify application_library_paths or c_strings will invalidate. app_parameters.original_libapp_paths = c_paths.data(); app_parameters.original_libapp_paths_size = c_paths.size(); @@ -238,10 +238,10 @@ void ConfigureShorebird(std::string code_cache_path, // https://stackoverflow.com/questions/26032039/convert-vectorstring-into-char-c std::vector c_paths{}; - for (const auto& string : settings.application_library_path) { + for (const auto& string : settings.application_library_paths) { c_paths.push_back(string.c_str()); } - // Do not modify application_library_path or c_strings will invalidate. + // Do not modify application_library_paths or c_strings will invalidate. app_parameters.original_libapp_paths = c_paths.data(); app_parameters.original_libapp_paths_size = c_paths.size(); @@ -273,11 +273,11 @@ void ConfigureShorebird(std::string code_cache_path, // On iOS we add the patch to the front of the list instead of clearing // the list, to allow dart_snapshot.cc to still find the base snapshot // for the vm isolate. - settings.application_library_path.insert( - settings.application_library_path.begin(), active_path); + settings.application_library_paths.insert( + settings.application_library_paths.begin(), active_path); #else - settings.application_library_path.clear(); - settings.application_library_path.emplace_back(active_path); + settings.application_library_paths.clear(); + settings.application_library_paths.emplace_back(active_path); #endif } else { FML_LOG(INFO) << "Shorebird updater: no active patch."; From 940a610a9d77820c938f54ac99f1457c9b3d52fe Mon Sep 17 00:00:00 2001 From: Eric Seidel Date: Fri, 14 Nov 2025 17:18:37 -0800 Subject: [PATCH 04/95] fix: attempt to fix builds --- engine/src/flutter/runtime/dart_snapshot.cc | 2 +- .../flutter/shell/platform/windows/flutter_windows_engine.cc | 3 --- 2 files changed, 1 insertion(+), 4 deletions(-) diff --git a/engine/src/flutter/runtime/dart_snapshot.cc b/engine/src/flutter/runtime/dart_snapshot.cc index 6188180860f5f..e8cb5db9a74bc 100644 --- a/engine/src/flutter/runtime/dart_snapshot.cc +++ b/engine/src/flutter/runtime/dart_snapshot.cc @@ -57,7 +57,7 @@ static std::shared_ptr SearchMapping( bool is_executable) { #if SHOREBIRD_USE_INTERPRETER // Detect when we're trying to load a Shorebird patch. - auto patch_path = native_library_path.front(); + auto patch_path = native_library_paths.front(); bool is_patch = patch_path.find(".vmcode") != std::string::npos; if (is_patch) { // We use this terrible hack to load in the patch and then extract the diff --git a/engine/src/flutter/shell/platform/windows/flutter_windows_engine.cc b/engine/src/flutter/shell/platform/windows/flutter_windows_engine.cc index b48c62b76dbb1..fa4d9b1b349fd 100644 --- a/engine/src/flutter/shell/platform/windows/flutter_windows_engine.cc +++ b/engine/src/flutter/shell/platform/windows/flutter_windows_engine.cc @@ -408,9 +408,6 @@ bool SetUpShorebird(std::string assets_path_string, std::string& patch_path) { } bool FlutterWindowsEngine::Run(std::string_view entrypoint) { - std::string assets_path_string = project_->assets_path().u8string(); - std::string icu_path_string = project_->icu_path().u8string(); - if (!project_->HasValidPaths()) { FML_LOG(ERROR) << "Missing or unresolvable paths to assets."; return false; From 3bfd2e2859a3fd9b39b5ef55f78f7b63e2626cef Mon Sep 17 00:00:00 2001 From: Eric Seidel Date: Mon, 17 Nov 2025 08:26:44 -0800 Subject: [PATCH 05/95] chore: add missing header for ios --- engine/src/flutter/runtime/dart_snapshot.cc | 1 + 1 file changed, 1 insertion(+) diff --git a/engine/src/flutter/runtime/dart_snapshot.cc b/engine/src/flutter/runtime/dart_snapshot.cc index e8cb5db9a74bc..626c6f1f3e735 100644 --- a/engine/src/flutter/runtime/dart_snapshot.cc +++ b/engine/src/flutter/runtime/dart_snapshot.cc @@ -6,6 +6,7 @@ #include +#include #include "flutter/fml/native_library.h" #include "flutter/fml/paths.h" #include "flutter/fml/trace_event.h" From f74be9fb4e1926beb96e12464f2594b3548d368a Mon Sep 17 00:00:00 2001 From: Eric Seidel Date: Mon, 17 Nov 2025 10:45:57 -0800 Subject: [PATCH 06/95] fix: linux android build --- engine/src/build/config/compiler/BUILD.gn | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/engine/src/build/config/compiler/BUILD.gn b/engine/src/build/config/compiler/BUILD.gn index 23ed57a7a006b..a7f5a3ae35bb9 100644 --- a/engine/src/build/config/compiler/BUILD.gn +++ b/engine/src/build/config/compiler/BUILD.gn @@ -667,7 +667,7 @@ config("runtime_library") { } # libunwind.a is located in the respective android cpu subdirectories. # The clang version needs to match the version in the lib_dirs line above. - lib_dirs += [ "${android_toolchain_root}/lib/clang/18/lib/linux/${current_android_cpu}/" ] + lib_dirs += [ "${android_toolchain_root}/lib/clang/19/lib/linux/${current_android_cpu}/" ] libs += [ "clang_rt.builtins-${current_android_cpu}-android" ] } From 67c9ecd4ad064f12732346d48f0f6c03ecb22367 Mon Sep 17 00:00:00 2001 From: Eric Seidel Date: Mon, 17 Nov 2025 15:23:56 -0800 Subject: [PATCH 07/95] fix: attempt to fix shorebird flutter_tools changes --- .../lib/src/build_system/targets/assets.dart | 6 ++---- packages/flutter_tools/lib/src/version.dart | 16 +++++++++++----- 2 files changed, 13 insertions(+), 9 deletions(-) diff --git a/packages/flutter_tools/lib/src/build_system/targets/assets.dart b/packages/flutter_tools/lib/src/build_system/targets/assets.dart index 4c7eb44cd87e6..7c630fedafb7c 100644 --- a/packages/flutter_tools/lib/src/build_system/targets/assets.dart +++ b/packages/flutter_tools/lib/src/build_system/targets/assets.dart @@ -194,12 +194,10 @@ Future copyAssets( updateShorebirdYaml( environment.defines[kFlavor], file.path, - environment: globals.platform.environment, + environment: environment.platform.environment, ); } on Exception catch (error) { - throw Exception( - 'Failed to generate shorebird configuration. Error: $error', - ); + throw Exception('Failed to generate shorebird configuration. Error: $error'); } } } diff --git a/packages/flutter_tools/lib/src/version.dart b/packages/flutter_tools/lib/src/version.dart index 6a654f5c642f9..1fef96d251f17 100644 --- a/packages/flutter_tools/lib/src/version.dart +++ b/packages/flutter_tools/lib/src/version.dart @@ -1087,11 +1087,17 @@ class GitTagVersion { } // Check if running on a Shorebird release branch. - final String shorebirdFlutterReleases = _runGit( - 'git for-each-ref --contains $gitRef --format %(refname:short) refs/remotes/origin/flutter_release/*', - processUtils, - workingDirectory, - ).trim(); + final String shorebirdFlutterReleases = git + .runSync([ + 'for-each-ref', + '--contains', + gitRef, + '--format', + '%(refname:short)', + 'refs/remotes/origin/flutter_release/*', + ], workingDirectory: workingDirectory) + .stdout + .trim(); final String? shorebirdFlutterVersion = LineSplitter.split( shorebirdFlutterReleases, ).map((e) => e.replaceFirst('origin/flutter_release/', '')).toList().firstOrNull; From 6cef8a2e0eab7aeda16692157beb2f1c0b177e14 Mon Sep 17 00:00:00 2001 From: Eric Seidel Date: Fri, 12 Dec 2025 09:04:44 -0800 Subject: [PATCH 08/95] feat: make it possible to load two patches into the runtime at once (#96) * fix: stop using globals for patch data * chore: run et format * chore: add missing files * test: add unittest * chore: run et format * chore: move elf_cache down into runtime * chore: rename elf* to patch* * chore: clean up logs * chore: clean up comments * chore: use Shorebird dart * chore: small cleanup --- bin/internal/update_dart_sdk.sh | 2 +- engine/src/flutter/runtime/BUILD.gn | 4 + engine/src/flutter/runtime/dart_snapshot.cc | 127 +++++--------- engine/src/flutter/runtime/shorebird/BUILD.gn | 15 ++ .../flutter/runtime/shorebird/patch_cache.cc | 160 ++++++++++++++++++ .../flutter/runtime/shorebird/patch_cache.h | 104 ++++++++++++ .../runtime/shorebird/patch_mapping.cc | 51 ++++++ .../flutter/runtime/shorebird/patch_mapping.h | 55 ++++++ .../flutter/shell/common/shorebird/BUILD.gn | 2 + .../common/shorebird/patch_cache_unittests.cc | 70 ++++++++ 10 files changed, 505 insertions(+), 85 deletions(-) create mode 100644 engine/src/flutter/runtime/shorebird/BUILD.gn create mode 100644 engine/src/flutter/runtime/shorebird/patch_cache.cc create mode 100644 engine/src/flutter/runtime/shorebird/patch_cache.h create mode 100644 engine/src/flutter/runtime/shorebird/patch_mapping.cc create mode 100644 engine/src/flutter/runtime/shorebird/patch_mapping.h create mode 100644 engine/src/flutter/shell/common/shorebird/patch_cache_unittests.cc diff --git a/bin/internal/update_dart_sdk.sh b/bin/internal/update_dart_sdk.sh index 41f419112e1f4..03c56c9e8079c 100755 --- a/bin/internal/update_dart_sdk.sh +++ b/bin/internal/update_dart_sdk.sh @@ -127,7 +127,7 @@ if [ ! -f "$ENGINE_STAMP" ] || [ "$ENGINE_VERSION" != "$(< "$ENGINE_STAMP")" ]; FIND=find fi - DART_SDK_BASE_URL="${FLUTTER_STORAGE_BASE_URL:-https://storage.googleapis.com}${ENGINE_REALM:+/$ENGINE_REALM}" + DART_SDK_BASE_URL="${FLUTTER_STORAGE_BASE_URL:-https://download.shorebird.dev}${ENGINE_REALM:+/$ENGINE_REALM}" DART_SDK_URL="$DART_SDK_BASE_URL/flutter_infra_release/flutter/$ENGINE_VERSION/$DART_ZIP_NAME" # if the sdk path exists, copy it to a temporary location diff --git a/engine/src/flutter/runtime/BUILD.gn b/engine/src/flutter/runtime/BUILD.gn index 2f7aea3d0d107..1d35652477aca 100644 --- a/engine/src/flutter/runtime/BUILD.gn +++ b/engine/src/flutter/runtime/BUILD.gn @@ -118,6 +118,10 @@ source_set("runtime") { "//flutter/third_party/tonic", "//flutter/txt", ] + + if (is_ios) { + deps += [ "//flutter/runtime/shorebird:patch_cache" ] + } } if (enable_unittests) { diff --git a/engine/src/flutter/runtime/dart_snapshot.cc b/engine/src/flutter/runtime/dart_snapshot.cc index 626c6f1f3e735..de35132f4544c 100644 --- a/engine/src/flutter/runtime/dart_snapshot.cc +++ b/engine/src/flutter/runtime/dart_snapshot.cc @@ -14,6 +14,10 @@ #include "flutter/runtime/dart_vm.h" #include "third_party/dart/runtime/include/dart_api.h" +#if SHOREBIRD_USE_INTERPRETER +#include "flutter/runtime/shorebird/patch_cache.h" // nogncheck +#endif + namespace flutter { const char* DartSnapshot::kVMDataSymbol = "kDartSnapshotData"; @@ -56,93 +60,33 @@ static std::shared_ptr SearchMapping( const std::vector& native_library_paths, const char* native_library_symbol_name, bool is_executable) { -#if SHOREBIRD_USE_INTERPRETER - // Detect when we're trying to load a Shorebird patch. - auto patch_path = native_library_paths.front(); - bool is_patch = patch_path.find(".vmcode") != std::string::npos; - if (is_patch) { - // We use this terrible hack to load in the patch and then extract the - // symbols from it when the path is not App.framework/App but rather - // foo.vmcode, etc. We read the symbols into static variables, but then I - // believe we need to hold onto the ELF itself, otherwise the symbols - // become invalid. - // "leaked_elf" is meant to indicate that we're not freeing the ELF. - static Dart_LoadedElf* leaked_elf = nullptr; - // The VM Snapshot is identical for all binaries produced by a given version - // of Dart. Our linker checks this and will fail to link if ever the VM - // snapshot changes. - const uint8_t* ignored_vm_data = nullptr; - const uint8_t* ignored_vm_instrs = nullptr; - static const uint8_t* isolate_data = nullptr; - static const uint8_t* isolate_instrs = nullptr; - if (leaked_elf == nullptr) { - const char* error = nullptr; - // vmcode files are elf files prefixed with a shorebird linker header. - auto elf_mapping = GetFileMapping(patch_path, false /* executable */); - int elf_file_offset = Shorebird_ReadLinkHeader(elf_mapping->GetMapping(), - elf_mapping->GetSize()); - - leaked_elf = Dart_LoadELF(patch_path.c_str(), elf_file_offset, &error, - &ignored_vm_data, &ignored_vm_instrs, - &isolate_data, &isolate_instrs, - /* load as read-only, not rx */ false); - if (leaked_elf != nullptr) { - FML_LOG(INFO) << "Loaded ELF"; - } else { - FML_LOG(FATAL) << "Failed to load ELF at " << patch_path - << " error: " << error; - abort(); - } - } - - FML_LOG(INFO) << "Loading symbol from ELF " << native_library_symbol_name; - - if (native_library_symbol_name == DartSnapshot::kIsolateDataSymbol) { - return std::make_unique(isolate_data, 0, - nullptr, true); - } else if (native_library_symbol_name == - DartSnapshot::kIsolateInstructionsSymbol) { - return std::make_unique(isolate_instrs, 0, - nullptr, true); - } - // Fall through to normal lookups for VM data and instructions. - // This fallthrough depends on the fact that NativeLibrary below can't - // read the ELF out of our .vmcode files. - } else { - // Only try to open the file if we're not loading a patch. -#endif - - // Ask the embedder. There is no fallback as we expect the embedders (via - // their embedding APIs) to just specify the mappings directly. - if (embedder_mapping_callback) { - // Note that mapping will be nullptr if the mapping callback returns an - // invalid mapping. If all the other methods for resolving the data also - // fail, the engine will stop with accompanying error logs. - if (auto mapping = embedder_mapping_callback()) { - return mapping; - } + // Ask the embedder. There is no fallback as we expect the embedders (via + // their embedding APIs) to just specify the mappings directly. + if (embedder_mapping_callback) { + // Note that mapping will be nullptr if the mapping callback returns an + // invalid mapping. If all the other methods for resolving the data also + // fail, the engine will stop with accompanying error logs. + if (auto mapping = embedder_mapping_callback()) { + return mapping; } + } - // Attempt to open file at path specified. - if (!file_path.empty()) { - if (auto file_mapping = GetFileMapping(file_path, is_executable)) { - return file_mapping; - } + // Attempt to open file at path specified. + if (!file_path.empty()) { + if (auto file_mapping = GetFileMapping(file_path, is_executable)) { + return file_mapping; } + } - // Look in application specified native library if specified. - for (const std::string& path : native_library_paths) { - auto native_library = fml::NativeLibrary::Create(path.c_str()); - auto symbol_mapping = std::make_unique( - native_library, native_library_symbol_name); - if (symbol_mapping->GetMapping() != nullptr) { - return symbol_mapping; - } + // Look in application specified native library if specified. + for (const std::string& path : native_library_paths) { + auto native_library = fml::NativeLibrary::Create(path.c_str()); + auto symbol_mapping = std::make_unique( + native_library, native_library_symbol_name); + if (symbol_mapping->GetMapping() != nullptr) { + return symbol_mapping; } - -#if SHOREBIRD_USE_INTERPRETER - } // !is_patch -#endif + } // Look inside the currently loaded process. { @@ -205,7 +149,14 @@ static std::shared_ptr ResolveIsolateData( nullptr, // release_func true // dontneed_safe ); -#else // DART_SNAPSHOT_STATIC_LINK +#else // DART_SNAPSHOT_STATIC_LINK +#if SHOREBIRD_USE_INTERPRETER + // Try loading from a Shorebird patch first. + if (auto mapping = TryLoadFromPatch(settings.application_library_paths, + DartSnapshot::kIsolateDataSymbol)) { + return mapping; + } +#endif // SHOREBIRD_USE_INTERPRETER return SearchMapping( settings.isolate_snapshot_data, // embedder_mapping_callback settings.isolate_snapshot_data_path, // file_path @@ -224,7 +175,15 @@ static std::shared_ptr ResolveIsolateInstructions( nullptr, // release_func true // dontneed_safe ); -#else // DART_SNAPSHOT_STATIC_LINK +#else // DART_SNAPSHOT_STATIC_LINK +#if SHOREBIRD_USE_INTERPRETER + // Try loading from a Shorebird patch first. + if (auto mapping = + TryLoadFromPatch(settings.application_library_paths, + DartSnapshot::kIsolateInstructionsSymbol)) { + return mapping; + } +#endif // SHOREBIRD_USE_INTERPRETER return SearchMapping( settings.isolate_snapshot_instr, // embedder_mapping_callback settings.isolate_snapshot_instr_path, // file_path diff --git a/engine/src/flutter/runtime/shorebird/BUILD.gn b/engine/src/flutter/runtime/shorebird/BUILD.gn new file mode 100644 index 0000000000000..facee07ba5cd1 --- /dev/null +++ b/engine/src/flutter/runtime/shorebird/BUILD.gn @@ -0,0 +1,15 @@ +import("//flutter/common/config.gni") + +source_set("patch_cache") { + sources = [ + "patch_cache.cc", + "patch_cache.h", + "patch_mapping.cc", + "patch_mapping.h", + ] + + deps = [ + "//flutter/fml", + "//flutter/runtime:libdart", + ] +} diff --git a/engine/src/flutter/runtime/shorebird/patch_cache.cc b/engine/src/flutter/runtime/shorebird/patch_cache.cc new file mode 100644 index 0000000000000..30c99aea3a011 --- /dev/null +++ b/engine/src/flutter/runtime/shorebird/patch_cache.cc @@ -0,0 +1,160 @@ +// Copyright 2013 The Flutter Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#include "flutter/runtime/shorebird/patch_cache.h" + +#include "flutter/fml/logging.h" +#include "flutter/fml/mapping.h" +#include "flutter/runtime/shorebird/patch_mapping.h" + +namespace flutter { + +namespace { + +// These symbol names match the constants in dart_snapshot.cc. +// We duplicate them here rather than extracting them into a header. +// They are actually defined down in Dart and will never change. +constexpr const char* kIsolateDataSymbol = "kDartIsolateSnapshotData"; +constexpr const char* kIsolateInstructionsSymbol = + "kDartIsolateSnapshotInstructions"; + +} // namespace + +// PatchCacheEntry implementation + +std::shared_ptr PatchCacheEntry::Create( + const std::string& path) { + // vmcode files currently use ELF internally after a prefix of a Shorebird + // linker header. + auto elf_mapping = fml::FileMapping::CreateReadOnly(path); + if (!elf_mapping) { + FML_LOG(ERROR) << "Failed to map file: " << path; + return nullptr; + } + + int elf_file_offset = Shorebird_ReadLinkHeader(elf_mapping->GetMapping(), + elf_mapping->GetSize()); + + const char* error = nullptr; + // The VM Snapshot is identical for all binaries produced by a given version + // of Dart. Our linker checks this and will fail to link if ever the VM + // snapshot changes. We ignore the VM data/instrs here. + const uint8_t* ignored_vm_data = nullptr; + const uint8_t* ignored_vm_instrs = nullptr; + const uint8_t* isolate_data = nullptr; + const uint8_t* isolate_instrs = nullptr; + + Dart_LoadedElf* elf = + Dart_LoadELF(path.c_str(), elf_file_offset, &error, &ignored_vm_data, + &ignored_vm_instrs, &isolate_data, &isolate_instrs, + /* load as read-only, not rx */ false); + + if (elf == nullptr) { + FML_LOG(ERROR) << "Failed to load patch at " << path << " error: " << error; + return nullptr; + } + + FML_LOG(INFO) << "Loaded patch from " << path; + + return std::shared_ptr( + new PatchCacheEntry(path, elf, isolate_data, isolate_instrs)); +} + +PatchCacheEntry::PatchCacheEntry(const std::string& path, + Dart_LoadedElf* elf, + const uint8_t* isolate_data, + const uint8_t* isolate_instrs) + : path_(path), + elf_(elf), + isolate_data_(isolate_data), + isolate_instrs_(isolate_instrs) {} + +PatchCacheEntry::~PatchCacheEntry() { + if (elf_ != nullptr) { + FML_LOG(INFO) << "Unloading patch from " << path_; + Dart_UnloadELF(elf_); + elf_ = nullptr; + } +} + +PatchCache& PatchCache::Instance() { + static PatchCache instance; + return instance; +} + +std::shared_ptr PatchCache::GetOrLoad( + const std::string& path) { + std::lock_guard lock(mutex_); + + // Check if we have a cached entry that's still alive + auto it = cache_.find(path); + if (it != cache_.end()) { + if (auto entry = it->second.lock()) { + FML_LOG(INFO) << "PatchCache hit for " << path; + return entry; + } + // Entry expired, remove it + cache_.erase(it); + } + + // Load a new entry + auto entry = PatchCacheEntry::Create(path); + if (entry) { + cache_[path] = entry; // Store weak_ptr + } + + return entry; +} + +void PatchCache::PruneExpired() { + std::lock_guard lock(mutex_); + + for (auto it = cache_.begin(); it != cache_.end();) { + if (it->second.expired()) { + it = cache_.erase(it); + } else { + ++it; + } + } +} + +std::shared_ptr TryLoadFromPatch( + const std::vector& native_library_paths, + const char* symbol_name) { + if (native_library_paths.empty()) { + return nullptr; + } + + // Check if the first path is a Shorebird patch (.vmcode file) + const auto& patch_path = native_library_paths.front(); + bool is_patch = patch_path.find(".vmcode") != std::string::npos; + if (!is_patch) { + return nullptr; + } + + // Patches only contain isolate data/instructions, not VM data/instructions. + // Return nullptr for VM symbols to allow fallback to the base app. + std::string symbol(symbol_name); + if (symbol != kIsolateDataSymbol && symbol != kIsolateInstructionsSymbol) { + return nullptr; + } + + // Load the patch using the cache. + auto cache_entry = PatchCache::Instance().GetOrLoad(patch_path); + if (!cache_entry) { + FML_LOG(FATAL) << "Failed to load symbol from patch at " << patch_path; + return nullptr; + } + + FML_LOG(INFO) << "Loading symbol from patch: " << symbol_name; + + if (symbol == kIsolateDataSymbol) { + return PatchMapping::CreateIsolateData(cache_entry); + } else { + FML_CHECK(symbol == kIsolateInstructionsSymbol); + return PatchMapping::CreateIsolateInstructions(cache_entry); + } +} + +} // namespace flutter diff --git a/engine/src/flutter/runtime/shorebird/patch_cache.h b/engine/src/flutter/runtime/shorebird/patch_cache.h new file mode 100644 index 0000000000000..1f2e14fab711d --- /dev/null +++ b/engine/src/flutter/runtime/shorebird/patch_cache.h @@ -0,0 +1,104 @@ +// Copyright 2013 The Flutter Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#ifndef FLUTTER_RUNTIME_SHOREBIRD_PATCH_CACHE_H_ +#define FLUTTER_RUNTIME_SHOREBIRD_PATCH_CACHE_H_ + +#include +#include +#include +#include +#include + +#include + +#include "flutter/fml/macros.h" +#include "flutter/fml/mapping.h" + +namespace flutter { + +/// A cache entry that holds a loaded patch file and its extracted snapshot +/// pointers. The patch is automatically unloaded when the last reference to +/// this entry is released. +class PatchCacheEntry { + public: + /// Creates a new cache entry by loading the patch file at the given path. + /// Returns nullptr if loading fails. + static std::shared_ptr Create(const std::string& path); + + ~PatchCacheEntry(); + + /// Returns the isolate snapshot data pointer. + const uint8_t* isolate_data() const { return isolate_data_; } + + /// Returns the isolate snapshot instructions pointer. + const uint8_t* isolate_instructions() const { return isolate_instrs_; } + + /// Returns the path this entry was loaded from. + const std::string& path() const { return path_; } + + private: + PatchCacheEntry(const std::string& path, + Dart_LoadedElf* elf, + const uint8_t* isolate_data, + const uint8_t* isolate_instrs); + + std::string path_; + Dart_LoadedElf* elf_; + const uint8_t* isolate_data_; + const uint8_t* isolate_instrs_; + + FML_DISALLOW_COPY_AND_ASSIGN(PatchCacheEntry); +}; + +/// A thread-safe cache for loaded patch files. Cache entries are automatically +/// removed when all references to them are released. +class PatchCache { + public: + /// Returns the singleton instance of the cache. + static PatchCache& Instance(); + + /// Gets or loads a patch file at the given path. If the file is already + /// cached and the entry is still alive, returns the existing entry. + /// Otherwise, loads the file and creates a new cache entry. + /// Returns nullptr if loading fails. + std::shared_ptr GetOrLoad(const std::string& path); + + /// Removes expired entries from the cache. This is called automatically + /// by GetOrLoad, but can also be called explicitly. + void PruneExpired(); + + private: + PatchCache() = default; + ~PatchCache() = default; + + std::mutex mutex_; + // We store weak references so entries are automatically cleaned up when + // all ElfMapping instances release their references. + std::map> cache_; + + FML_DISALLOW_COPY_AND_ASSIGN(PatchCache); +}; + +/// Checks if the first path in native_library_paths is a Shorebird patch +/// (.vmcode file) and if so, attempts to load the requested symbol from +/// the patch. +/// +/// @param native_library_paths The list of library paths to check. The first +/// path is checked for the .vmcode extension. +/// @param symbol_name The symbol to load (kIsolateDataSymbol or +/// kIsolateInstructionsSymbol). +/// @return A mapping for the requested symbol if this is a patch and the +/// symbol is available in the patch, nullptr otherwise. +/// +/// Note: Patches only contain isolate data/instructions, not VM +/// data/instructions. For VM symbols, this will always return nullptr, +/// allowing the caller to fall back to loading from the base app. +std::shared_ptr TryLoadFromPatch( + const std::vector& native_library_paths, + const char* symbol_name); + +} // namespace flutter + +#endif // FLUTTER_RUNTIME_SHOREBIRD_PATCH_CACHE_H_ diff --git a/engine/src/flutter/runtime/shorebird/patch_mapping.cc b/engine/src/flutter/runtime/shorebird/patch_mapping.cc new file mode 100644 index 0000000000000..d444cda817c34 --- /dev/null +++ b/engine/src/flutter/runtime/shorebird/patch_mapping.cc @@ -0,0 +1,51 @@ +// Copyright 2013 The Flutter Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#include "flutter/runtime/shorebird/patch_mapping.h" + +#include "third_party/dart/runtime/include/dart_native_api.h" + +namespace flutter { + +std::shared_ptr PatchMapping::CreateIsolateData( + std::shared_ptr entry) { + if (!entry) { + return nullptr; + } + const uint8_t* data = entry->isolate_data(); + size_t size = Dart_SnapshotDataSize(data); + return std::shared_ptr(new PatchMapping(entry, data, size)); +} + +std::shared_ptr PatchMapping::CreateIsolateInstructions( + std::shared_ptr entry) { + if (!entry) { + return nullptr; + } + const uint8_t* data = entry->isolate_instructions(); + size_t size = Dart_SnapshotInstrSize(data); + return std::shared_ptr(new PatchMapping(entry, data, size)); +} + +PatchMapping::PatchMapping(std::shared_ptr entry, + const uint8_t* data, + size_t size) + : cache_entry_(std::move(entry)), data_(data), size_(size) {} + +PatchMapping::~PatchMapping() = default; + +size_t PatchMapping::GetSize() const { + return size_; +} + +const uint8_t* PatchMapping::GetMapping() const { + return data_; +} + +bool PatchMapping::IsDontNeedSafe() const { + // Patch mappings are file-backed and safe for madvise(DONTNEED). + return true; +} + +} // namespace flutter diff --git a/engine/src/flutter/runtime/shorebird/patch_mapping.h b/engine/src/flutter/runtime/shorebird/patch_mapping.h new file mode 100644 index 0000000000000..8c2e494a9004c --- /dev/null +++ b/engine/src/flutter/runtime/shorebird/patch_mapping.h @@ -0,0 +1,55 @@ +// Copyright 2013 The Flutter Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#ifndef FLUTTER_RUNTIME_SHOREBIRD_PATCH_MAPPING_H_ +#define FLUTTER_RUNTIME_SHOREBIRD_PATCH_MAPPING_H_ + +#include + +#include "flutter/fml/macros.h" +#include "flutter/fml/mapping.h" +#include "flutter/runtime/shorebird/patch_cache.h" + +namespace flutter { + +/// A Mapping implementation that references data from a cached patch file. +/// Holding a reference to this mapping keeps the underlying patch loaded. +class PatchMapping final : public fml::Mapping { + public: + /// Creates a mapping for the isolate snapshot data from the given cache + /// entry. + static std::shared_ptr CreateIsolateData( + std::shared_ptr entry); + + /// Creates a mapping for the isolate snapshot instructions from the given + /// cache entry. + static std::shared_ptr CreateIsolateInstructions( + std::shared_ptr entry); + + ~PatchMapping() override; + + // |fml::Mapping| + size_t GetSize() const override; + + // |fml::Mapping| + const uint8_t* GetMapping() const override; + + // |fml::Mapping| + bool IsDontNeedSafe() const override; + + private: + PatchMapping(std::shared_ptr entry, + const uint8_t* data, + size_t size); + + std::shared_ptr cache_entry_; + const uint8_t* data_; + size_t size_; + + FML_DISALLOW_COPY_AND_ASSIGN(PatchMapping); +}; + +} // namespace flutter + +#endif // FLUTTER_RUNTIME_SHOREBIRD_PATCH_MAPPING_H_ diff --git a/engine/src/flutter/shell/common/shorebird/BUILD.gn b/engine/src/flutter/shell/common/shorebird/BUILD.gn index 2c7def6991542..dc2fc5515bd7c 100644 --- a/engine/src/flutter/shell/common/shorebird/BUILD.gn +++ b/engine/src/flutter/shell/common/shorebird/BUILD.gn @@ -42,6 +42,7 @@ if (enable_unittests) { testonly = true sources = [ + "patch_cache_unittests.cc", "shorebird_unittests.cc", "snapshots_data_handle_unittests.cc", ] @@ -53,6 +54,7 @@ if (enable_unittests) { ":shorebird_fixtures", ":snapshots_data_handle", "//flutter/runtime", + "//flutter/runtime/shorebird:patch_cache", "//flutter/testing", "//flutter/testing:fixture_test", ] diff --git a/engine/src/flutter/shell/common/shorebird/patch_cache_unittests.cc b/engine/src/flutter/shell/common/shorebird/patch_cache_unittests.cc new file mode 100644 index 0000000000000..51fc20f9ca562 --- /dev/null +++ b/engine/src/flutter/shell/common/shorebird/patch_cache_unittests.cc @@ -0,0 +1,70 @@ +// Copyright 2013 The Flutter Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#include "flutter/runtime/shorebird/patch_cache.h" + +#include +#include + +#include "gtest/gtest.h" + +namespace flutter { +namespace testing { + +TEST(PatchCache, InstanceReturnsSameInstance) { + PatchCache& instance1 = PatchCache::Instance(); + PatchCache& instance2 = PatchCache::Instance(); + EXPECT_EQ(&instance1, &instance2); +} + +TEST(TryLoadFromPatch, ReturnsNullptrForEmptyPaths) { + std::vector empty_paths; + auto result = TryLoadFromPatch(empty_paths, "kDartIsolateSnapshotData"); + EXPECT_EQ(result, nullptr); +} + +TEST(TryLoadFromPatch, ReturnsNullptrForNonVmcodePath) { + std::vector paths = {"/path/to/some/file.so"}; + auto result = TryLoadFromPatch(paths, "kDartIsolateSnapshotData"); + EXPECT_EQ(result, nullptr); +} + +TEST(TryLoadFromPatch, ReturnsNullptrForVmSymbol) { + // Even with a .vmcode path, VM symbols should return nullptr + // (we can't actually load the file, but we can verify the symbol check) + std::vector paths = {"/path/to/patch.vmcode"}; + + // VM data symbol should return nullptr (patches don't contain VM snapshots) + auto result_vm_data = TryLoadFromPatch(paths, "kDartVmSnapshotData"); + EXPECT_EQ(result_vm_data, nullptr); + + // VM instructions symbol should return nullptr + auto result_vm_instrs = + TryLoadFromPatch(paths, "kDartVmSnapshotInstructions"); + EXPECT_EQ(result_vm_instrs, nullptr); +} + +TEST(TryLoadFromPatch, ReturnsNullptrForUnknownSymbol) { + std::vector paths = {"/path/to/patch.vmcode"}; + auto result = TryLoadFromPatch(paths, "kSomeUnknownSymbol"); + EXPECT_EQ(result, nullptr); +} + +TEST(TryLoadFromPatch, ChecksOnlyFirstPath) { + // Only the first path should be checked for .vmcode extension + std::vector paths = {"/path/to/regular.so", + "/path/to/patch.vmcode"}; + auto result = TryLoadFromPatch(paths, "kDartIsolateSnapshotData"); + // Should return nullptr because first path is not .vmcode + EXPECT_EQ(result, nullptr); +} + +TEST(PatchCache, GetOrLoadReturnsNullptrForNonexistentFile) { + auto result = + PatchCache::Instance().GetOrLoad("/nonexistent/path/to/file.vmcode"); + EXPECT_EQ(result, nullptr); +} + +} // namespace testing +} // namespace flutter From cbfde6dabc6f14448dcd34f8211fa32783812e65 Mon Sep 17 00:00:00 2001 From: Eric Seidel Date: Fri, 12 Dec 2025 14:36:18 -0800 Subject: [PATCH 09/95] feat: shorebird flutter should work without setting FLUTTER_STORAGE_BASE_URL (#97) * fix: make dart/flutter work without FLUTTER_STORAGE_BASE_URL * feat: shorebird flutter should work without setting FLUTTER_STORAGE_BASE_URL * fix: flutter_tools test fixes * fix: enable running flutter_tools tests * chore: remove unnecessary workflow --- .github/workflows/shorebird_ci.yml | 5 +- bin/internal/update_dart_sdk.ps1 | 2 +- dev/bots/post_process_docs.dart | 2 +- dev/bots/unpublish_package.dart | 2 +- .../settings.gradle | 2 +- .../settings.gradle.kts | 2 +- dev/tools/create_api_docs.dart | 4 +- engine/src/flutter/build/zip_bundle.gni | 2 +- .../impeller/toolkit/interop/README.md | 2 +- .../web_ui/dev/steps/copy_artifacts_step.dart | 4 +- .../gradle/aar_init_script.gradle | 2 +- .../src/main/kotlin/FlutterPluginConstants.kt | 2 +- packages/flutter_tools/lib/src/cache.dart | 4 +- .../lib/src/http_host_validator.dart | 2 +- .../test/general.shard/base/build_test.dart | 160 ++++++++++++---- .../build_system/targets/common_test.dart | 12 ++ .../build_system/targets/macos_test.dart | 180 ++++++++++++------ .../test/general.shard/cache_test.dart | 4 +- .../test/general.shard/version_test.dart | 14 ++ 19 files changed, 288 insertions(+), 119 deletions(-) diff --git a/.github/workflows/shorebird_ci.yml b/.github/workflows/shorebird_ci.yml index ec147b4489255..7b712b1583b87 100644 --- a/.github/workflows/shorebird_ci.yml +++ b/.github/workflows/shorebird_ci.yml @@ -46,10 +46,7 @@ jobs: - name: ๐Ÿฆ Run Flutter Tools Tests # TODO(eseidel): Find a nice way to run this on windows. if: ${{ matrix.os == 'ubuntu-latest' || matrix.os == 'macos-latest' }} - # TODO(eseidel): We can't run all flutter_tools tests until we make - # our changes not throw exceptions on missing shorebird.yaml. - # https://github.com/shorebirdtech/shorebird/issues/2392 - run: ../../bin/flutter test test/general.shard/shorebird + run: ../../bin/flutter test test/general.shard working-directory: packages/flutter_tools - name: ๐Ÿฆ Run Shorebird Tests diff --git a/bin/internal/update_dart_sdk.ps1 b/bin/internal/update_dart_sdk.ps1 index 318ef62add853..a588137834eb6 100644 --- a/bin/internal/update_dart_sdk.ps1 +++ b/bin/internal/update_dart_sdk.ps1 @@ -41,7 +41,7 @@ if ((Test-Path $engineStamp) -and ($engineVersion -eq (Get-Content $engineStamp) $dartSdkBaseUrl = $Env:FLUTTER_STORAGE_BASE_URL if (-not $dartSdkBaseUrl) { - $dartSdkBaseUrl = "https://storage.googleapis.com" + $dartSdkBaseUrl = "https://download.shorebird.dev" } if ($engineRealm) { $dartSdkBaseUrl = "$dartSdkBaseUrl/$engineRealm" diff --git a/dev/bots/post_process_docs.dart b/dev/bots/post_process_docs.dart index 905cd8e82bf67..9a6fee8d80ab8 100644 --- a/dev/bots/post_process_docs.dart +++ b/dev/bots/post_process_docs.dart @@ -37,7 +37,7 @@ Future postProcess() async { await runProcessWithValidations([ 'curl', '-L', - 'https://storage.googleapis.com/flutter_infra_release/flutter/$revision/api_docs.zip', + 'https://download.shorebird.dev/flutter_infra_release/flutter/$revision/api_docs.zip', '--output', zipDestination, '--fail', diff --git a/dev/bots/unpublish_package.dart b/dev/bots/unpublish_package.dart index e7c32fb365638..bf758b7cdd9d9 100644 --- a/dev/bots/unpublish_package.dart +++ b/dev/bots/unpublish_package.dart @@ -26,7 +26,7 @@ import 'prepare_package/transactional_update.dart'; const String gsBase = 'gs://flutter_infra_release'; const String releaseFolder = '/releases'; const String gsReleaseFolder = '$gsBase$releaseFolder'; -const String baseUrl = 'https://storage.googleapis.com/flutter_infra_release'; +const String baseUrl = 'https://download.shorebird.dev/flutter_infra_release'; /// Exception class for when a process fails to run, so we can catch /// it and provide something more readable than a stack trace. diff --git a/dev/integration_tests/pure_android_host_apps/android_host_app_v2_embedding/settings.gradle b/dev/integration_tests/pure_android_host_apps/android_host_app_v2_embedding/settings.gradle index 5368feb7ecde7..66364cdf4a190 100644 --- a/dev/integration_tests/pure_android_host_apps/android_host_app_v2_embedding/settings.gradle +++ b/dev/integration_tests/pure_android_host_apps/android_host_app_v2_embedding/settings.gradle @@ -7,7 +7,7 @@ dependencyResolutionManagement { repositories { google() mavenCentral() - def flutterStorageUrl = System.getenv("FLUTTER_STORAGE_BASE_URL") ?: "https://storage.googleapis.com" + def flutterStorageUrl = System.getenv("FLUTTER_STORAGE_BASE_URL") ?: "https://download.shorebird.dev" maven { url = uri("$flutterStorageUrl/download.flutter.io") } diff --git a/dev/integration_tests/pure_android_host_apps/host_app_kotlin_gradle_dsl/settings.gradle.kts b/dev/integration_tests/pure_android_host_apps/host_app_kotlin_gradle_dsl/settings.gradle.kts index f6d75bce11757..da25b49a46f7f 100644 --- a/dev/integration_tests/pure_android_host_apps/host_app_kotlin_gradle_dsl/settings.gradle.kts +++ b/dev/integration_tests/pure_android_host_apps/host_app_kotlin_gradle_dsl/settings.gradle.kts @@ -16,7 +16,7 @@ dependencyResolutionManagement { repositories { google() mavenCentral() - val flutterStorageUrl = System.getenv("FLUTTER_STORAGE_BASE_URL") ?: "https://storage.googleapis.com" + val flutterStorageUrl = System.getenv("FLUTTER_STORAGE_BASE_URL") ?: "https://download.shorebird.dev" maven("$flutterStorageUrl/download.flutter.io") } } diff --git a/dev/tools/create_api_docs.dart b/dev/tools/create_api_docs.dart index c3e7d34cafc52..77e46db14aec5 100644 --- a/dev/tools/create_api_docs.dart +++ b/dev/tools/create_api_docs.dart @@ -923,8 +923,8 @@ class PlatformDocGenerator { for (final String platform in kPlatformDocs.keys) { final String zipFile = kPlatformDocs[platform]!.zipName; - final url = - 'https://storage.googleapis.com/${realm}flutter_infra_release/flutter/$engineRevision/$zipFile'; + final String url = + 'https://download.shorebird.dev/${realm}flutter_infra_release/flutter/$engineRevision/$zipFile'; await _extractDocs(url, platform, kPlatformDocs[platform]!, outputDir); } } diff --git a/engine/src/flutter/build/zip_bundle.gni b/engine/src/flutter/build/zip_bundle.gni index ccfa08822af48..85fd9aa8acb8c 100644 --- a/engine/src/flutter/build/zip_bundle.gni +++ b/engine/src/flutter/build/zip_bundle.gni @@ -55,7 +55,7 @@ template("zip_bundle") { license_path = rebase_path("//flutter/sky/packages/sky_engine/LICENSE", "//flutter") git_url = "https://github.com/flutter/engine/tree/$engine_version" - sky_engine_url = "https://storage.googleapis.com/flutter_infra_release/flutter/$engine_version/sky_engine.zip" + sky_engine_url = "https://download.shorebird.dev/flutter_infra_release/flutter/$engine_version/sky_engine.zip" outputs = [ license_readme ] contents = [ "# $target_name", diff --git a/engine/src/flutter/impeller/toolkit/interop/README.md b/engine/src/flutter/impeller/toolkit/interop/README.md index 6e75a53a11775..279379217e143 100644 --- a/engine/src/flutter/impeller/toolkit/interop/README.md +++ b/engine/src/flutter/impeller/toolkit/interop/README.md @@ -27,7 +27,7 @@ A single-header C API for 2D graphics and text rendering. [Impeller](../../READM Users may plug in a custom toolchain into the Flutter Engine build system to build the `libimpeller.so` dynamic library. However, for the common platforms, the CI bots upload a tarball containing the library and headers. This URL for the SDK tarball for a particular platform can be constructed as follows: ```sh -https://storage.googleapis.com/flutter_infra_release/flutter/$FLUTTER_SHA/$PLATFORM_ARCH/impeller_sdk.zip +https://download.shorebird.dev/flutter_infra_release/flutter/$FLUTTER_SHA/$PLATFORM_ARCH/impeller_sdk.zip ``` The `$FLUTTER_SHA` is the Git hash in the [Flutter repository](https://github.com/flutter/flutter). The `$PLATFORM_ARCH` can be determined from the table below. diff --git a/engine/src/flutter/lib/web_ui/dev/steps/copy_artifacts_step.dart b/engine/src/flutter/lib/web_ui/dev/steps/copy_artifacts_step.dart index ed1b2010dc9f1..61b22d84de5f7 100644 --- a/engine/src/flutter/lib/web_ui/dev/steps/copy_artifacts_step.dart +++ b/engine/src/flutter/lib/web_ui/dev/steps/copy_artifacts_step.dart @@ -55,8 +55,8 @@ class CopyArtifactsStep implements PipelineStep { 'Could not generate artifact bucket url for unknown realm.', ), }; - final url = Uri.https( - 'storage.googleapis.com', + final Uri url = Uri.https( + 'download.shorebird.dev', '${realmComponent}flutter_infra_release/flutter/${realm == LuciRealm.Try ? gitRevision : contentHash}/flutter-web-sdk.zip', ); final http.Response response = await http.Client().get(url); diff --git a/packages/flutter_tools/gradle/aar_init_script.gradle b/packages/flutter_tools/gradle/aar_init_script.gradle index b1bcfeef403b1..283e0343c28d6 100644 --- a/packages/flutter_tools/gradle/aar_init_script.gradle +++ b/packages/flutter_tools/gradle/aar_init_script.gradle @@ -42,7 +42,7 @@ void configureProject(Project project, String outputDir) { return } - String storageUrl = System.getenv('FLUTTER_STORAGE_BASE_URL') ?: "https://storage.googleapis.com" + String storageUrl = System.getenv('FLUTTER_STORAGE_BASE_URL') ?: "https://download.shorebird.dev" String engineRealm = Paths.get(getFlutterRoot(project), "bin", "cache", "engine.realm") .toFile().text.trim() diff --git a/packages/flutter_tools/gradle/src/main/kotlin/FlutterPluginConstants.kt b/packages/flutter_tools/gradle/src/main/kotlin/FlutterPluginConstants.kt index 13eb2be26758b..61e8e3c17c006 100644 --- a/packages/flutter_tools/gradle/src/main/kotlin/FlutterPluginConstants.kt +++ b/packages/flutter_tools/gradle/src/main/kotlin/FlutterPluginConstants.kt @@ -21,7 +21,7 @@ object FlutterPluginConstants { const val INTERMEDIATES_DIR = "intermediates" const val FLUTTER_STORAGE_BASE_URL = "FLUTTER_STORAGE_BASE_URL" - const val DEFAULT_MAVEN_HOST = "https://storage.googleapis.com" + const val DEFAULT_MAVEN_HOST = "https://download.shorebird.dev" /** Maps platforms to ABI architectures. */ @JvmStatic val PLATFORM_ARCH_MAP = diff --git a/packages/flutter_tools/lib/src/cache.dart b/packages/flutter_tools/lib/src/cache.dart index 85db533f39c1f..cac3d265ad677 100644 --- a/packages/flutter_tools/lib/src/cache.dart +++ b/packages/flutter_tools/lib/src/cache.dart @@ -532,7 +532,7 @@ class Cache { /// The base for URLs that store Flutter engine artifacts that are fetched /// during the installation of the Flutter SDK. /// - /// By default the base URL is https://storage.googleapis.com. However, if + /// By default the base URL is https://download.shorebird.dev. However, if /// `FLUTTER_STORAGE_BASE_URL` environment variable ([kFlutterStorageBaseUrl]) /// is provided, the environment variable value is returned instead. /// @@ -544,7 +544,7 @@ class Cache { String? overrideUrl = _platform.environment[kFlutterStorageBaseUrl]; if (overrideUrl == null) { return storageRealm.isEmpty - ? 'https://storage.googleapis.com' + ? 'https://download.shorebird.dev' : 'https://storage.googleapis.com/$storageRealm'; } // Shorebird's artifact proxy is a trusted source. diff --git a/packages/flutter_tools/lib/src/http_host_validator.dart b/packages/flutter_tools/lib/src/http_host_validator.dart index 9702f83bffd08..8363221166398 100644 --- a/packages/flutter_tools/lib/src/http_host_validator.dart +++ b/packages/flutter_tools/lib/src/http_host_validator.dart @@ -11,7 +11,7 @@ import 'doctor_validator.dart'; import 'features.dart'; /// Common Flutter HTTP hosts. -const kCloudHost = 'https://storage.googleapis.com/'; +const kCloudHost = 'https://download.shorebird.dev/'; const kCocoaPods = 'https://cocoapods.org/'; const kGitHub = 'https://github.com/'; const kMaven = 'https://maven.google.com/'; diff --git a/packages/flutter_tools/test/general.shard/base/build_test.dart b/packages/flutter_tools/test/general.shard/base/build_test.dart index 32783a6ca0873..eb7df9999b7e6 100644 --- a/packages/flutter_tools/test/general.shard/base/build_test.dart +++ b/packages/flutter_tools/test/general.shard/base/build_test.dart @@ -12,6 +12,42 @@ import 'package:flutter_tools/src/macos/xcode.dart'; import '../../src/common.dart'; import '../../src/fake_process_manager.dart'; +const kWhichSysctlCommand = FakeCommand(command: ['which', 'sysctl']); + +const kARMCheckCommand = FakeCommand(command: ['sysctl', 'hw.optional.arm64'], exitCode: 1); + +const kDefaultClang = [ + '-miphoneos-version-min=13.0', + '-isysroot', + 'path/to/sdk', + '-dynamiclib', + '-Xlinker', + '-rpath', + '-Xlinker', + '@executable_path/Frameworks', + '-Xlinker', + '-rpath', + '-Xlinker', + '@loader_path/Frameworks', + '-fapplication-extension', + '-install_name', + '@rpath/App.framework/App', + '-o', + 'build/foo/App.framework/App', + 'build/foo/snapshot_assembly.o', +]; + +// Shorebird link info arguments added for iOS/macOS builds. +// These correspond to the dumpLinkInfoArgs in AOTSnapshotter.build(). +const kLinkInfoArgs = [ + '--print_class_table_link_debug_info_to=build/App.class_table.json', + '--print_class_table_link_info_to=build/App.ct.link', + '--print_field_table_link_debug_info_to=build/App.field_table.json', + '--print_field_table_link_info_to=build/App.ft.link', + '--print_dispatch_table_link_debug_info_to=build/App.dispatch_table.json', + '--print_dispatch_table_link_info_to=build/App.dt.link', +]; + void main() { const kWhichSysctlCommand = FakeCommand(command: ['which', 'sysctl']); @@ -173,6 +209,7 @@ void main() { testWithoutContext('builds iOS snapshot with dwarfStackTraces', () async { final String outputPath = fileSystem.path.join('build', 'foo'); + final String assembly = fileSystem.path.join(outputPath, 'snapshot_assembly.S'); final String debugPath = fileSystem.path.join('foo', 'app.ios-arm64.symbols'); final String genSnapshotPath = artifacts.getArtifactPath( Artifact.genSnapshotArm64, @@ -184,12 +221,9 @@ void main() { command: [ genSnapshotPath, '--deterministic', - '--snapshot_kind=app-aot-macho-dylib', - '--macho=$outputPath/App.framework/App', - '--macho-object=$outputPath/app.o', - '--macho-min-os-version=15.0', - '--macho-rpath=@executable_path/Frameworks,@loader_path/Frameworks', - '--macho-install-name=@rpath/App.framework/App', + ...kLinkInfoArgs, + '--snapshot_kind=app-aot-assembly', + '--assembly=$assembly', '--dwarf-stack-traces', '--resolve-dwarf-paths', '--save-debugging-info=$debugPath', @@ -197,24 +231,40 @@ void main() { ], ), kWhichSysctlCommand, - kx64CheckCommand, - FakeCommand( + kARMCheckCommand, + const FakeCommand( + command: [ + 'xcrun', + 'cc', + '-arch', + 'arm64', + '-miphoneos-version-min=13.0', + '-isysroot', + 'path/to/sdk', + '-c', + 'build/foo/snapshot_assembly.S', + '-o', + 'build/foo/snapshot_assembly.o', + ], + ), + const FakeCommand(command: ['xcrun', 'clang', '-arch', 'arm64', ...kDefaultClang]), + const FakeCommand( command: [ 'xcrun', 'dsymutil', '-o', - '$outputPath/App.framework.dSYM', - '$outputPath/App.framework/App', + 'build/foo/App.framework.dSYM', + 'build/foo/App.framework/App', ], ), - FakeCommand( + const FakeCommand( command: [ 'xcrun', 'strip', '-x', - '$outputPath/App.framework/App', + 'build/foo/App.framework/App', '-o', - '$outputPath/App.framework/App', + 'build/foo/App.framework/App', ], ), ]); @@ -236,6 +286,7 @@ void main() { testWithoutContext('builds iOS snapshot with obfuscate', () async { final String outputPath = fileSystem.path.join('build', 'foo'); + final String assembly = fileSystem.path.join(outputPath, 'snapshot_assembly.S'); final String genSnapshotPath = artifacts.getArtifactPath( Artifact.genSnapshotArm64, platform: TargetPlatform.ios, @@ -246,35 +297,48 @@ void main() { command: [ genSnapshotPath, '--deterministic', - '--snapshot_kind=app-aot-macho-dylib', - '--macho=$outputPath/App.framework/App', - '--macho-object=$outputPath/app.o', - '--macho-min-os-version=15.0', - '--macho-rpath=@executable_path/Frameworks,@loader_path/Frameworks', - '--macho-install-name=@rpath/App.framework/App', + ...kLinkInfoArgs, + '--snapshot_kind=app-aot-assembly', + '--assembly=$assembly', '--obfuscate', 'main.dill', ], ), kWhichSysctlCommand, - kx64CheckCommand, - FakeCommand( + kARMCheckCommand, + const FakeCommand( + command: [ + 'xcrun', + 'cc', + '-arch', + 'arm64', + '-miphoneos-version-min=13.0', + '-isysroot', + 'path/to/sdk', + '-c', + 'build/foo/snapshot_assembly.S', + '-o', + 'build/foo/snapshot_assembly.o', + ], + ), + const FakeCommand(command: ['xcrun', 'clang', '-arch', 'arm64', ...kDefaultClang]), + const FakeCommand( command: [ 'xcrun', 'dsymutil', '-o', - '$outputPath/App.framework.dSYM', - '$outputPath/App.framework/App', + 'build/foo/App.framework.dSYM', + 'build/foo/App.framework/App', ], ), - FakeCommand( + const FakeCommand( command: [ 'xcrun', 'strip', '-x', - '$outputPath/App.framework/App', + 'build/foo/App.framework/App', '-o', - '$outputPath/App.framework/App', + 'build/foo/App.framework/App', ], ), ]); @@ -305,34 +369,47 @@ void main() { command: [ genSnapshotPath, '--deterministic', - '--snapshot_kind=app-aot-macho-dylib', - '--macho=$outputPath/App.framework/App', - '--macho-object=$outputPath/app.o', - '--macho-min-os-version=15.0', - '--macho-rpath=@executable_path/Frameworks,@loader_path/Frameworks', - '--macho-install-name=@rpath/App.framework/App', + ...kLinkInfoArgs, + '--snapshot_kind=app-aot-assembly', + '--assembly=${fileSystem.path.join(outputPath, 'snapshot_assembly.S')}', 'main.dill', ], ), kWhichSysctlCommand, - kx64CheckCommand, - FakeCommand( + kARMCheckCommand, + const FakeCommand( + command: [ + 'xcrun', + 'cc', + '-arch', + 'arm64', + '-miphoneos-version-min=13.0', + '-isysroot', + 'path/to/sdk', + '-c', + 'build/foo/snapshot_assembly.S', + '-o', + 'build/foo/snapshot_assembly.o', + ], + ), + const FakeCommand(command: ['xcrun', 'clang', '-arch', 'arm64', ...kDefaultClang]), + const FakeCommand( command: [ 'xcrun', 'dsymutil', '-o', - '$outputPath/App.framework.dSYM', - '$outputPath/App.framework/App', + 'build/foo/App.framework.dSYM', + 'build/foo/App.framework/App', ], ), - FakeCommand( + const FakeCommand( command: [ 'xcrun', 'strip', '-x', - '$outputPath/App.framework/App', + 'build/foo/App.framework/App', '-o', - '$outputPath/App.framework/App', + 'build/foo/App.framework/App', ], ), ]); @@ -364,6 +441,7 @@ void main() { '--deterministic', '--snapshot_kind=app-aot-elf', '--elf=build/foo/app.so', + '--strip', '--no-sim-use-hardfp', '--no-use-integer-division', 'main.dill', @@ -397,6 +475,7 @@ void main() { '--deterministic', '--snapshot_kind=app-aot-elf', '--elf=build/foo/app.so', + '--strip', '--no-sim-use-hardfp', '--no-use-integer-division', '--dwarf-stack-traces', @@ -433,6 +512,7 @@ void main() { '--deterministic', '--snapshot_kind=app-aot-elf', '--elf=build/foo/app.so', + '--strip', '--no-sim-use-hardfp', '--no-use-integer-division', '--obfuscate', @@ -468,6 +548,7 @@ void main() { '--deterministic', '--snapshot_kind=app-aot-elf', '--elf=build/foo/app.so', + '--strip', '--no-sim-use-hardfp', '--no-use-integer-division', 'main.dill', @@ -502,6 +583,7 @@ void main() { '--deterministic', '--snapshot_kind=app-aot-elf', '--elf=build/foo/app.so', + '--strip', 'main.dill', ], ), diff --git a/packages/flutter_tools/test/general.shard/build_system/targets/common_test.dart b/packages/flutter_tools/test/general.shard/build_system/targets/common_test.dart index f4fc8918de625..a5e20b2386361 100644 --- a/packages/flutter_tools/test/general.shard/build_system/targets/common_test.dart +++ b/packages/flutter_tools/test/general.shard/build_system/targets/common_test.dart @@ -28,6 +28,17 @@ const kBoundaryKey = '4d2d9609-c662-4571-afde-31410f96caa6'; const kElfAot = '--snapshot_kind=app-aot-elf'; const kMachoDylibAot = '--snapshot_kind=app-aot-macho-dylib'; +/// Generate Shorebird link info arguments for iOS/macOS AOT builds. +/// The [buildPath] should be the build directory path (outputDir.parent.path). +List linkInfoArgsFor(String buildPath) => [ + '--print_class_table_link_debug_info_to=$buildPath/App.class_table.json', + '--print_class_table_link_info_to=$buildPath/App.ct.link', + '--print_field_table_link_debug_info_to=$buildPath/App.field_table.json', + '--print_field_table_link_info_to=$buildPath/App.ft.link', + '--print_dispatch_table_link_debug_info_to=$buildPath/App.dispatch_table.json', + '--print_dispatch_table_link_info_to=$buildPath/App.dt.link', +]; + final Platform macPlatform = FakePlatform( operatingSystem: 'macos', environment: {}, @@ -805,6 +816,7 @@ void main() { // This path is not known by the cache due to the iOS gen_snapshot split. 'Artifact.genSnapshotArm64.TargetPlatform.ios.profile', '--deterministic', + ...linkInfoArgsFor(build), '--write-v8-snapshot-profile-to=code_size_1/snapshot.arm64.json', '--trace-precompiler-to=code_size_1/trace.arm64.json', kMachoDylibAot, diff --git a/packages/flutter_tools/test/general.shard/build_system/targets/macos_test.dart b/packages/flutter_tools/test/general.shard/build_system/targets/macos_test.dart index ec6ea8dfc47c3..07fafef080da1 100644 --- a/packages/flutter_tools/test/general.shard/build_system/targets/macos_test.dart +++ b/packages/flutter_tools/test/general.shard/build_system/targets/macos_test.dart @@ -7,11 +7,9 @@ import 'package:file_testing/file_testing.dart'; import 'package:flutter_tools/src/artifacts.dart'; import 'package:flutter_tools/src/base/file_system.dart'; import 'package:flutter_tools/src/base/logger.dart'; -import 'package:flutter_tools/src/base/version.dart'; import 'package:flutter_tools/src/build_info.dart'; import 'package:flutter_tools/src/build_system/build_system.dart'; import 'package:flutter_tools/src/build_system/targets/macos.dart'; -import 'package:flutter_tools/src/features.dart'; import 'package:flutter_tools/src/ios/xcodeproj.dart'; import 'package:flutter_tools/src/project.dart'; import 'package:test/fake.dart'; @@ -23,6 +21,17 @@ import '../../../src/fake_process_manager.dart'; import '../../../src/fakes.dart'; import '../../../src/package_config.dart'; +/// Generate Shorebird link info arguments for iOS/macOS AOT builds. +/// The [buildPath] should be the build directory path (outputDir.parent.path). +List linkInfoArgsFor(String buildPath) => [ + '--print_class_table_link_debug_info_to=$buildPath/App.class_table.json', + '--print_class_table_link_info_to=$buildPath/App.ct.link', + '--print_field_table_link_debug_info_to=$buildPath/App.field_table.json', + '--print_field_table_link_info_to=$buildPath/App.ft.link', + '--print_dispatch_table_link_debug_info_to=$buildPath/App.dispatch_table.json', + '--print_dispatch_table_link_info_to=$buildPath/App.dt.link', +]; + void main() { late Environment environment; late MemoryFileSystem fileSystem; @@ -805,28 +814,36 @@ void main() { environment.defines[kXcodeAction] = 'install'; environment.defines[kFlavor] = 'internal'; + // Set up engine artifacts fileSystem .file('bin/cache/artifacts/engine/darwin-x64/vm_isolate_snapshot.bin') .createSync(recursive: true); fileSystem .file('bin/cache/artifacts/engine/darwin-x64/isolate_snapshot.bin') .createSync(recursive: true); + + // Set up App.framework binary fileSystem .file(fileSystem.path .join(environment.buildDir.path, 'App.framework', 'App')) .createSync(recursive: true); - final String shorebirdYamlPath = fileSystem.path.join( - environment.buildDir.path, - 'App.framework', - 'Versions', - 'A', - 'Resources', - 'flutter_assets', - 'shorebird.yaml', - ); - fileSystem.file(fileSystem.path - .join(environment.buildDir.path, 'App.framework', 'App')) - ..createSync(recursive: true) + + // Set up native_assets.json (required by MacOSBundleFlutterAssets) + environment.buildDir.childFile('native_assets.json').createSync(); + + // Set up pubspec.yaml with shorebird.yaml as an asset + fileSystem.file('pubspec.yaml') + ..createSync() + ..writeAsStringSync(''' +name: example +flutter: + assets: + - shorebird.yaml +'''); + + // Create the shorebird.yaml asset file + fileSystem.file('shorebird.yaml') + ..createSync() ..writeAsStringSync(''' # Some other text that should be removed app_id: base-app-id @@ -835,8 +852,21 @@ flavors: stable: stable-app-id '''); + // Set up package config + writePackageConfigFiles(directory: fileSystem.currentDirectory, mainLibName: 'example'); + await const ReleaseMacOSBundleFlutterAssets().build(environment); + // The output is in environment.outputDir, not buildDir + final String shorebirdYamlPath = fileSystem.path.join( + environment.outputDir.path, + 'App.framework', + 'Versions', + 'A', + 'Resources', + 'flutter_assets', + 'shorebird.yaml', + ); expect(fileSystem.file(shorebirdYamlPath).readAsStringSync(), 'app_id: internal-app-id'); }, @@ -906,17 +936,15 @@ flavors: .childFile('x86_64/App.framework.dSYM/Contents/Resources/DWARF/App') .createSync(recursive: true); + final build = environment.buildDir.path; processManager.addCommands([ FakeCommand( command: [ 'Artifact.genSnapshotArm64.TargetPlatform.darwin.release', '--deterministic', - '--snapshot_kind=app-aot-macho-dylib', - '--macho=${environment.buildDir.childFile('arm64/App.framework/App').path}', - '--macho-object=${environment.buildDir.childFile('arm64/app.o').path}', - '--macho-min-os-version=12.0', - '--macho-rpath=@executable_path/Frameworks,@loader_path/Frameworks', - '--macho-install-name=@rpath/App.framework/App', + ...linkInfoArgsFor(build), + '--snapshot_kind=app-aot-assembly', + '--assembly=${environment.buildDir.childFile('arm64/snapshot_assembly.S').path}', environment.buildDir.childFile('app.dill').path, ], ), @@ -924,15 +952,82 @@ flavors: command: [ 'Artifact.genSnapshotX64.TargetPlatform.darwin.release', '--deterministic', - '--snapshot_kind=app-aot-macho-dylib', - '--macho=${environment.buildDir.childFile('x86_64/App.framework/App').path}', - '--macho-object=${environment.buildDir.childFile('x86_64/app.o').path}', - '--macho-min-os-version=12.0', - '--macho-rpath=@executable_path/Frameworks,@loader_path/Frameworks', - '--macho-install-name=@rpath/App.framework/App', + ...linkInfoArgsFor(build), + '--snapshot_kind=app-aot-assembly', + '--assembly=${environment.buildDir.childFile('x86_64/snapshot_assembly.S').path}', environment.buildDir.childFile('app.dill').path, ], ), + FakeCommand( + command: [ + 'xcrun', + 'cc', + '-arch', + 'arm64', + '-c', + environment.buildDir.childFile('arm64/snapshot_assembly.S').path, + '-o', + environment.buildDir.childFile('arm64/snapshot_assembly.o').path, + ], + ), + FakeCommand( + command: [ + 'xcrun', + 'cc', + '-arch', + 'x86_64', + '-c', + environment.buildDir.childFile('x86_64/snapshot_assembly.S').path, + '-o', + environment.buildDir.childFile('x86_64/snapshot_assembly.o').path, + ], + ), + FakeCommand( + command: [ + 'xcrun', + 'clang', + '-arch', + 'arm64', + '-dynamiclib', + '-Xlinker', + '-rpath', + '-Xlinker', + '@executable_path/Frameworks', + '-Xlinker', + '-rpath', + '-Xlinker', + '@loader_path/Frameworks', + '-fapplication-extension', + '-install_name', + '@rpath/App.framework/App', + '-o', + environment.buildDir.childFile('arm64/App.framework/App').path, + environment.buildDir.childFile('arm64/snapshot_assembly.o').path, + ], + ), + FakeCommand( + command: [ + 'xcrun', + 'clang', + '-arch', + 'x86_64', + '-dynamiclib', + '-Xlinker', + '-rpath', + '-Xlinker', + '@executable_path/Frameworks', + '-Xlinker', + '-rpath', + '-Xlinker', + '@loader_path/Frameworks', + '-fapplication-extension', + '-install_name', + '@rpath/App.framework/App', + '-o', + environment.buildDir.childFile('x86_64/App.framework/App').path, + environment.buildDir.childFile('x86_64/snapshot_assembly.o').path, + ], + ), FakeCommand( command: [ 'xcrun', @@ -1009,45 +1104,14 @@ flavors: ProcessManager: () => processManager, }, ); - - group('FlutterMacOS output', () { - late MemoryFileSystem testFileSystem; - - setUp(() { - testFileSystem = MemoryFileSystem.test(); - testFileSystem.file('pubspec.yaml').createSync(recursive: true); - testFileSystem.directory('macos').createSync(recursive: true); - testFileSystem - .directory('macos/Flutter/ephemeral/Packages/.packages/FlutterFramework') - .createSync(recursive: true); - }); - testUsingContext( - 'included when not using SwiftPM', - () async { - const Target target = ReleaseUnpackMacOS(); - expect(target.outputs.contains(kFlutterMacOSFrameworkBinarySource), isTrue); - }, - overrides: { - FeatureFlags: () => TestFeatureFlags(), - XcodeProjectInterpreter: () => FakeXcodeProjectInterpreter(version: Version(15, 0, 0)), - }, - ); - }); } class FakeXcodeProjectInterpreter extends Fake implements XcodeProjectInterpreter { - FakeXcodeProjectInterpreter({ - this.isInstalled = true, - this.version, - this.schemes = const ['Runner'], - }); + FakeXcodeProjectInterpreter({this.isInstalled = true, this.schemes = const ['Runner']}); @override final bool isInstalled; - @override - final Version? version; - List schemes; @override diff --git a/packages/flutter_tools/test/general.shard/cache_test.dart b/packages/flutter_tools/test/general.shard/cache_test.dart index ce8133a794701..d0ce7ac81e3bc 100644 --- a/packages/flutter_tools/test/general.shard/cache_test.dart +++ b/packages/flutter_tools/test/general.shard/cache_test.dart @@ -1400,7 +1400,7 @@ void main() { expect(messages, ['Web SDK']); expect(downloads, [ - 'https://storage.googleapis.com/flutter_infra_release/flutter/hijklmnop/flutter-web-sdk.zip', + 'https://download.shorebird.dev/flutter_infra_release/flutter/hijklmnop/flutter-web-sdk.zip', ]); expect(locations, ['/bin/cache/flutter_web_sdk']); @@ -1521,7 +1521,7 @@ void main() { expect(messages, ['Engine Information']); expect(downloads, [ - 'https://storage.googleapis.com/flutter_infra_release/flutter/hijklmnop/engine_stamp.json', + 'https://download.shorebird.dev/flutter_infra_release/flutter/hijklmnop/engine_stamp.json', ]); expect(locations, ['/bin/cache']); // file copy is done by the real uploader; not the fake. diff --git a/packages/flutter_tools/test/general.shard/version_test.dart b/packages/flutter_tools/test/general.shard/version_test.dart index 90cf4a2ccb181..e52ecd768348e 100644 --- a/packages/flutter_tools/test/general.shard/version_test.dart +++ b/packages/flutter_tools/test/general.shard/version_test.dart @@ -67,6 +67,20 @@ void main() { required int commitsBetweenRefs, }) { return [ + // Shorebird release branch check (returns empty to fall through to + // regular version lookup). + FakeCommand( + command: [ + 'git', + 'for-each-ref', + '--contains', + headRef, + '--format', + '%(refname:short)', + 'refs/remotes/origin/flutter_release/*', + ], + stdout: '', + ), FakeCommand( command: const [ 'git', From 2b3dde57153bbe71356a40b4f3f0919acd964f5e Mon Sep 17 00:00:00 2001 From: Eric Seidel Date: Fri, 12 Dec 2025 14:50:17 -0800 Subject: [PATCH 10/95] chore: move build_engine scripts into this repo (#98) * chore: move build_engine scripts into this repo * chore: fix path of content_aware_hash.sh --- shorebird/ci/internal/generate_manifest.sh | 93 ++++++++++ shorebird/ci/internal/linux_build.sh | 105 +++++++++++ shorebird/ci/internal/linux_setup.sh | 30 +++ shorebird/ci/internal/linux_test_build.sh | 29 +++ shorebird/ci/internal/linux_upload.sh | 138 ++++++++++++++ shorebird/ci/internal/mac_build.sh | 204 +++++++++++++++++++++ shorebird/ci/internal/mac_setup.sh | 11 ++ shorebird/ci/internal/mac_upload.sh | 180 ++++++++++++++++++ shorebird/ci/internal/win_build.sh | 63 +++++++ shorebird/ci/internal/win_setup.sh | 8 + shorebird/ci/internal/win_upload.sh | 90 +++++++++ shorebird/ci/linux_build_and_upload.sh | 32 ++++ shorebird/ci/mac_build_and_upload.sh | 32 ++++ shorebird/ci/win_build_and_upload.sh | 32 ++++ 14 files changed, 1047 insertions(+) create mode 100755 shorebird/ci/internal/generate_manifest.sh create mode 100755 shorebird/ci/internal/linux_build.sh create mode 100755 shorebird/ci/internal/linux_setup.sh create mode 100755 shorebird/ci/internal/linux_test_build.sh create mode 100755 shorebird/ci/internal/linux_upload.sh create mode 100755 shorebird/ci/internal/mac_build.sh create mode 100755 shorebird/ci/internal/mac_setup.sh create mode 100755 shorebird/ci/internal/mac_upload.sh create mode 100755 shorebird/ci/internal/win_build.sh create mode 100755 shorebird/ci/internal/win_setup.sh create mode 100755 shorebird/ci/internal/win_upload.sh create mode 100755 shorebird/ci/linux_build_and_upload.sh create mode 100755 shorebird/ci/mac_build_and_upload.sh create mode 100755 shorebird/ci/win_build_and_upload.sh diff --git a/shorebird/ci/internal/generate_manifest.sh b/shorebird/ci/internal/generate_manifest.sh new file mode 100755 index 0000000000000..4c95de6baf098 --- /dev/null +++ b/shorebird/ci/internal/generate_manifest.sh @@ -0,0 +1,93 @@ +#!/bin/bash + +# This script outputs an artifact_manifest.yaml mapping +# a shorebird engine revision to a flutter engine revision. +# Usage: +# ./generate_manifest.sh > artifact_manifest.yaml + +set -e + +# NOTE: If you edit this file you also may need to edit the global list +# of all known artifacts in the artifact_proxy's config.dart + +if [ "$#" -ne 1 ]; then + echo "Usage: ./generate_manifest.sh " + exit 1 +fi + +FLUTTER_ENGINE_REVISION=$1 + +cat <, but that no longer seems needed. +# We always use the hermetic NDK from the engine repo. +ANDROID_NDK_HOME="$ENGINE_SRC/flutter/third_party/android_tools/ndk" \ +cargo ndk \ + --target armv7-linux-androideabi \ + --target aarch64-linux-android \ + --target i686-linux-android \ + --target x86_64-linux-android \ + build --release + +cargo build --release --target x86_64-unknown-linux-gnu + +# Build the patch tool. +cd $UPDATER_SRC/patch +cargo build --release + +# Compile the engine using the steps here: +# https://github.com/flutter/flutter/wiki/Compiling-the-engine#compiling-for-android-from-macos-or-linux +cd $ENGINE_SRC + +NINJA="ninja" +GN=./flutter/tools/gn +# We could probably use our own prebuilt dart SDK, by modifying the gn files. +GN_ARGS="--no-rbe --no-enable-unittests" + +# We could use Linux to generate all of our Android binaries, but we don't yet. +# https://github.com/flutter/engine/blob/e590b24f3962fda3ec9144dcee3f7565b195839a/ci/builders/linux_android_aot_engine.json#L40 + +# Build the default and gen_snapshot targets. +# +# Linux doesn't seem to use "archive_gen_snapshot" as a target name yet. +# https://github.com/flutter/flutter/issues/105351#issuecomment-1650686247 +ANDROID_TARGETS="default gen_snapshot" + +# Android arm64 release +$GN $GN_ARGS --android --android-cpu=arm64 --runtime-mode=release +$NINJA -C ./out/android_release_arm64 $ANDROID_TARGETS + +# Android arm32 release +$GN $GN_ARGS --runtime-mode=release --android +$NINJA -C out/android_release $ANDROID_TARGETS + +# Android x64 release +$GN $GN_ARGS --android --android-cpu=x64 --runtime-mode=release +$NINJA -C ./out/android_release_x64 $ANDROID_TARGETS + +# Build Dart and Flutter +$GN $GN_ARGS --runtime-mode=release --no-prebuilt-dart-sdk +# build Dart and the linux shell and flutter_patched_sdk_product.zip +$NINJA -C out/host_release dart_sdk flutter/shell/platform/linux:flutter_gtk flutter/build/archives:flutter_patched_sdk +# We want to build flutter/tools/font_subset, but that doesn't work with +# --no-prebuilt-dart-sdk. +# https://github.com/flutter/flutter/issues/164531 + +# Build debug Linux artifacts +# These are output to the `linux-x64` directory in host_debug, and are used +# by `flutter build linux --release`. +$GN $GN_ARGS --no-prebuilt-dart-sdk +$NINJA -C ./out/host_debug flutter/build/archives:artifacts + +# Shorebird AOT Tools (Linker) +mkdir -p $ENGINE_OUT/host_release/aot_tools + +# Dart kernel (.dill) files are not stable and can change with the version of Dart, so we +# can't use this machine's `dart`. Here we're using the version of Dart that this +# version of the engine depends on, which should be the same version that +# `flutter` ends up depending on. +DART=$ENGINE_OUT/host_release/dart-sdk/bin/dart +AOT_TOOLS_PKG=$ENGINE_SRC/flutter/third_party/dart/pkg/aot_tools +# This should be part of `gclient sync` https://github.com/shorebirdtech/_build_engine/issues/113 +(cd $AOT_TOOLS_PKG; $DART pub get) +# This should be built as part of Dart and then pulled down as part of the engine build. +# https://github.com/shorebirdtech/_build_engine/issues/88 +$DART compile kernel $AOT_TOOLS_PKG/bin/aot_tools.dart -o $ENGINE_OUT/host_release/aot_tools/aot-tools.dill + +mkdir -p $ENGINE_OUT/host_release/updater_tools +UPDATER_TOOLS_PKG=$ENGINE_SRC/flutter/third_party/updater/updater_tools +# This should be part of `gclient sync` https://github.com/shorebirdtech/_build_engine/issues/113 + +# We could also build the `patch` tool for Linux here. diff --git a/shorebird/ci/internal/linux_setup.sh b/shorebird/ci/internal/linux_setup.sh new file mode 100755 index 0000000000000..626ad211536eb --- /dev/null +++ b/shorebird/ci/internal/linux_setup.sh @@ -0,0 +1,30 @@ +#!/bin/bash -e + +# Usage: +# ./linux_setup.sh + +# Per https://github.com/flutter/flutter/wiki/Setting-up-the-Engine-development-environment +# Subset of ./flutter/build/install-build-deps-linux-desktop.sh +sudo apt -y install libfreetype6-dev pkg-config + +# This assumes rust is installed, but could also install rust/cargo. + +# Need NDK from https://developer.android.com/ndk/downloads +# The NDK version should match that of DEPS, e.g. +# https://github.com/flutter/flutter/blame/b45fa18946ecc2d9b4009952c636ba7e2ffbb787/DEPS#L615 +# Example: +# curl -O https://dl.google.com/android/repository/android-ndk-r27d-linux.zip +# unzip android-ndk-r27d-linux.zip +# On the GHA runners we set this in .github/workflows/build_engine.yaml +# env: +# NDK_HOME: /home/gha/bin/android-ndk-r27d + +# We require an old version of cargo-ndk to support the old NDK Flutter +# engine currently uses. +cargo install cargo-ndk +rustup target add \ + aarch64-linux-android \ + armv7-linux-androideabi \ + x86_64-linux-android \ + i686-linux-android \ + x86_64-unknown-linux-gnu diff --git a/shorebird/ci/internal/linux_test_build.sh b/shorebird/ci/internal/linux_test_build.sh new file mode 100755 index 0000000000000..be3c54da547ec --- /dev/null +++ b/shorebird/ci/internal/linux_test_build.sh @@ -0,0 +1,29 @@ +#!/bin/bash -e + +# The path to the Flutter engine. +# Convert to an absolute path so we don't need to worry about cd'ing back to +# the root directory between commands. +ENGINE_ROOT=$(realpath $1) +ENGINE_SRC=$ENGINE_ROOT/src + +cd $ENGINE_SRC + +UPDATER_SRC=$ENGINE_SRC/flutter/third_party/updater +(cd $UPDATER_SRC && + ANDROID_NDK_HOME="$ENGINE_SRC/flutter/third_party/android_tools/ndk" \ + cargo ndk \ + --target armv7-linux-androideabi \ + --target aarch64-linux-android \ + --target i686-linux-android \ + --target x86_64-linux-android \ + build --release && + cargo build --release --target x86_64-unknown-linux-gnu +) + +# Generate an unoptimized debug build of the engine (expected by the test script). +./flutter/tools/gn --unoptimized --no-rbe +ninja -C out/host_debug_unopt + +# Generate an unoptimized android debug build for java engine tests +./flutter/tools/gn --android --unoptimized --no-rbe +ninja -C out/android_debug_unopt diff --git a/shorebird/ci/internal/linux_upload.sh b/shorebird/ci/internal/linux_upload.sh new file mode 100755 index 0000000000000..873f0f3edb969 --- /dev/null +++ b/shorebird/ci/internal/linux_upload.sh @@ -0,0 +1,138 @@ +#!/bin/bash -e + +# Usage: +# ./linux_upload.sh engine_path git_hash + +# Convert to an absolute path so we don't need to worry about cd'ing back to +# the root directory between commands. +ENGINE_ROOT=$(realpath $1) +ENGINE_HASH=$2 + +STORAGE_BUCKET="download.shorebird.dev" +SHOREBIRD_ROOT=gs://$STORAGE_BUCKET/shorebird/$ENGINE_HASH + +ENGINE_SRC=$ENGINE_ROOT/src +ENGINE_OUT=$ENGINE_SRC/out +ENGINE_FLUTTER=$ENGINE_SRC/flutter +# FLUTTER_ROOT is the Flutter monorepo root (parent of engine/) +FLUTTER_ROOT=$(dirname $ENGINE_ROOT) + +cd $FLUTTER_ROOT + +# Compute the content-aware hash for the Dart SDK. +# This allows Flutter checkouts that haven't changed engine content to share +# the same pre-built Dart SDK, even if they have different git commit SHAs. +CONTENT_HASH=$($FLUTTER_ROOT/bin/internal/content_aware_hash.sh) + +# We do not generate a manifest file, we assume another builder did that. +# TODO(bryanoltman): should we generate a manifest file as part of an upload +# script, or should it be done once all build and uploads have completed? +# See https://github.com/shorebirdtech/build_engine/issues/25 + +# TODO(eseidel): This should not be in shell, it's too complicated/repetitive. + +HOST_ARCH='linux-x64' + +INFRA_ROOT="gs://$STORAGE_BUCKET/flutter_infra_release/flutter/$ENGINE_HASH" +MAVEN_VER="1.0.0-$ENGINE_HASH" +MAVEN_ROOT="gs://$STORAGE_BUCKET/download.flutter.io/io/flutter" + +# Dart SDK +# This gets uploaded to flutter_infra_release/flutter/\$engine/dart-sdk-$HOST_ARCH.zip +# We also upload to the content-aware hash path to support local development branches. +CONTENT_INFRA_ROOT="gs://$STORAGE_BUCKET/flutter_infra_release/flutter/$CONTENT_HASH" + +HOST_RELEASE=$ENGINE_OUT/host_release +DART_ZIP_FILE=dart-sdk-$HOST_ARCH.zip +( + cd $HOST_RELEASE; + zip -r $DART_ZIP_FILE dart-sdk +) +ZIPS_DEST=$INFRA_ROOT/$DART_ZIP_FILE +gsutil cp $HOST_RELEASE/$DART_ZIP_FILE $ZIPS_DEST +# Also upload to content-aware hash path +gsutil cp $HOST_RELEASE/$DART_ZIP_FILE $CONTENT_INFRA_ROOT/$DART_ZIP_FILE + +# Android Arm64 release Flutter artifacts +ARCH_OUT=$ENGINE_OUT/android_release_arm64 +ZIPS_OUT=$ARCH_OUT/zip_archives/android-arm64-release +ZIPS_DEST=$INFRA_ROOT/android-arm64-release +gsutil cp $ZIPS_OUT/artifacts.zip $ZIPS_DEST/artifacts.zip +gsutil cp $ZIPS_OUT/$HOST_ARCH.zip $ZIPS_DEST/$HOST_ARCH.zip +gsutil cp $ZIPS_OUT/symbols.zip $ZIPS_DEST/symbols.zip +# Android Arm64 release Maven artifacts +ARCH_PATH=$ARCH_OUT/arm64_v8a_release +MAVEN_PATH=$MAVEN_ROOT/arm64_v8a_release/$MAVEN_VER/arm64_v8a_release-$MAVEN_VER +gsutil cp $ARCH_PATH.pom $MAVEN_PATH.pom +gsutil cp $ARCH_PATH.jar $MAVEN_PATH.jar +gsutil cp $ARCH_PATH.maven-metadata.xml $MAVEN_PATH.maven-metadata.xml + +# Android Arm32 release Flutter artifacts +ARCH_OUT=$ENGINE_OUT/android_release +ZIPS_OUT=$ARCH_OUT/zip_archives/android-arm-release +ZIPS_DEST=$INFRA_ROOT/android-arm-release +gsutil cp $ZIPS_OUT/artifacts.zip $ZIPS_DEST/artifacts.zip +gsutil cp $ZIPS_OUT/$HOST_ARCH.zip $ZIPS_DEST/$HOST_ARCH.zip +gsutil cp $ZIPS_OUT/symbols.zip $ZIPS_DEST/symbols.zip +# Android Arm32 release Maven artifacts +ARCH_PATH=$ARCH_OUT/armeabi_v7a_release +MAVEN_PATH=$MAVEN_ROOT/armeabi_v7a_release/$MAVEN_VER/armeabi_v7a_release-$MAVEN_VER +gsutil cp $ARCH_PATH.pom $MAVEN_PATH.pom +gsutil cp $ARCH_PATH.jar $MAVEN_PATH.jar +gsutil cp $ARCH_PATH.maven-metadata.xml $MAVEN_PATH.maven-metadata.xml + +# Not sure which flutter_embedding_release files to use? 32 or 64 bit? +# It does not seem to contain the libflutter.so file, but does seem to +# differ between the two build dirs. +ARCH_OUT=$ENGINE_OUT/android_release +ARCH_PATH=$ARCH_OUT/flutter_embedding_release +MAVEN_PATH=$MAVEN_ROOT/flutter_embedding_release/$MAVEN_VER/flutter_embedding_release-$MAVEN_VER +gsutil cp $ARCH_PATH.pom $MAVEN_PATH.pom +gsutil cp $ARCH_PATH.jar $MAVEN_PATH.jar +gsutil cp $ARCH_PATH.maven-metadata.xml $MAVEN_PATH.maven-metadata.xml + +# Android x64 release Flutter artifacts +ARCH_OUT=$ENGINE_OUT/android_release_x64 +ZIPS_OUT=$ARCH_OUT/zip_archives/android-x64-release +ZIPS_DEST=$INFRA_ROOT/android-x64-release +gsutil cp $ZIPS_OUT/artifacts.zip $ZIPS_DEST/artifacts.zip +gsutil cp $ZIPS_OUT/$HOST_ARCH.zip $ZIPS_DEST/$HOST_ARCH.zip +gsutil cp $ZIPS_OUT/symbols.zip $ZIPS_DEST/symbols.zip +# Android x64 release Maven artifacts +ARCH_PATH=$ARCH_OUT/x86_64_release +MAVEN_PATH=$MAVEN_ROOT/x86_64_release/$MAVEN_VER/x86_64_release-$MAVEN_VER +gsutil cp $ARCH_PATH.pom $MAVEN_PATH.pom +gsutil cp $ARCH_PATH.jar $MAVEN_PATH.jar +gsutil cp $ARCH_PATH.maven-metadata.xml $MAVEN_PATH.maven-metadata.xml + +# Shorebird AOT Tools (Linker) +gsutil cp $ENGINE_OUT/host_release/aot_tools/aot-tools.dill $SHOREBIRD_ROOT/aot-tools.dill + +# Common Product-mode artifacts +ARCH_OUT=$ENGINE_OUT/host_release +ZIPS_OUT=$ARCH_OUT/zip_archives +ZIPS_DEST=$INFRA_ROOT +gsutil cp $ZIPS_OUT/flutter_patched_sdk_product.zip $ZIPS_DEST/flutter_patched_sdk_product.zip + +# We want to build flutter/tools/font_subset, but that doesn't work with +# --no-prebuilt-dart-sdk. +# https://github.com/flutter/flutter/issues/164531 +# Linux x64 host_release font_subset (ConstFinder) +# ARCH_OUT=$ENGINE_OUT/host_release +# ZIPS_OUT=$ARCH_OUT/zip_archives/$HOST_ARCH +# ZIPS_DEST=$INFRA_ROOT/linux-x64-release +# gsutil cp $ZIPS_OUT/font-subset.zip $ZIPS_DEST/font-subset.zip + +# Linux Desktop Support +ARCH_OUT=$ENGINE_OUT/host_release +ZIPS_OUT=$ARCH_OUT/zip_archives/linux-x64-release +ZIPS_DEST=$INFRA_ROOT/linux-x64-release +gsutil cp $ZIPS_OUT/linux-x64-flutter-gtk.zip $ZIPS_DEST/linux-x64-flutter-gtk.zip + +ARCH_OUT=$ENGINE_OUT/host_debug +ZIPS_OUT=$ARCH_OUT/zip_archives/$HOST_ARCH +ZIPS_DEST=$INFRA_ROOT/$HOST_ARCH +gsutil cp $ZIPS_OUT/artifacts.zip $ZIPS_DEST/artifacts.zip + +# We could upload patch if we built it here. +# gsutil cp $ENGINE_OUT/host_release/patch.zip $SHOREBIRD_ROOT/patch-win-x64.zip diff --git a/shorebird/ci/internal/mac_build.sh b/shorebird/ci/internal/mac_build.sh new file mode 100755 index 0000000000000..a860da431f269 --- /dev/null +++ b/shorebird/ci/internal/mac_build.sh @@ -0,0 +1,204 @@ +#!/bin/bash -e + +# FIXME: This script should be deleted and instead these steps be part +# of the GN build process. +# I haven't investigated how to build rust from GN with the Android NDK yet. + +# Usage: +# ./mac_build.sh engine_path + +if [ "$#" -ne 1 ]; then + echo "Usage: $0 engine_path" + exit 1 +fi + +# Convert to an absolute path so we don't need to worry about cd'ing back to +# the root directory between commands. +ENGINE_ROOT=$(realpath $1) + +ENGINE_SRC=$ENGINE_ROOT/src +ENGINE_OUT=$ENGINE_SRC/out +UPDATER_SRC=$ENGINE_SRC/flutter/third_party/updater +HOST_ARCH='darwin-x64' + +# Build the Rust library. +cd $UPDATER_SRC/library + +# Build iOS and MacOS +cargo build \ + --target aarch64-apple-ios \ + --target x86_64-apple-ios \ + --target aarch64-apple-darwin \ + --target x86_64-apple-darwin \ + --release + +# Build the patch tool. +# Again, this belongs as part of the gn build. +cd $UPDATER_SRC/patch +cargo build --release + +# Compile the engine using the steps here: +# https://github.com/flutter/flutter/wiki/Compiling-the-engine#compiling-for-android-from-macos-or-linux +cd $ENGINE_SRC + +NINJA="ninja" +GN=./flutter/tools/gn +ET=./flutter/bin/et +# We could probably use our own prebuilt dart SDK, by modifying the gn files. +# `--no-enable-unittests` is needed on Flutter 3.10.1 and 3.10.2 to avoid +# https://github.com/flutter/flutter/issues/128135 +GN_ARGS="--no-rbe --no-enable-unittests" + +# FIXME: These build commands likely could build fewer targets. + +# Mac doesn't seem to use "archive_gen_snapshot" as a target name yet. +# https://github.com/flutter/flutter/issues/105351#issuecomment-1650686247 +ANDROID_TARGETS="flutter/shell/platform/android:gen_snapshot" + +# Because Flutter does not yet build universal binaries for macOS, we need to +# ensure we're building for x64 for the time being so we can support both Intel +# and Apple Silicon Macs. We do this by telling gn to use host_cpu="x64". + +# Android arm64 release +$GN $GN_ARGS --android --android-cpu=arm64 --runtime-mode=release --gn-args='host_cpu="x64"' +$NINJA -C ./out/android_release_arm64 $ANDROID_TARGETS + +# Android arm32 release +$GN $GN_ARGS --runtime-mode=release --android --gn-args='host_cpu="x64"' +$NINJA -C out/android_release $ANDROID_TARGETS + +# Android x64 release +$GN $GN_ARGS --android --android-cpu=x64 --runtime-mode=release --gn-args='host_cpu="x64"' +$NINJA -C ./out/android_release_x64 $ANDROID_TARGETS + +# We only need two targets (per the mac builders): +# "flutter/shell/platform/darwin/ios:flutter_framework", +# "flutter/lib/snapshot:generate_snapshot_bins", which builds both gen_snapshot and analyze_snapshot binaries. +# https://github.com/flutter/engine/blob/main/ci/builders/mac_ios_engine.json#L139 +# https://github.com/flutter/engine/blob/main/ci/builders/README.md +# The files created by these targets are packaged into a framework and an artifacts.zip file +# by the create_full_ios_framework.py and create_macos_framework.py scripts. + +IOS_TARGETS="flutter/shell/platform/darwin/ios:flutter_framework flutter/lib/snapshot:generate_snapshot_bins" +# You will also need to build vm_platform_strong.dill if you're using a local engine build. + +# From ci/builders/mac_host_engine.json in the engine repo +MACOS_TARGETS="flutter/shell/platform/darwin/macos:zip_macos_flutter_framework flutter/lib/snapshot:generate_snapshot_bins flutter/build/archives:artifacts" + +# Build x64 Dart SDK +$GN $GN_ARGS --runtime-mode=release --mac-cpu=x64 --no-prebuilt-dart-sdk +$NINJA -C out/host_release dart_sdk +# We want to build flutter/tools/font_subset, but that doesn't work with +# --no-prebuilt-dart-sdk. +# https://github.com/flutter/flutter/issues/164531 + +# Build arm64 Dart SDK +$GN $GN_ARGS --runtime-mode=release --mac-cpu=arm64 --no-prebuilt-dart-sdk +$NINJA -C out/host_release_arm64 dart_sdk +# We want to build flutter/tools/font_subset, but that doesn't work with +# --no-prebuilt-dart-sdk. +# https://github.com/flutter/flutter/issues/164531 + +# iOS arm64 release +$GN $GN_ARGS --runtime-mode=release --ios --gn-arg='shorebird_runtime=true' +$NINJA -C out/ios_release $IOS_TARGETS + +$GN $GN_ARGS --ios --runtime-mode=release --darwin-extension-safe --xcode-symlinks --gn-arg='shorebird_runtime=true' +$NINJA -C out/ios_release_extension_safe $IOS_TARGETS + +# iOS simulator-x64 release +$GN $GN_ARGS --runtime-mode=debug --ios --simulator +$NINJA -C out/ios_debug_sim $IOS_TARGETS + +$GN $GN_ARGS --runtime-mode=debug --darwin-extension-safe --ios --simulator +$NINJA -C out/ios_debug_sim_extension_safe $IOS_TARGETS + +# iOS simulator-arm64 release +$GN $GN_ARGS --runtime-mode=debug --ios --simulator --simulator-cpu=arm64 +$NINJA -C out/ios_debug_sim_arm64 $IOS_TARGETS + +$GN $GN_ARGS --runtime-mode=debug --darwin-extension-safe --ios --simulator --simulator-cpu=arm64 +$NINJA -C out/ios_debug_sim_arm64_extension_safe $IOS_TARGETS + +# macOS arm64 release +$GN $GN_ARGS --runtime-mode=release --mac --mac-cpu=arm64 +$NINJA -C out/mac_release_arm64 $MACOS_TARGETS + +# macOS x64 release +# Note: we don't enable the simulator here because the simulator is an arm64 simulator, +# which won't work for x64 apps. +$GN $GN_ARGS --runtime-mode=release --mac --mac-cpu=x64 +$NINJA -C out/mac_release $MACOS_TARGETS + +# The python scripts below fail if the out/release directory already exists. +rm -rf out/release + +# We have to create a composite Flutter.framework for iOS and macOS, matching +# what the Flutter builders do: +IOS_FRAMEWORK_OUT=out/release +echo "Building Flutter.framework for iOS" +python3 flutter/sky/tools/create_ios_framework.py \ + --dst $IOS_FRAMEWORK_OUT \ + --arm64-out-dir out/ios_release \ + --simulator-x64-out-dir out/ios_debug_sim \ + --simulator-arm64-out-dir out/ios_debug_sim_arm64 \ + --dsym \ + --strip +echo "Built Flutter.framework for iOS" + +MAC_FRAMEWORK_OUT=out/release/framework +echo "Building Flutter.framework for macOS" +python3 flutter/sky/tools/create_macos_framework.py \ + --dst $MAC_FRAMEWORK_OUT \ + --arm64-out-dir out/mac_release_arm64 \ + --x64-out-dir out/mac_release \ + --dsym \ + --strip \ + --zip +echo "Built Flutter.framework for macOS" + +echo "Creating macOS gen_snapshot" +python3 flutter/sky/tools/create_macos_gen_snapshots.py \ + --dst out/release/snapshot \ + --arm64-path out/mac_release_arm64/universal/gen_snapshot_arm64 \ + --x64-path out/mac_release/universal/gen_snapshot_x64 \ + --zip +echo "Created macOS gen_snapshot" + +# Zip the dSYMs +zip -r $IOS_FRAMEWORK_OUT/Flutter.framework.dSYM.zip $IOS_FRAMEWORK_OUT/Flutter.framework.dSYM +zip -r $MAC_FRAMEWORK_OUT/FlutterMacOS.framework.dSYM.zip $MAC_FRAMEWORK_OUT/FlutterMacOS.framework.dSYM + +sign_flutter_xcframework() { + pushd $ENGINE_OUT/release + + # Unzip the artifacts zip file, which contains the Flutter.xcframework. + rm -rf artifacts + unzip artifacts.zip -d artifacts + + # Keep a copy of the old artifacts.zip for now, we may decide to remove this later + mv artifacts.zip artifacts.old.zip + + # Sign the Flutter.xcframework + cd artifacts + + # In case the artifacts are already signed, remove the signature + codesign -v --remove-signature Flutter.xcframework + codesign -v --sign "Apple Distribution: Code Town Inc (6V53YACS2W)" Flutter.xcframework + + # Zip the artifacts back up + zip -r "../artifacts.zip" * + + # Cleanup + cd .. + rm -rf artifacts + + popd +} + +sign_flutter_xcframework + +# Create out/engine_stamp.json +# We can remove this explicit step once we're using et in any of the lines +# above. +$ET stamp diff --git a/shorebird/ci/internal/mac_setup.sh b/shorebird/ci/internal/mac_setup.sh new file mode 100755 index 0000000000000..8453f8b9b1a9e --- /dev/null +++ b/shorebird/ci/internal/mac_setup.sh @@ -0,0 +1,11 @@ +#!/bin/bash -e + +# Usage: +# ./mac_setup.sh + +# This assumes rust is installed, but could also install rust/cargo. +rustup target add \ + x86_64-apple-ios \ + aarch64-apple-ios \ + aarch64-apple-darwin \ + x86_64-apple-darwin diff --git a/shorebird/ci/internal/mac_upload.sh b/shorebird/ci/internal/mac_upload.sh new file mode 100755 index 0000000000000..8ec266246a8e1 --- /dev/null +++ b/shorebird/ci/internal/mac_upload.sh @@ -0,0 +1,180 @@ +#!/bin/bash -e + +# Usage: +# ./mac_upload.sh engine_path git_hash + +# Convert to an absolute path so we don't need to worry about cd'ing back to +# the root directory between commands. +ENGINE_ROOT=$(realpath $1) +ENGINE_HASH=$2 + +# Get the absolute path to the directory of this script. +SCRIPT_DIR=$(cd $(dirname $0) && pwd) + +STORAGE_BUCKET="download.shorebird.dev" +SHOREBIRD_ROOT=gs://$STORAGE_BUCKET/shorebird/$ENGINE_HASH + +ENGINE_SRC=$ENGINE_ROOT/src +ENGINE_OUT=$ENGINE_SRC/out +ENGINE_FLUTTER=$ENGINE_SRC/flutter +# FLUTTER_ROOT is the Flutter monorepo root (parent of engine/) +FLUTTER_ROOT=$(dirname $ENGINE_ROOT) + +cd $FLUTTER_ROOT + +# Compute the content-aware hash for the Dart SDK. +# This allows Flutter checkouts that haven't changed engine content to share +# the same pre-built Dart SDK, even if they have different git commit SHAs. +CONTENT_HASH=$($FLUTTER_ROOT/bin/internal/content_aware_hash.sh) +# Can't just `git merge-base` because the engine branches for each +# major version (e.g. 3.7, 3.8) (e.g. upstream/flutter-3.7-candidate.1) +# but it's not clear which branch we're forked from, only that we took +# some tag and added our commits (but we don't know what tag). +BASE_FLUTTER_TAG=`git describe --tags --abbrev=0` +# Read the first line from bin/internal/engine.version file and trim whitespace. +BASE_ENGINE_HASH=`git show $BASE_FLUTTER_TAG:bin/internal/engine.version | head -n 1 | tr -d '[:space:]'` + +# Build the artifacts manifest: +MANIFEST_FILE=`mktemp` +# Note that any uploads which are *not* listed in the manifest will be +# ignored by the artifact proxy. +# if you add uploads here, they also need to be reflected in the manifest. +$SCRIPT_DIR/generate_manifest.sh $BASE_ENGINE_HASH > $MANIFEST_FILE + +# FIXME: This should not be in shell, it's too complicated/repetitive. +# Only need the libflutter.so (and flutter.jar) artifacts +# Artifact list: https://github.com/shorebirdtech/shorebird/blob/main/packages/artifact_proxy/lib/config.dart + +HOST_ARCH='darwin-x64' +ARM64_HOST_ARCH='darwin-arm64' + +INFRA_ROOT="gs://$STORAGE_BUCKET/flutter_infra_release/flutter/$ENGINE_HASH" + +# engine_stamp.json +ENGINE_STAMP_FILE=$ENGINE_OUT/engine_stamp.json +gsutil cp $ENGINE_STAMP_FILE $INFRA_ROOT/engine_stamp.json + +# Dart SDK +# This gets uploaded to flutter_infra_release/flutter/\$engine/dart-sdk-$HOST_ARCH.zip +# We also upload to the content-aware hash path to support local development branches. +CONTENT_INFRA_ROOT="gs://$STORAGE_BUCKET/flutter_infra_release/flutter/$CONTENT_HASH" + +# x64 Dart SDK +HOST_RELEASE=$ENGINE_OUT/host_release +DART_ZIP_FILE=dart-sdk-$HOST_ARCH.zip +( + cd $HOST_RELEASE; + zip -r $DART_ZIP_FILE dart-sdk +) +ZIPS_DEST=$INFRA_ROOT/$DART_ZIP_FILE +gsutil cp $HOST_RELEASE/$DART_ZIP_FILE $ZIPS_DEST +# Also upload to content-aware hash path +gsutil cp $HOST_RELEASE/$DART_ZIP_FILE $CONTENT_INFRA_ROOT/$DART_ZIP_FILE + +# arm64 Dart SDK +HOST_RELEASE_ARM64=$ENGINE_OUT/host_release_arm64 +DART_ZIP_FILE=dart-sdk-$ARM64_HOST_ARCH.zip +( + cd $HOST_RELEASE_ARM64; + zip -r $DART_ZIP_FILE dart-sdk +) +ZIPS_DEST=$INFRA_ROOT/$DART_ZIP_FILE +gsutil cp $HOST_RELEASE_ARM64/$DART_ZIP_FILE $ZIPS_DEST +# Also upload to content-aware hash path +gsutil cp $HOST_RELEASE_ARM64/$DART_ZIP_FILE $CONTENT_INFRA_ROOT/$DART_ZIP_FILE + +# We want to build flutter/tools/font_subset, but that doesn't work with +# --no-prebuilt-dart-sdk. +# https://github.com/flutter/flutter/issues/164531 +# # mac x64 host_release font_subset (ConstFinder) +# ARCH_OUT=$ENGINE_OUT/host_release +# ZIPS_OUT=$ARCH_OUT/zip_archives/$HOST_ARCH +# ZIPS_DEST=$INFRA_ROOT/darwin-x64-release +# gsutil cp $ZIPS_OUT/font-subset.zip $ZIPS_DEST/font-subset.zip + +# # mac arm64 host_release font_subset (ConstFinder) +# ARCH_OUT=$ENGINE_OUT/host_release_arm64 +# ZIPS_OUT=$ARCH_OUT/zip_archives/$HOST_ARCH +# ZIPS_DEST=$INFRA_ROOT/darwin-arm64-release +# gsutil cp $ZIPS_OUT/font-subset.zip $ZIPS_DEST/font-subset.zip + +# Android Arm64 release gen_snapshot +ARCH_OUT=$ENGINE_OUT/android_release_arm64 +ZIPS_OUT=$ARCH_OUT/zip_archives/android-arm64-release +ZIPS_DEST=$INFRA_ROOT/android-arm64-release +gsutil cp $ZIPS_OUT/$HOST_ARCH.zip $ZIPS_DEST/$HOST_ARCH.zip + +# Android Arm32 release gen_snapshot +ARCH_OUT=$ENGINE_OUT/android_release +ZIPS_OUT=$ARCH_OUT/zip_archives/android-arm-release +ZIPS_DEST=$INFRA_ROOT/android-arm-release +gsutil cp $ZIPS_OUT/$HOST_ARCH.zip $ZIPS_DEST/$HOST_ARCH.zip + +# Android x64 release gen_snapshot +ARCH_OUT=$ENGINE_OUT/android_release_x64 +ZIPS_OUT=$ARCH_OUT/zip_archives/android-x64-release +ZIPS_DEST=$INFRA_ROOT/android-x64-release +gsutil cp $ZIPS_OUT/$HOST_ARCH.zip $ZIPS_DEST/$HOST_ARCH.zip + +# Match the upload pattern from iOS: +# https://github.com/flutter/engine/commit/1d7f0c66c316a37105601b13136f890f6595aebc + +# iOS release Flutter artifacts +ARCH_OUT=$ENGINE_OUT/release +ZIPS_OUT=$ARCH_OUT +ZIPS_DEST=$INFRA_ROOT/ios-release +gsutil cp $ZIPS_OUT/artifacts.zip $ZIPS_DEST/artifacts.zip + +# iOS dSYM +gsutil cp $ZIPS_OUT/Flutter.framework.dSYM.zip $ZIPS_DEST/Flutter.framework.dSYM.zip + +# macOS framework +ARCH_OUT=$ENGINE_OUT/release +ZIPS_OUT=$ARCH_OUT/framework +ZIPS_DEST=$INFRA_ROOT/darwin-x64-release +gsutil cp $ZIPS_OUT/framework.zip $ZIPS_DEST/framework.zip + +# macOS gen_snapshot +ARCH_OUT=$ENGINE_OUT/release +ZIPS_OUT=$ARCH_OUT/snapshot +ZIPS_DEST=$INFRA_ROOT/darwin-x64-release +gsutil cp $ZIPS_OUT/gen_snapshot.zip $ZIPS_DEST/gen_snapshot.zip + +# FIXME: these should go where we're putting the arm64 macOS artifacts +# (darwin-x64-release), however, arm macs use darwin-x64-release and we +# currently only support those. We need to find a way to support both. +# macOS x64 release artifacts +# ARCH_OUT=$ENGINE_OUT/mac_release +# ZIPS_OUT=$ARCH_OUT/zip_archives/darwin-x64-release +# ZIPS_DEST=$INFRA_ROOT/darwin-x64-release +# gsutil cp $ZIPS_OUT/artifacts.zip $ZIPS_DEST/artifacts.zip + +# macOS arm64 release artifacts +ARCH_OUT=$ENGINE_OUT/mac_release_arm64 +ZIPS_OUT=$ARCH_OUT/zip_archives/darwin-arm64-release +# This looks wrong - why are we putting arm64 artifacts in darwin-x64-release +# instead of darwin-arm64-release? This is because arm macs use darwin-x64-release +# and we need to use the artifacts we've built for arm64 macs. +ZIPS_DEST=$INFRA_ROOT/darwin-x64-release +gsutil cp $ZIPS_OUT/artifacts.zip $ZIPS_DEST/artifacts.zip + +# macOS dSYM (used for symbolication, not by Flutter) +ARCH_OUT=$ENGINE_OUT/release +ZIPS_OUT=$ARCH_OUT/framework +ZIPS_DEST=$INFRA_ROOT/darwin-x64 +gsutil cp $ZIPS_OUT/FlutterMacOS.framework.dSYM.zip $ZIPS_DEST/FlutterMacOS.framework.dSYM.zip + +TMP_DIR=$(mktemp -d) + +PATCH_VERSION=0.2.1 +GH_RELEASE=https://github.com/shorebirdtech/updater/releases/download/patch-v$PATCH_VERSION/ +cd $TMP_DIR +curl -L $GH_RELEASE/patch-x86_64-apple-darwin.zip -o patch-x86_64-apple-darwin.zip +curl -L $GH_RELEASE/patch-x86_64-pc-windows-msvc.zip -o patch-x86_64-pc-windows-msvc.zip +curl -L $GH_RELEASE/patch-x86_64-unknown-linux-musl.zip -o patch-x86_64-unknown-linux-musl.zip + +gsutil cp patch-x86_64-apple-darwin.zip $SHOREBIRD_ROOT/patch-darwin-x64.zip +gsutil cp patch-x86_64-pc-windows-msvc.zip $SHOREBIRD_ROOT/patch-windows-x64.zip +gsutil cp patch-x86_64-unknown-linux-musl.zip $SHOREBIRD_ROOT/patch-linux-x64.zip + +gsutil cp $MANIFEST_FILE $SHOREBIRD_ROOT/artifacts_manifest.yaml diff --git a/shorebird/ci/internal/win_build.sh b/shorebird/ci/internal/win_build.sh new file mode 100755 index 0000000000000..2df02670aa78f --- /dev/null +++ b/shorebird/ci/internal/win_build.sh @@ -0,0 +1,63 @@ +#!/bin/bash -e + +# Usage: +# ./win_build.sh engine_path + +ENGINE_ROOT=$1 + +ENGINE_SRC=$ENGINE_ROOT/src +ENGINE_OUT=$ENGINE_SRC/out +UPDATER_SRC=$ENGINE_SRC/flutter/third_party/updater +HOST_ARCH='windows-x64' + +# Build the Rust library. +cd $UPDATER_SRC/library + +cargo build --release \ + --target x86_64-pc-windows-msvc + +# Compile the engine using the steps here: +# https://github.com/flutter/flutter/wiki/Compiling-the-engine#compiling-for-android-from-macos-or-linux +cd $ENGINE_SRC + +NINJA="ninja" +GN=./flutter/tools/gn +# We could probably use our own prebuilt dart SDK, by modifying the gn files. +GN_ARGS="--no-rbe --no-enable-unittests" + +# Windows only needs gen_snapshot for each Android CPU type. +# See https://github.com/flutter/engine/blob/e590b24f3962fda3ec9144dcee3f7565b195839a/ci/builders/windows_android_aot_engine.json + +TARGETS="archive_win_gen_snapshot" + +# Build host_release +$GN $GN_ARGS --runtime-mode=release --no-prebuilt-dart-sdk +$NINJA -C out/host_release dart_sdk +# We want to build flutter/tools/font_subset, but that doesn't work with +# --no-prebuilt-dart-sdk. +# https://github.com/flutter/flutter/issues/164531 + +# Build windows desktop targets +$GN $GN_ARGS --runtime-mode=release --no-prebuilt-dart-sdk +$NINJA -C ./out/host_release flutter/build/archives:windows_flutter gen_snapshot windows flutter/build/archives:artifacts + +# Build debug Windows artifacts +# These are output to the `windows-x64` directory in host_debug, and are used +# by `flutter build windows --release`. +$GN $GN_ARGS --no-prebuilt-dart-sdk +$NINJA -C ./out/host_debug flutter/build/archives:artifacts + +# If this gives you trouble, try using VS2019 instead. I had trouble with 2022. +# Android arm64 release +$GN $GN_ARGS --android --android-cpu=arm64 --runtime-mode=release +$NINJA -C ./out/android_release_arm64 $TARGETS + +# Android arm32 release +$GN $GN_ARGS --runtime-mode=release --android +$NINJA -C out/android_release $TARGETS + +# Android x64 release +$GN $GN_ARGS --android --android-cpu=x64 --runtime-mode=release +$NINJA -C ./out/android_release_x64 $TARGETS + +# We could also build the `patch` tool for Windows here. diff --git a/shorebird/ci/internal/win_setup.sh b/shorebird/ci/internal/win_setup.sh new file mode 100755 index 0000000000000..971135644f66b --- /dev/null +++ b/shorebird/ci/internal/win_setup.sh @@ -0,0 +1,8 @@ +#!/bin/bash -e + +# Usage: +# ./windows_setup.sh + +# Add the MSVC toolchain to Rust. +rustup target add \ + x86_64-pc-windows-msvc diff --git a/shorebird/ci/internal/win_upload.sh b/shorebird/ci/internal/win_upload.sh new file mode 100755 index 0000000000000..952457109e40c --- /dev/null +++ b/shorebird/ci/internal/win_upload.sh @@ -0,0 +1,90 @@ +#!/bin/bash -e + +# Usage: +# ./win_upload.sh engine_path git_hash +ENGINE_ROOT=$1 +ENGINE_HASH=$2 + +STORAGE_BUCKET="download.shorebird.dev" +SHOREBIRD_ROOT=gs://$STORAGE_BUCKET/shorebird/$ENGINE_HASH + +ENGINE_SRC=$ENGINE_ROOT/src +ENGINE_OUT=$ENGINE_SRC/out +ENGINE_FLUTTER=$ENGINE_SRC/flutter +# FLUTTER_ROOT is the Flutter monorepo root (parent of engine/) +FLUTTER_ROOT=$(dirname $ENGINE_ROOT) + +cd $FLUTTER_ROOT + +# Compute the content-aware hash for the Dart SDK. +# This allows Flutter checkouts that haven't changed engine content to share +# the same pre-built Dart SDK, even if they have different git commit SHAs. +CONTENT_HASH=$($FLUTTER_ROOT/bin/internal/content_aware_hash.sh) + +# We do not generate a manifest file, we assume another builder did that. + +# TODO(eseidel): This should not be in shell, it's too complicated/repetitive. + +HOST_ARCH='windows-x64' + +INFRA_ROOT="gs://$STORAGE_BUCKET/flutter_infra_release/flutter/$ENGINE_HASH" + +# Dart SDK +# This gets uploaded to flutter_infra_release/flutter/\$engine/dart-sdk-$HOST_ARCH.zip +# We also upload to the content-aware hash path to support local development branches. +CONTENT_INFRA_ROOT="gs://$STORAGE_BUCKET/flutter_infra_release/flutter/$CONTENT_HASH" + +DART_SDK_DIR=$ENGINE_OUT/host_release/dart-sdk +DART_ZIP_FILE=dart-sdk-$HOST_ARCH.zip + +# Use 7zip to compress the Dart SDK, as zip isn't available on Windows and +# Powershell, which we would normally use in the form of +# `powershell Compress-Archive dart-sdk dart-sdk.zip`, doesn't play nicely +# with git bash paths (e.g. /c/Users/... instead of C:/Users/...) +/c/Program\ Files/7-Zip/7z a $DART_ZIP_FILE $DART_SDK_DIR +ZIPS_DEST=$INFRA_ROOT/$DART_ZIP_FILE +gsutil cp $DART_ZIP_FILE $ZIPS_DEST +# Also upload to content-aware hash path +gsutil cp $DART_ZIP_FILE $CONTENT_INFRA_ROOT/$DART_ZIP_FILE + +# Android Arm64 release gen_snapshot +ARCH_OUT=$ENGINE_OUT/android_release_arm64 +ZIPS_OUT=$ARCH_OUT/zip_archives/android-arm64-release +ZIPS_DEST=$INFRA_ROOT/android-arm64-release +gsutil cp $ZIPS_OUT/$HOST_ARCH.zip $ZIPS_DEST/$HOST_ARCH.zip + +# Android Arm32 release gen_snapshot +ARCH_OUT=$ENGINE_OUT/android_release +ZIPS_OUT=$ARCH_OUT/zip_archives/android-arm-release +ZIPS_DEST=$INFRA_ROOT/android-arm-release +gsutil cp $ZIPS_OUT/$HOST_ARCH.zip $ZIPS_DEST/$HOST_ARCH.zip + +# Android x64 release gen_snapshot +ARCH_OUT=$ENGINE_OUT/android_release_x64 +ZIPS_OUT=$ARCH_OUT/zip_archives/android-x64-release +ZIPS_DEST=$INFRA_ROOT/android-x64-release +gsutil cp $ZIPS_OUT/$HOST_ARCH.zip $ZIPS_DEST/$HOST_ARCH.zip + +# We could upload patch if we built it here. +# gsutil cp $ENGINE_OUT/host_release/patch.zip $SHOREBIRD_ROOT/patch-win-x64.zip + +# Engine release artifacts +ARCH_OUT=$ENGINE_OUT/host_release +ZIPS_OUT=$ARCH_OUT/zip_archives/$HOST_ARCH-release +ZIPS_DEST=$INFRA_ROOT/$HOST_ARCH-release +gsutil cp $ZIPS_OUT/$HOST_ARCH-flutter.zip $ZIPS_DEST/$HOST_ARCH-flutter.zip + +# We want to build flutter/tools/font_subset, but that doesn't work with +# --no-prebuilt-dart-sdk. +# https://github.com/flutter/flutter/issues/164531 +# # Windows x64 host_release font_subset (ConstFinder) +# ARCH_OUT=$ENGINE_OUT/host_release +# ZIPS_OUT=$ARCH_OUT/zip_archives/$HOST_ARCH +# ZIPS_DEST=$INFRA_ROOT/windows-x64-release +# gsutil cp $ZIPS_OUT/font-subset.zip $ZIPS_DEST/font-subset.zip + +# Engine debug artifacts (not sure why this is needed?) +ARCH_OUT=$ENGINE_OUT/host_debug +ZIPS_OUT=$ARCH_OUT/zip_archives/$HOST_ARCH +ZIPS_DEST=$INFRA_ROOT/$HOST_ARCH +gsutil cp $ZIPS_OUT/artifacts.zip $ZIPS_DEST/artifacts.zip diff --git a/shorebird/ci/linux_build_and_upload.sh b/shorebird/ci/linux_build_and_upload.sh new file mode 100755 index 0000000000000..9657e14178695 --- /dev/null +++ b/shorebird/ci/linux_build_and_upload.sh @@ -0,0 +1,32 @@ +#!/bin/bash -e + +# Usage: +# ./linux_build_and_upload.sh flutter_root engine_hash +# +# This is the main entrypoint for building and uploading Linux engine artifacts. +# It is called from the _build_engine repository's CI scripts. + +if [ "$#" -ne 2 ]; then + echo "Usage: $0 flutter_root engine_hash" + exit 1 +fi + +FLUTTER_ROOT=$1 +ENGINE_HASH=$2 +ENGINE_ROOT=$FLUTTER_ROOT/engine + +# Get the absolute path to the directory of this script. +SCRIPT_DIR=$(cd $(dirname $0) && pwd) + +echo "Building engine at $ENGINE_ROOT and uploading to gs://download.shorebird.dev" + +cd $SCRIPT_DIR + +# Run the setup script. +./internal/linux_setup.sh + +# Then run the build. +./internal/linux_build.sh $ENGINE_ROOT + +# Copy Shorebird engine artifacts to Google Cloud Storage. +./internal/linux_upload.sh $ENGINE_ROOT $ENGINE_HASH diff --git a/shorebird/ci/mac_build_and_upload.sh b/shorebird/ci/mac_build_and_upload.sh new file mode 100755 index 0000000000000..9ee302620d207 --- /dev/null +++ b/shorebird/ci/mac_build_and_upload.sh @@ -0,0 +1,32 @@ +#!/bin/bash -e + +# Usage: +# ./mac_build_and_upload.sh flutter_root engine_hash +# +# This is the main entrypoint for building and uploading macOS engine artifacts. +# It is called from the _build_engine repository's CI scripts. + +if [ "$#" -ne 2 ]; then + echo "Usage: $0 flutter_root engine_hash" + exit 1 +fi + +FLUTTER_ROOT=$1 +ENGINE_HASH=$2 +ENGINE_ROOT=$FLUTTER_ROOT/engine + +# Get the absolute path to the directory of this script. +SCRIPT_DIR=$(cd $(dirname $0) && pwd) + +echo "Building engine at $ENGINE_ROOT and uploading to gs://download.shorebird.dev" + +cd $SCRIPT_DIR + +# Run the setup script. +./internal/mac_setup.sh + +# Then run the build. +./internal/mac_build.sh $ENGINE_ROOT + +# Copy Shorebird engine artifacts to Google Cloud Storage. +./internal/mac_upload.sh $ENGINE_ROOT $ENGINE_HASH diff --git a/shorebird/ci/win_build_and_upload.sh b/shorebird/ci/win_build_and_upload.sh new file mode 100755 index 0000000000000..bbda403ff2c67 --- /dev/null +++ b/shorebird/ci/win_build_and_upload.sh @@ -0,0 +1,32 @@ +#!/bin/bash -e + +# Usage: +# ./win_build_and_upload.sh flutter_root engine_hash +# +# This is the main entrypoint for building and uploading Windows engine artifacts. +# It is called from the _build_engine repository's CI scripts. + +if [ "$#" -ne 2 ]; then + echo "Usage: $0 flutter_root engine_hash" + exit 1 +fi + +FLUTTER_ROOT=$1 +ENGINE_HASH=$2 +ENGINE_ROOT=$FLUTTER_ROOT/engine + +# Get the absolute path to the directory of this script. +SCRIPT_DIR=$(cd $(dirname $0) && pwd) + +echo "Building engine at $ENGINE_ROOT and uploading to gs://download.shorebird.dev" + +cd $SCRIPT_DIR + +# Run the setup script. +./internal/win_setup.sh + +# Then run the build. +./internal/win_build.sh $ENGINE_ROOT + +# Copy Shorebird engine artifacts to Google Cloud Storage. +./internal/win_upload.sh $ENGINE_ROOT $ENGINE_HASH From 6472b9e5e0b91f0680547bed3e8fc90116f211b1 Mon Sep 17 00:00:00 2001 From: Eric Seidel Date: Mon, 15 Dec 2025 16:55:22 -0800 Subject: [PATCH 11/95] chore: update to use new CreateGroupIsolate API (#99) * chore: roll dart to 6a78a2deaee05bc74775fcfa2ff27aa53c96efca * wip * chore: run et format * chore: attempt to clean up shorebird.cc * chore: fix build * chore: remove FLUTTER_STORAGE_BASE_URL override --- .github/workflows/shorebird_ci.yml | 5 -- engine/src/flutter/runtime/dart_isolate.cc | 31 +++++++++- .../runtime/dart_isolate_group_data.cc | 8 ++- .../flutter/runtime/dart_isolate_group_data.h | 8 ++- .../shell/common/shorebird/shorebird.cc | 61 +++++++++++++------ .../shell/common/shorebird/shorebird.h | 19 ++++++ 6 files changed, 105 insertions(+), 27 deletions(-) diff --git a/.github/workflows/shorebird_ci.yml b/.github/workflows/shorebird_ci.yml index 7b712b1583b87..9597d6a599344 100644 --- a/.github/workflows/shorebird_ci.yml +++ b/.github/workflows/shorebird_ci.yml @@ -21,11 +21,6 @@ jobs: name: ๐Ÿฆ Shorebird Test - # TODO(eseidel): This is also set inside shorebird_tests, unclear if - # if it's needed here as well. - env: - FLUTTER_STORAGE_BASE_URL: https://download.shorebird.dev - steps: - name: ๐Ÿ“š Git Checkout uses: actions/checkout@v4 diff --git a/engine/src/flutter/runtime/dart_isolate.cc b/engine/src/flutter/runtime/dart_isolate.cc index 8ea56ae2408d9..4e5b74ca28ac8 100644 --- a/engine/src/flutter/runtime/dart_isolate.cc +++ b/engine/src/flutter/runtime/dart_isolate.cc @@ -21,6 +21,10 @@ #include "flutter/runtime/isolate_configuration.h" #include "flutter/runtime/platform_isolate_manager.h" #include "fml/message_loop_task_queues.h" + +#if SHOREBIRD_USE_INTERPRETER +#include "flutter/shell/common/shorebird/shorebird.h" // nogncheck +#endif #include "fml/task_source.h" #include "fml/time/time_point.h" #include "third_party/dart/runtime/include/bin/native_assets_api.h" @@ -245,6 +249,12 @@ std::weak_ptr DartIsolate::CreateRootIsolate( } else { // The child isolate preparer is null but will be set when the isolate is // being prepared to run. +#if SHOREBIRD_USE_INTERPRETER + // Get the base snapshot for Shorebird linking support (may be null). + fml::RefPtr base_snapshot = GetBaseIsolateSnapshot(); +#else + fml::RefPtr base_snapshot = nullptr; +#endif isolate_group_data = std::make_unique>( std::shared_ptr(new DartIsolateGroupData( @@ -254,13 +264,30 @@ std::weak_ptr DartIsolate::CreateRootIsolate( context.advisory_script_entrypoint, // advisory entrypoint nullptr, // child isolate preparer isolate_create_callback, // isolate create callback - isolate_shutdown_callback, // isolate shutdown callback - std::move(native_assets_manager) // + isolate_shutdown_callback, // isolate shutdown callback + std::move(native_assets_manager), // native assets manager + std::move(base_snapshot) // base snapshot (Shorebird) ))); isolate_maker = [](std::shared_ptr* isolate_group_data, std::shared_ptr* isolate_data, Dart_IsolateFlags* flags, char** error) { +#if SHOREBIRD_USE_INTERPRETER + auto base_snapshot = (*isolate_group_data)->GetBaseSnapshot(); + if (base_snapshot) { + // Use the Shorebird API that accepts base snapshot for linking. + return Dart_CreateIsolateGroupWithBaseSnapshot( + (*isolate_group_data)->GetAdvisoryScriptURI().c_str(), + (*isolate_group_data)->GetAdvisoryScriptEntrypoint().c_str(), + (*isolate_group_data)->GetIsolateSnapshot()->GetDataMapping(), + (*isolate_group_data) + ->GetIsolateSnapshot() + ->GetInstructionsMapping(), + base_snapshot->GetDataMapping(), + base_snapshot->GetInstructionsMapping(), flags, isolate_group_data, + isolate_data, error); + } +#endif return Dart_CreateIsolateGroup( (*isolate_group_data)->GetAdvisoryScriptURI().c_str(), (*isolate_group_data)->GetAdvisoryScriptEntrypoint().c_str(), diff --git a/engine/src/flutter/runtime/dart_isolate_group_data.cc b/engine/src/flutter/runtime/dart_isolate_group_data.cc index 0e844d3e3c417..a8ecffee5b1ea 100644 --- a/engine/src/flutter/runtime/dart_isolate_group_data.cc +++ b/engine/src/flutter/runtime/dart_isolate_group_data.cc @@ -18,9 +18,11 @@ DartIsolateGroupData::DartIsolateGroupData( const ChildIsolatePreparer& child_isolate_preparer, const fml::closure& isolate_create_callback, const fml::closure& isolate_shutdown_callback, - std::shared_ptr native_assets_manager) + std::shared_ptr native_assets_manager, + fml::RefPtr base_snapshot) : settings_(settings), isolate_snapshot_(std::move(isolate_snapshot)), + base_snapshot_(std::move(base_snapshot)), advisory_script_uri_(std::move(advisory_script_uri)), advisory_script_entrypoint_(std::move(advisory_script_entrypoint)), child_isolate_preparer_(child_isolate_preparer), @@ -41,6 +43,10 @@ fml::RefPtr DartIsolateGroupData::GetIsolateSnapshot() return isolate_snapshot_; } +fml::RefPtr DartIsolateGroupData::GetBaseSnapshot() const { + return base_snapshot_; +} + const std::string& DartIsolateGroupData::GetAdvisoryScriptURI() const { return advisory_script_uri_; } diff --git a/engine/src/flutter/runtime/dart_isolate_group_data.h b/engine/src/flutter/runtime/dart_isolate_group_data.h index 8ff29272150ab..1502b2fc8e290 100644 --- a/engine/src/flutter/runtime/dart_isolate_group_data.h +++ b/engine/src/flutter/runtime/dart_isolate_group_data.h @@ -39,7 +39,8 @@ class DartIsolateGroupData : public PlatformMessageHandlerStorage { const ChildIsolatePreparer& child_isolate_preparer, const fml::closure& isolate_create_callback, const fml::closure& isolate_shutdown_callback, - std::shared_ptr native_assets_manager = nullptr); + std::shared_ptr native_assets_manager = nullptr, + fml::RefPtr base_snapshot = nullptr); ~DartIsolateGroupData(); @@ -47,6 +48,10 @@ class DartIsolateGroupData : public PlatformMessageHandlerStorage { fml::RefPtr GetIsolateSnapshot() const; + /// Returns the base snapshot for Shorebird linking support. + /// May be null if not using Shorebird or if no patch is active. + fml::RefPtr GetBaseSnapshot() const; + const std::string& GetAdvisoryScriptURI() const; const std::string& GetAdvisoryScriptEntrypoint() const; @@ -81,6 +86,7 @@ class DartIsolateGroupData : public PlatformMessageHandlerStorage { std::vector> kernel_buffers_; const Settings settings_; const fml::RefPtr isolate_snapshot_; + const fml::RefPtr base_snapshot_; const std::string advisory_script_uri_; const std::string advisory_script_entrypoint_; mutable std::mutex child_isolate_preparer_mutex_; diff --git a/engine/src/flutter/shell/common/shorebird/shorebird.cc b/engine/src/flutter/shell/common/shorebird/shorebird.cc index ec2bc6bf3c9cd..8cb32cfc9916b 100644 --- a/engine/src/flutter/shell/common/shorebird/shorebird.cc +++ b/engine/src/flutter/shell/common/shorebird/shorebird.cc @@ -45,24 +45,38 @@ extern "C" __attribute__((weak)) unsigned long getauxval(unsigned long type) { } #endif -// TODO(eseidel): I believe we need to leak these or we'll sometimes crash -// when using the base snapshot in mixed mode. This likely will not play -// nicely with multi-engine support and will need to be refactored. +#if SHOREBIRD_USE_INTERPRETER + +// Global references to the base (unpatched) snapshots from the App.framework. +// These are process-global because: +// 1. The Shorebird updater library is a process-global singleton with its own +// internal state. FileCallbacksImpl provides it access to the base snapshot +// data for patch generation/validation. +// 2. The base snapshots are immutable (baked into the IPA) so sharing them +// across isolate groups is safe. +// 3. GetBaseIsolateSnapshot() returns the isolate snapshot for use when +// creating isolate groups with Dart_CreateIsolateGroupWithBaseSnapshot(), +// which needs the base snapshot for linking patched code. +// +// Note: This design doesn't support multiple engines with different base +// snapshots, but I'm not aware of any use cases for that on iOS. static fml::RefPtr vm_snapshot; static fml::RefPtr isolate_snapshot; -void SetBaseSnapshot(Settings& settings) { - // These mappings happen to be to static data in the App.framework, but - // we still need to seem to hold onto the DartSnapshot objects to keep - // the mappings alive. +void StoreBaseSnapshots(Settings& settings) { + // Create DartSnapshot objects that hold references to the symbol mappings + // in the App.framework. The snapshots are static data in the framework, + // but we need DartSnapshot objects to keep the NativeLibrary refs alive. vm_snapshot = DartSnapshot::VMSnapshotFromSettings(settings); isolate_snapshot = DartSnapshot::IsolateSnapshotFromSettings(settings); - Shorebird_SetBaseSnapshots(isolate_snapshot->GetDataMapping(), - isolate_snapshot->GetInstructionsMapping(), - vm_snapshot->GetDataMapping(), - vm_snapshot->GetInstructionsMapping()); } +fml::RefPtr GetBaseIsolateSnapshot() { + return isolate_snapshot; +} + +#endif // SHOREBIRD_USE_INTERPRETER + class FileCallbacksImpl { public: static void* Open(); @@ -104,7 +118,9 @@ std::string GetValueFromYaml(const std::string& yaml, const std::string& key) { return ""; } -// FIXME: consolidate this with the other ConfigureShorebird +/// Newer api, used by Desktop implementations. +/// Does not directly manipulate Settings. +// FIXME: Consolidate this with the other ConfigureShorebird() API. bool ConfigureShorebird(const ShorebirdConfigArgs& args, std::string& patch_path) { patch_path = fml::PathToUtf8(args.release_app_library_path); @@ -201,6 +217,8 @@ bool ConfigureShorebird(const ShorebirdConfigArgs& args, return true; } +/// Older api used by iOS and Android, directly manipulates Settings. +// FIXME: Consolidate this with the other ConfigureShorebird() API. void ConfigureShorebird(std::string code_cache_path, std::string app_storage_path, Settings& settings, @@ -251,15 +269,14 @@ void ConfigureShorebird(std::string code_cache_path, shorebird_yaml.c_str()); } - // We've decided not to support synchronous updates on launch for now. - // It's a terrible user experience (having the app hang on launch) and - // instead we will provide examples of how to build a custom update UI - // within Dart, including updating as part of login, etc. + // We do not support synchronous updates on launch, it's a terrible UX. + // Users can implement custom check-for-updates using + // package:shorebird_code_push. // https://github.com/shorebirdtech/shorebird/issues/950 - // We only set the base snapshot on iOS for now. + // We store the base snapshot on iOS for use when creating the isolate group. #if SHOREBIRD_USE_INTERPRETER - SetBaseSnapshot(settings); + StoreBaseSnapshots(settings); #endif shorebird_validate_next_boot_patch(); @@ -312,9 +329,17 @@ void ConfigureShorebird(std::string code_cache_path, } void* FileCallbacksImpl::Open() { +#if SHOREBIRD_USE_INTERPRETER return SnapshotsDataHandle::createForSnapshots(*vm_snapshot, *isolate_snapshot) .release(); +#else + // SnapshotsDataHandle exists on all platforms (for testing)but is only used + // on iOS. iOS patches are generated from just the Dart parts of the snapshot, + // excluding the Mach-O specific headers which contain dates and paths that + // make them change on every build. + return nullptr; +#endif // SHOREBIRD_USE_INTERPRETER } uintptr_t FileCallbacksImpl::Read(void* file, diff --git a/engine/src/flutter/shell/common/shorebird/shorebird.h b/engine/src/flutter/shell/common/shorebird/shorebird.h index c6873c5014a00..50c6f59e1fa60 100644 --- a/engine/src/flutter/shell/common/shorebird/shorebird.h +++ b/engine/src/flutter/shell/common/shorebird/shorebird.h @@ -2,15 +2,22 @@ #define FLUTTER_SHELL_COMMON_SHOREBIRD_SHOREBIRD_H_ #include "flutter/common/settings.h" +#include "flutter/fml/memory/ref_ptr.h" #include "shell/platform/embedder/embedder.h" namespace flutter { +class DartSnapshot; + +/// Version and build number of the release. +/// Used by ShorebirdConfigArgs. struct ReleaseVersion { std::string version; std::string build_number; }; +/// Arguments for ConfigureShorebird. +/// Used by Desktop implementations. struct ShorebirdConfigArgs { std::string code_cache_path; std::string app_storage_path; @@ -30,9 +37,12 @@ struct ShorebirdConfigArgs { release_version(release_version) {} }; +/// Newer api, used by Desktop implementations. +/// Does not directly manipulate Settings. bool ConfigureShorebird(const ShorebirdConfigArgs& args, std::string& patch_path); +/// Older api used by iOS and Android, directly manipulates Settings. void ConfigureShorebird(std::string code_cache_path, std::string app_storage_path, Settings& settings, @@ -40,8 +50,17 @@ void ConfigureShorebird(std::string code_cache_path, const std::string& version, const std::string& version_code); +/// Used for reading app_id from shorebird.yaml. +/// Exposed for testing. std::string GetValueFromYaml(const std::string& yaml, const std::string& key); +#if SHOREBIRD_USE_INTERPRETER +/// Returns the base isolate snapshot for Shorebird linking support. +/// Must be called after ConfigureShorebird() has stored the base snapshots. +/// May return null if not using Shorebird interpreter mode. +fml::RefPtr GetBaseIsolateSnapshot(); +#endif // SHOREBIRD_USE_INTERPRETER + } // namespace flutter #endif // FLUTTER_SHELL_COMMON_SHOREBIRD_SHOREBIRD_H_ From b8251d11995294a6984e7a954203f072d7c68441 Mon Sep 17 00:00:00 2001 From: Bryan Oltman Date: Fri, 19 Dec 2025 13:18:34 -0500 Subject: [PATCH 12/95] bump updater rev --- DEPS | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/DEPS b/DEPS index 26bfea672a03e..16a59de7e7f01 100644 --- a/DEPS +++ b/DEPS @@ -20,7 +20,7 @@ vars = { "dart_sdk_revision": "b65ce89c8057d6880e00693a7b0ecd7b9e5f61ca", "dart_sdk_git": "git@github.com:shorebirdtech/dart-sdk.git", "updater_git": "https://github.com/shorebirdtech/updater.git", - "updater_rev": "76f005940db57c38b479cee858abc0cfbd12ac28", + "updater_rev": "9db198a6344f7b9ec2843e06a459938a520e314f", # WARNING: DO NOT EDIT canvaskit_cipd_instance MANUALLY # See `lib/web_ui/README.md` for how to roll CanvasKit to a new version. From be7c0f736f492aaead927f845830daa33ba87906 Mon Sep 17 00:00:00 2001 From: Eric Seidel Date: Tue, 30 Dec 2025 16:10:40 -0800 Subject: [PATCH 13/95] fix: crash after patching on iOS --- engine/src/flutter/runtime/dart_isolate.cc | 20 +++++++++- shorebird/docs/BUILDING.md | 44 ++++++++++++++++++++++ 2 files changed, 63 insertions(+), 1 deletion(-) create mode 100644 shorebird/docs/BUILDING.md diff --git a/engine/src/flutter/runtime/dart_isolate.cc b/engine/src/flutter/runtime/dart_isolate.cc index 4e5b74ca28ac8..6b0c8558a9753 100644 --- a/engine/src/flutter/runtime/dart_isolate.cc +++ b/engine/src/flutter/runtime/dart_isolate.cc @@ -1136,7 +1136,9 @@ Dart_Isolate DartIsolate::DartIsolateGroupCreateCallback( advisory_script_entrypoint, parent_group_data.GetChildIsolatePreparer(), parent_group_data.GetIsolateCreateCallback(), - parent_group_data.GetIsolateShutdownCallback()))); + parent_group_data.GetIsolateShutdownCallback(), + nullptr, // native_assets_manager + parent_group_data.GetBaseSnapshot()))); TaskRunners null_task_runners(advisory_script_uri, /* platform= */ nullptr, @@ -1158,6 +1160,22 @@ Dart_Isolate DartIsolate::DartIsolateGroupCreateCallback( [](std::shared_ptr* isolate_group_data, std::shared_ptr* isolate_data, Dart_IsolateFlags* flags, char** error) { +#if SHOREBIRD_USE_INTERPRETER + auto base_snapshot = (*isolate_group_data)->GetBaseSnapshot(); + if (base_snapshot) { + // Use the Shorebird API that accepts base snapshot for linking. + return Dart_CreateIsolateGroupWithBaseSnapshot( + (*isolate_group_data)->GetAdvisoryScriptURI().c_str(), + (*isolate_group_data)->GetAdvisoryScriptEntrypoint().c_str(), + (*isolate_group_data)->GetIsolateSnapshot()->GetDataMapping(), + (*isolate_group_data) + ->GetIsolateSnapshot() + ->GetInstructionsMapping(), + base_snapshot->GetDataMapping(), + base_snapshot->GetInstructionsMapping(), flags, + isolate_group_data, isolate_data, error); + } +#endif return Dart_CreateIsolateGroup( (*isolate_group_data)->GetAdvisoryScriptURI().c_str(), (*isolate_group_data)->GetAdvisoryScriptEntrypoint().c_str(), diff --git a/shorebird/docs/BUILDING.md b/shorebird/docs/BUILDING.md new file mode 100644 index 0000000000000..dfd3c655a1324 --- /dev/null +++ b/shorebird/docs/BUILDING.md @@ -0,0 +1,44 @@ +# Building the Shorebird Engine Locally + +This document explains how to build the Shorebird engine locally for different platforms. + +All commands assume you are running from the `engine/src` directory. + +## Prerequisites + +- Follow the standard Flutter engine setup instructions +- Ensure you have the necessary toolchains installed for your target platform + +## macOS + +### iOS (arm64) + +```bash +./flutter/tools/gn --no-rbe --no-enable-unittests --runtime-mode=release --ios --gn-arg='shorebird_runtime=true' +ninja -C out/ios_release flutter/shell/platform/darwin/ios:flutter_framework flutter/lib/snapshot:generate_snapshot_bins +``` + +### Android arm64 + +```bash +./flutter/tools/gn --no-rbe --no-enable-unittests --android --android-cpu=arm64 --runtime-mode=release --gn-args='host_cpu="x64"' +ninja -C out/android_release_arm64 flutter/shell/platform/android:gen_snapshot +``` + +## Windows + +### Android arm64 + +```bash +./flutter/tools/gn --no-rbe --no-enable-unittests --android --android-cpu=arm64 --runtime-mode=release +ninja -C out/android_release_arm64 archive_win_gen_snapshot +``` + +## Linux + +### Android arm64 + +```bash +./flutter/tools/gn --no-rbe --no-enable-unittests --android --android-cpu=arm64 --runtime-mode=release +ninja -C out/android_release_arm64 default gen_snapshot +``` From cf0ea2bd17a76df887a26679d3890d914c6d3297 Mon Sep 17 00:00:00 2001 From: Eric Seidel Date: Mon, 5 Jan 2026 16:53:37 -0800 Subject: [PATCH 14/95] fix: revert to using Shorebird_SetBaseSnapshots to stop iOS crashers --- engine/src/flutter/runtime/dart_isolate.cc | 51 ++----------------- .../runtime/dart_isolate_group_data.cc | 8 +-- .../flutter/runtime/dart_isolate_group_data.h | 8 +-- .../flutter/runtime/shorebird/patch_cache.cc | 7 ++- .../shell/common/shorebird/shorebird.cc | 38 ++++++-------- .../shell/common/shorebird/shorebird.h | 7 --- 6 files changed, 25 insertions(+), 94 deletions(-) diff --git a/engine/src/flutter/runtime/dart_isolate.cc b/engine/src/flutter/runtime/dart_isolate.cc index 6b0c8558a9753..782cc0e4a775c 100644 --- a/engine/src/flutter/runtime/dart_isolate.cc +++ b/engine/src/flutter/runtime/dart_isolate.cc @@ -21,10 +21,6 @@ #include "flutter/runtime/isolate_configuration.h" #include "flutter/runtime/platform_isolate_manager.h" #include "fml/message_loop_task_queues.h" - -#if SHOREBIRD_USE_INTERPRETER -#include "flutter/shell/common/shorebird/shorebird.h" // nogncheck -#endif #include "fml/task_source.h" #include "fml/time/time_point.h" #include "third_party/dart/runtime/include/bin/native_assets_api.h" @@ -249,12 +245,6 @@ std::weak_ptr DartIsolate::CreateRootIsolate( } else { // The child isolate preparer is null but will be set when the isolate is // being prepared to run. -#if SHOREBIRD_USE_INTERPRETER - // Get the base snapshot for Shorebird linking support (may be null). - fml::RefPtr base_snapshot = GetBaseIsolateSnapshot(); -#else - fml::RefPtr base_snapshot = nullptr; -#endif isolate_group_data = std::make_unique>( std::shared_ptr(new DartIsolateGroupData( @@ -264,30 +254,13 @@ std::weak_ptr DartIsolate::CreateRootIsolate( context.advisory_script_entrypoint, // advisory entrypoint nullptr, // child isolate preparer isolate_create_callback, // isolate create callback - isolate_shutdown_callback, // isolate shutdown callback - std::move(native_assets_manager), // native assets manager - std::move(base_snapshot) // base snapshot (Shorebird) + isolate_shutdown_callback, // isolate shutdown callback + std::move(native_assets_manager) // ))); isolate_maker = [](std::shared_ptr* isolate_group_data, std::shared_ptr* isolate_data, Dart_IsolateFlags* flags, char** error) { -#if SHOREBIRD_USE_INTERPRETER - auto base_snapshot = (*isolate_group_data)->GetBaseSnapshot(); - if (base_snapshot) { - // Use the Shorebird API that accepts base snapshot for linking. - return Dart_CreateIsolateGroupWithBaseSnapshot( - (*isolate_group_data)->GetAdvisoryScriptURI().c_str(), - (*isolate_group_data)->GetAdvisoryScriptEntrypoint().c_str(), - (*isolate_group_data)->GetIsolateSnapshot()->GetDataMapping(), - (*isolate_group_data) - ->GetIsolateSnapshot() - ->GetInstructionsMapping(), - base_snapshot->GetDataMapping(), - base_snapshot->GetInstructionsMapping(), flags, isolate_group_data, - isolate_data, error); - } -#endif return Dart_CreateIsolateGroup( (*isolate_group_data)->GetAdvisoryScriptURI().c_str(), (*isolate_group_data)->GetAdvisoryScriptEntrypoint().c_str(), @@ -1137,8 +1110,8 @@ Dart_Isolate DartIsolate::DartIsolateGroupCreateCallback( parent_group_data.GetChildIsolatePreparer(), parent_group_data.GetIsolateCreateCallback(), parent_group_data.GetIsolateShutdownCallback(), - nullptr, // native_assets_manager - parent_group_data.GetBaseSnapshot()))); + nullptr // native_assets_manager + ))); TaskRunners null_task_runners(advisory_script_uri, /* platform= */ nullptr, @@ -1160,22 +1133,6 @@ Dart_Isolate DartIsolate::DartIsolateGroupCreateCallback( [](std::shared_ptr* isolate_group_data, std::shared_ptr* isolate_data, Dart_IsolateFlags* flags, char** error) { -#if SHOREBIRD_USE_INTERPRETER - auto base_snapshot = (*isolate_group_data)->GetBaseSnapshot(); - if (base_snapshot) { - // Use the Shorebird API that accepts base snapshot for linking. - return Dart_CreateIsolateGroupWithBaseSnapshot( - (*isolate_group_data)->GetAdvisoryScriptURI().c_str(), - (*isolate_group_data)->GetAdvisoryScriptEntrypoint().c_str(), - (*isolate_group_data)->GetIsolateSnapshot()->GetDataMapping(), - (*isolate_group_data) - ->GetIsolateSnapshot() - ->GetInstructionsMapping(), - base_snapshot->GetDataMapping(), - base_snapshot->GetInstructionsMapping(), flags, - isolate_group_data, isolate_data, error); - } -#endif return Dart_CreateIsolateGroup( (*isolate_group_data)->GetAdvisoryScriptURI().c_str(), (*isolate_group_data)->GetAdvisoryScriptEntrypoint().c_str(), diff --git a/engine/src/flutter/runtime/dart_isolate_group_data.cc b/engine/src/flutter/runtime/dart_isolate_group_data.cc index a8ecffee5b1ea..0e844d3e3c417 100644 --- a/engine/src/flutter/runtime/dart_isolate_group_data.cc +++ b/engine/src/flutter/runtime/dart_isolate_group_data.cc @@ -18,11 +18,9 @@ DartIsolateGroupData::DartIsolateGroupData( const ChildIsolatePreparer& child_isolate_preparer, const fml::closure& isolate_create_callback, const fml::closure& isolate_shutdown_callback, - std::shared_ptr native_assets_manager, - fml::RefPtr base_snapshot) + std::shared_ptr native_assets_manager) : settings_(settings), isolate_snapshot_(std::move(isolate_snapshot)), - base_snapshot_(std::move(base_snapshot)), advisory_script_uri_(std::move(advisory_script_uri)), advisory_script_entrypoint_(std::move(advisory_script_entrypoint)), child_isolate_preparer_(child_isolate_preparer), @@ -43,10 +41,6 @@ fml::RefPtr DartIsolateGroupData::GetIsolateSnapshot() return isolate_snapshot_; } -fml::RefPtr DartIsolateGroupData::GetBaseSnapshot() const { - return base_snapshot_; -} - const std::string& DartIsolateGroupData::GetAdvisoryScriptURI() const { return advisory_script_uri_; } diff --git a/engine/src/flutter/runtime/dart_isolate_group_data.h b/engine/src/flutter/runtime/dart_isolate_group_data.h index 1502b2fc8e290..8ff29272150ab 100644 --- a/engine/src/flutter/runtime/dart_isolate_group_data.h +++ b/engine/src/flutter/runtime/dart_isolate_group_data.h @@ -39,8 +39,7 @@ class DartIsolateGroupData : public PlatformMessageHandlerStorage { const ChildIsolatePreparer& child_isolate_preparer, const fml::closure& isolate_create_callback, const fml::closure& isolate_shutdown_callback, - std::shared_ptr native_assets_manager = nullptr, - fml::RefPtr base_snapshot = nullptr); + std::shared_ptr native_assets_manager = nullptr); ~DartIsolateGroupData(); @@ -48,10 +47,6 @@ class DartIsolateGroupData : public PlatformMessageHandlerStorage { fml::RefPtr GetIsolateSnapshot() const; - /// Returns the base snapshot for Shorebird linking support. - /// May be null if not using Shorebird or if no patch is active. - fml::RefPtr GetBaseSnapshot() const; - const std::string& GetAdvisoryScriptURI() const; const std::string& GetAdvisoryScriptEntrypoint() const; @@ -86,7 +81,6 @@ class DartIsolateGroupData : public PlatformMessageHandlerStorage { std::vector> kernel_buffers_; const Settings settings_; const fml::RefPtr isolate_snapshot_; - const fml::RefPtr base_snapshot_; const std::string advisory_script_uri_; const std::string advisory_script_entrypoint_; mutable std::mutex child_isolate_preparer_mutex_; diff --git a/engine/src/flutter/runtime/shorebird/patch_cache.cc b/engine/src/flutter/runtime/shorebird/patch_cache.cc index 30c99aea3a011..d3dbe617ebfea 100644 --- a/engine/src/flutter/runtime/shorebird/patch_cache.cc +++ b/engine/src/flutter/runtime/shorebird/patch_cache.cc @@ -45,10 +45,9 @@ std::shared_ptr PatchCacheEntry::Create( const uint8_t* isolate_data = nullptr; const uint8_t* isolate_instrs = nullptr; - Dart_LoadedElf* elf = - Dart_LoadELF(path.c_str(), elf_file_offset, &error, &ignored_vm_data, - &ignored_vm_instrs, &isolate_data, &isolate_instrs, - /* load as read-only, not rx */ false); + Dart_LoadedElf* elf = Dart_LoadELF( + path.c_str(), elf_file_offset, &error, &ignored_vm_data, + &ignored_vm_instrs, &isolate_data, &isolate_instrs, dart::bin::kReadOnly); if (elf == nullptr) { FML_LOG(ERROR) << "Failed to load patch at " << path << " error: " << error; diff --git a/engine/src/flutter/shell/common/shorebird/shorebird.cc b/engine/src/flutter/shell/common/shorebird/shorebird.cc index 8cb32cfc9916b..9a4136b48bbe6 100644 --- a/engine/src/flutter/shell/common/shorebird/shorebird.cc +++ b/engine/src/flutter/shell/common/shorebird/shorebird.cc @@ -46,7 +46,6 @@ extern "C" __attribute__((weak)) unsigned long getauxval(unsigned long type) { #endif #if SHOREBIRD_USE_INTERPRETER - // Global references to the base (unpatched) snapshots from the App.framework. // These are process-global because: // 1. The Shorebird updater library is a process-global singleton with its own @@ -54,27 +53,23 @@ extern "C" __attribute__((weak)) unsigned long getauxval(unsigned long type) { // data for patch generation/validation. // 2. The base snapshots are immutable (baked into the IPA) so sharing them // across isolate groups is safe. -// 3. GetBaseIsolateSnapshot() returns the isolate snapshot for use when -// creating isolate groups with Dart_CreateIsolateGroupWithBaseSnapshot(), -// which needs the base snapshot for linking patched code. // // Note: This design doesn't support multiple engines with different base // snapshots, but I'm not aware of any use cases for that on iOS. static fml::RefPtr vm_snapshot; static fml::RefPtr isolate_snapshot; -void StoreBaseSnapshots(Settings& settings) { - // Create DartSnapshot objects that hold references to the symbol mappings - // in the App.framework. The snapshots are static data in the framework, - // but we need DartSnapshot objects to keep the NativeLibrary refs alive. +void SetBaseSnapshot(Settings& settings) { + // These mappings happen to be to static data in the App.framework, but + // we still need to seem to hold onto the DartSnapshot objects to keep + // the mappings alive. vm_snapshot = DartSnapshot::VMSnapshotFromSettings(settings); isolate_snapshot = DartSnapshot::IsolateSnapshotFromSettings(settings); + Shorebird_SetBaseSnapshots(isolate_snapshot->GetDataMapping(), + isolate_snapshot->GetInstructionsMapping(), + vm_snapshot->GetDataMapping(), + vm_snapshot->GetInstructionsMapping()); } - -fml::RefPtr GetBaseIsolateSnapshot() { - return isolate_snapshot; -} - #endif // SHOREBIRD_USE_INTERPRETER class FileCallbacksImpl { @@ -120,7 +115,7 @@ std::string GetValueFromYaml(const std::string& yaml, const std::string& key) { /// Newer api, used by Desktop implementations. /// Does not directly manipulate Settings. -// FIXME: Consolidate this with the other ConfigureShorebird() API. +// TODO(eseidel): Consolidate this with the other ConfigureShorebird() API. bool ConfigureShorebird(const ShorebirdConfigArgs& args, std::string& patch_path) { patch_path = fml::PathToUtf8(args.release_app_library_path); @@ -170,10 +165,9 @@ bool ConfigureShorebird(const ShorebirdConfigArgs& args, args.shorebird_yaml.c_str()); } - // We've decided not to support synchronous updates on launch for now. - // It's a terrible user experience (having the app hang on launch) and - // instead we will provide examples of how to build a custom update UI - // within Dart, including updating as part of login, etc. + // We do not support synchronous updates on launch, it's a terrible UX. + // Users can implement custom check-for-updates using + // package:shorebird_code_push. // https://github.com/shorebirdtech/shorebird/issues/950 FML_LOG(INFO) << "Checking for active patch"; @@ -218,7 +212,7 @@ bool ConfigureShorebird(const ShorebirdConfigArgs& args, } /// Older api used by iOS and Android, directly manipulates Settings. -// FIXME: Consolidate this with the other ConfigureShorebird() API. +// TODO(eseidel): Consolidate this with the other ConfigureShorebird() API. void ConfigureShorebird(std::string code_cache_path, std::string app_storage_path, Settings& settings, @@ -274,9 +268,9 @@ void ConfigureShorebird(std::string code_cache_path, // package:shorebird_code_push. // https://github.com/shorebirdtech/shorebird/issues/950 - // We store the base snapshot on iOS for use when creating the isolate group. + // We only set the base snapshot on iOS for now. #if SHOREBIRD_USE_INTERPRETER - StoreBaseSnapshots(settings); + SetBaseSnapshot(settings); #endif shorebird_validate_next_boot_patch(); @@ -334,7 +328,7 @@ void* FileCallbacksImpl::Open() { *isolate_snapshot) .release(); #else - // SnapshotsDataHandle exists on all platforms (for testing)but is only used + // SnapshotsDataHandle exists on all platforms (for testing) but is only used // on iOS. iOS patches are generated from just the Dart parts of the snapshot, // excluding the Mach-O specific headers which contain dates and paths that // make them change on every build. diff --git a/engine/src/flutter/shell/common/shorebird/shorebird.h b/engine/src/flutter/shell/common/shorebird/shorebird.h index 50c6f59e1fa60..ab5c9162ea0f2 100644 --- a/engine/src/flutter/shell/common/shorebird/shorebird.h +++ b/engine/src/flutter/shell/common/shorebird/shorebird.h @@ -54,13 +54,6 @@ void ConfigureShorebird(std::string code_cache_path, /// Exposed for testing. std::string GetValueFromYaml(const std::string& yaml, const std::string& key); -#if SHOREBIRD_USE_INTERPRETER -/// Returns the base isolate snapshot for Shorebird linking support. -/// Must be called after ConfigureShorebird() has stored the base snapshots. -/// May return null if not using Shorebird interpreter mode. -fml::RefPtr GetBaseIsolateSnapshot(); -#endif // SHOREBIRD_USE_INTERPRETER - } // namespace flutter #endif // FLUTTER_SHELL_COMMON_SHOREBIRD_SHOREBIRD_H_ From bfb3c9974ce5e1d571c78fa3722f95c0e186df7b Mon Sep 17 00:00:00 2001 From: Eric Seidel Date: Thu, 8 Jan 2026 16:15:45 -0800 Subject: [PATCH 15/95] chore: fix linux --- engine/src/flutter/shell/common/shorebird/shorebird.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/engine/src/flutter/shell/common/shorebird/shorebird.cc b/engine/src/flutter/shell/common/shorebird/shorebird.cc index 9a4136b48bbe6..81469a480f735 100644 --- a/engine/src/flutter/shell/common/shorebird/shorebird.cc +++ b/engine/src/flutter/shell/common/shorebird/shorebird.cc @@ -118,7 +118,7 @@ std::string GetValueFromYaml(const std::string& yaml, const std::string& key) { // TODO(eseidel): Consolidate this with the other ConfigureShorebird() API. bool ConfigureShorebird(const ShorebirdConfigArgs& args, std::string& patch_path) { - patch_path = fml::PathToUtf8(args.release_app_library_path); + patch_path = args.release_app_library_path; auto shorebird_updater_dir_name = "shorebird_updater"; // Parse app id from shorebird.yaml From 742d0a3156895c9a5a5278f3428559fa27361b62 Mon Sep 17 00:00:00 2001 From: Eric Seidel Date: Sun, 11 Jan 2026 15:06:13 -0800 Subject: [PATCH 16/95] feat: add support for patch_verification_mode (#100) * feat: allow patch_verification_mode * test: update tests * chore: rename to patch_verification --- packages/flutter_tools/lib/src/shorebird/shorebird_yaml.dart | 1 + .../test/general.shard/shorebird/shorebird_yaml_test.dart | 3 +++ 2 files changed, 4 insertions(+) diff --git a/packages/flutter_tools/lib/src/shorebird/shorebird_yaml.dart b/packages/flutter_tools/lib/src/shorebird/shorebird_yaml.dart index 9e40f392b7eb8..a741b2683d765 100644 --- a/packages/flutter_tools/lib/src/shorebird/shorebird_yaml.dart +++ b/packages/flutter_tools/lib/src/shorebird/shorebird_yaml.dart @@ -56,6 +56,7 @@ Map compileShorebirdYaml(YamlMap yamlMap, {required String? fla } copyIfSet('base_url'); copyIfSet('auto_update'); + copyIfSet('patch_verification'); final String? shorebirdPublicKeyEnvVar = environment['SHOREBIRD_PUBLIC_KEY']; if (shorebirdPublicKeyEnvVar != null) { compiled['patch_public_key'] = shorebirdPublicKeyEnvVar; diff --git a/packages/flutter_tools/test/general.shard/shorebird/shorebird_yaml_test.dart b/packages/flutter_tools/test/general.shard/shorebird/shorebird_yaml_test.dart index 602de37681555..7e05fec171409 100644 --- a/packages/flutter_tools/test/general.shard/shorebird/shorebird_yaml_test.dart +++ b/packages/flutter_tools/test/general.shard/shorebird/shorebird_yaml_test.dart @@ -52,6 +52,7 @@ flavors: foo: 2-a bar: 3-a base_url: https://example.com +patch_verification: strict '''; final YamlDocument input = loadYamlDocument(yamlContents); final YamlMap yamlMap = input.contents as YamlMap; @@ -61,6 +62,7 @@ base_url: https://example.com 'app_id': '1-a', 'auto_update': false, 'base_url': 'https://example.com', + 'patch_verification': 'strict', }); final Map compiled2 = compileShorebirdYaml(yamlMap, flavor: 'foo', environment: {'SHOREBIRD_PUBLIC_KEY': '4-a'}); @@ -68,6 +70,7 @@ base_url: https://example.com 'app_id': '2-a', 'auto_update': false, 'base_url': 'https://example.com', + 'patch_verification': 'strict', 'patch_public_key': '4-a', }); }); From db53a5abd4c16c35b1107d2f0605a6c956150ed4 Mon Sep 17 00:00:00 2001 From: Eric Seidel Date: Sun, 11 Jan 2026 15:07:00 -0800 Subject: [PATCH 17/95] chore: roll updater to 8691c8f60e69f8eb1f35361f4a22e8c9a7fdf93c to include patch_verification option --- DEPS | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/DEPS b/DEPS index 16a59de7e7f01..94c821d0ca661 100644 --- a/DEPS +++ b/DEPS @@ -20,7 +20,7 @@ vars = { "dart_sdk_revision": "b65ce89c8057d6880e00693a7b0ecd7b9e5f61ca", "dart_sdk_git": "git@github.com:shorebirdtech/dart-sdk.git", "updater_git": "https://github.com/shorebirdtech/updater.git", - "updater_rev": "9db198a6344f7b9ec2843e06a459938a520e314f", + "updater_rev": "8691c8f60e69f8eb1f35361f4a22e8c9a7fdf93c", # WARNING: DO NOT EDIT canvaskit_cipd_instance MANUALLY # See `lib/web_ui/README.md` for how to roll CanvasKit to a new version. From ee041cd1b606048527dd84b0b4c8573fddd2cb01 Mon Sep 17 00:00:00 2001 From: Eric Seidel Date: Thu, 22 Jan 2026 22:20:15 -0800 Subject: [PATCH 18/95] fix: use of FlutterEngineGroup breaks patching (#101) * es/report_start_fix * fix: second callsite --- engine/src/flutter/runtime/shorebird/BUILD.gn | 5 +++ .../flutter/runtime/shorebird/patch_cache.cc | 14 +++++++ .../shell/common/shorebird/shorebird.cc | 39 +++++-------------- 3 files changed, 28 insertions(+), 30 deletions(-) diff --git a/engine/src/flutter/runtime/shorebird/BUILD.gn b/engine/src/flutter/runtime/shorebird/BUILD.gn index facee07ba5cd1..81bf3f588b3f8 100644 --- a/engine/src/flutter/runtime/shorebird/BUILD.gn +++ b/engine/src/flutter/runtime/shorebird/BUILD.gn @@ -12,4 +12,9 @@ source_set("patch_cache") { "//flutter/fml", "//flutter/runtime:libdart", ] + + # For shorebird_report_launch_start() in updater.h + # The include path "third_party/updater/library/include/updater.h" is relative + # to //flutter/, which is already in the default include path. + include_dirs = [ "//flutter" ] } diff --git a/engine/src/flutter/runtime/shorebird/patch_cache.cc b/engine/src/flutter/runtime/shorebird/patch_cache.cc index d3dbe617ebfea..5610eb3393d60 100644 --- a/engine/src/flutter/runtime/shorebird/patch_cache.cc +++ b/engine/src/flutter/runtime/shorebird/patch_cache.cc @@ -4,9 +4,12 @@ #include "flutter/runtime/shorebird/patch_cache.h" +#include + #include "flutter/fml/logging.h" #include "flutter/fml/mapping.h" #include "flutter/runtime/shorebird/patch_mapping.h" +#include "third_party/updater/library/include/updater.h" namespace flutter { @@ -148,7 +151,18 @@ std::shared_ptr TryLoadFromPatch( FML_LOG(INFO) << "Loading symbol from patch: " << symbol_name; + // Report launch_start when we're actually about to use a patch. + // This is called at exactly the right time - right before the patched + // snapshot is loaded. We use std::once_flag to ensure it's only called + // once per process, and only for the first symbol (isolate data). + // This fixes the FlutterEngineGroup issue where report_launch_start was + // called too early (in ConfigureShorebird) before any patch was used. + static std::once_flag launch_start_flag; if (symbol == kIsolateDataSymbol) { + std::call_once(launch_start_flag, []() { + FML_LOG(INFO) << "Reporting launch start for patch"; + shorebird_report_launch_start(); + }); return PatchMapping::CreateIsolateData(cache_entry); } else { FML_CHECK(symbol == kIsolateInstructionsSymbol); diff --git a/engine/src/flutter/shell/common/shorebird/shorebird.cc b/engine/src/flutter/shell/common/shorebird/shorebird.cc index 81469a480f735..d040e07e0285d 100644 --- a/engine/src/flutter/shell/common/shorebird/shorebird.cc +++ b/engine/src/flutter/shell/common/shorebird/shorebird.cc @@ -181,25 +181,14 @@ bool ConfigureShorebird(const ShorebirdConfigArgs& args, FML_LOG(INFO) << "Shorebird updater: no active patch."; } - // We are careful only to report a launch start in the case where it's the - // first time we've configured shorebird this process. Otherwise we could end - // up in a case where we report a launch start, but never a completion (e.g. - // from package:flutter_work_manager which sometimes creates a FlutterEngine - // (and thus configures shorebird) but never runs it. The proper fix for this - // is probably to move the launch_start() call to be later in the lifecycle - // (when the snapshot is loaded and run, rather than when FlutterEngine is - // initialized). This "hack" will still have a problem where FlutterEngine is - // initialized but never run before the app is quit, could still cause us to - // suddenly mark-bad a patch that was never actually attempted to launch. + // Note: shorebird_report_launch_start() is now called from TryLoadFromPatch() + // in runtime/shorebird/patch_cache.cc, right before the patched snapshot is + // actually loaded. This fixes issues with FlutterEngineGroup and other cases + // where ConfigureShorebird() is called but no Shell is created. if (!init_result) { return false; } - // Once start_update_thread is called, the next_boot_patch* functions may - // change their return values if the shorebird_report_launch_failed - // function is called. - shorebird_report_launch_start(); - if (shorebird_should_auto_update()) { FML_LOG(INFO) << "Starting Shorebird update"; shorebird_start_update_thread(); @@ -294,25 +283,15 @@ void ConfigureShorebird(std::string code_cache_path, FML_LOG(INFO) << "Shorebird updater: no active patch."; } - // We are careful only to report a launch start in the case where it's the - // first time we've configured shorebird this process. Otherwise we could end - // up in a case where we report a launch start, but never a completion (e.g. - // from package:flutter_work_manager which sometimes creates a FlutterEngine - // (and thus configures shorebird) but never runs it. The proper fix for this - // is probably to move the launch_start() call to be later in the lifecycle - // (when the snapshot is loaded and run, rather than when FlutterEngine is - // initialized). This "hack" will still have a problem where FlutterEngine is - // initialized but never run before the app is quit, could still cause us to - // suddenly mark-bad a patch that was never actually attempted to launch. + // Note: shorebird_report_launch_start() is now called from TryLoadFromPatch() + // in runtime/shorebird/patch_cache.cc, right before the patched snapshot is + // actually loaded. This fixes issues with FlutterEngineGroup and other cases + // where ConfigureShorebird() is called but no Shell is created. + if (!init_result) { return; } - // Once start_update_thread is called, the next_boot_patch* functions may - // change their return values if the shorebird_report_launch_failed - // function is called. - shorebird_report_launch_start(); - if (shorebird_should_auto_update()) { FML_LOG(INFO) << "Starting Shorebird update"; shorebird_start_update_thread(); From 7602177bbbf754adf73d51441a618b291efcb86f Mon Sep 17 00:00:00 2001 From: Eric Seidel Date: Fri, 23 Jan 2026 15:30:24 -0800 Subject: [PATCH 19/95] chore: add a C++ interface onto the updater (#102) * chore: add a C++ interface onto the updater * chore: centralize SHOREBIRD_PLATFORM_SUPPORTED * test: fix tests --- engine/src/flutter/runtime/shorebird/BUILD.gn | 6 +- .../flutter/runtime/shorebird/patch_cache.cc | 5 +- engine/src/flutter/shell/common/BUILD.gn | 29 +-- engine/src/flutter/shell/common/shell.cc | 11 +- .../flutter/shell/common/shell_unittests.cc | 62 ++++++ .../flutter/shell/common/shorebird/BUILD.gn | 57 +++++- .../shell/common/shorebird/shorebird.cc | 103 ++++------ .../flutter/shell/common/shorebird/updater.cc | 166 +++++++++++++++ .../flutter/shell/common/shorebird/updater.h | 191 ++++++++++++++++++ .../common/shorebird/updater_unittests.cc | 127 ++++++++++++ .../flutter/shell/platform/android/BUILD.gn | 13 -- .../shell/platform/android/flutter_main.cc | 2 - .../shell/platform/darwin/ios/BUILD.gn | 8 - 13 files changed, 647 insertions(+), 133 deletions(-) create mode 100644 engine/src/flutter/shell/common/shorebird/updater.cc create mode 100644 engine/src/flutter/shell/common/shorebird/updater.h create mode 100644 engine/src/flutter/shell/common/shorebird/updater_unittests.cc diff --git a/engine/src/flutter/runtime/shorebird/BUILD.gn b/engine/src/flutter/runtime/shorebird/BUILD.gn index 81bf3f588b3f8..1f856e36d3f48 100644 --- a/engine/src/flutter/runtime/shorebird/BUILD.gn +++ b/engine/src/flutter/runtime/shorebird/BUILD.gn @@ -11,10 +11,6 @@ source_set("patch_cache") { deps = [ "//flutter/fml", "//flutter/runtime:libdart", + "//flutter/shell/common/shorebird:updater", ] - - # For shorebird_report_launch_start() in updater.h - # The include path "third_party/updater/library/include/updater.h" is relative - # to //flutter/, which is already in the default include path. - include_dirs = [ "//flutter" ] } diff --git a/engine/src/flutter/runtime/shorebird/patch_cache.cc b/engine/src/flutter/runtime/shorebird/patch_cache.cc index 5610eb3393d60..b835d01b98639 100644 --- a/engine/src/flutter/runtime/shorebird/patch_cache.cc +++ b/engine/src/flutter/runtime/shorebird/patch_cache.cc @@ -9,7 +9,8 @@ #include "flutter/fml/logging.h" #include "flutter/fml/mapping.h" #include "flutter/runtime/shorebird/patch_mapping.h" -#include "third_party/updater/library/include/updater.h" +#include "flutter/shell/common/shorebird/updater.h" +#include "third_party/dart/runtime/include/dart_api.h" namespace flutter { @@ -161,7 +162,7 @@ std::shared_ptr TryLoadFromPatch( if (symbol == kIsolateDataSymbol) { std::call_once(launch_start_flag, []() { FML_LOG(INFO) << "Reporting launch start for patch"; - shorebird_report_launch_start(); + shorebird::Updater::Instance().ReportLaunchStart(); }); return PatchMapping::CreateIsolateData(cache_entry); } else { diff --git a/engine/src/flutter/shell/common/BUILD.gn b/engine/src/flutter/shell/common/BUILD.gn index bbe42f5a987f8..5eb7b979fe2f0 100644 --- a/engine/src/flutter/shell/common/BUILD.gn +++ b/engine/src/flutter/shell/common/BUILD.gn @@ -153,13 +153,12 @@ source_set("common") { "//flutter/lib/ui", "//flutter/runtime", "//flutter/shell/common:base64", + "//flutter/shell/common/shorebird:updater", "//flutter/shell/geometry", "//flutter/shell/profiling", "//flutter/skia", ] - include_dirs = [ "//flutter/updater" ] - if (impeller_supports_rendering) { sources += [ "snapshot_controller_impeller.cc", @@ -168,31 +167,6 @@ source_set("common") { deps += [ "//flutter/impeller" ] } - - # Needed to compile flutter_tester for macOS. - if (host_os == "mac" && target_os == "mac") { - if (target_cpu == "arm64") { - libs = [ "//flutter/third_party/updater/target/aarch64-apple-darwin/release/libupdater.a" ] - } else if (target_cpu == "x64") { - libs = [ "//flutter/third_party/updater/target/x86_64-apple-darwin/release/libupdater.a" ] - } - } - - # Needed to compile flutter_tester for Windows. - if (host_os == "win" && target_os == "win") { - if (target_cpu == "x64") { - libs = [ - "userenv.lib", - "//flutter/third_party/updater/target/x86_64-pc-windows-msvc/release/updater.lib", - ] - } - } - - if (host_os == "linux" && target_os == "linux") { - if (target_cpu == "x64") { - libs = [ "//flutter/third_party/updater/target/x86_64-unknown-linux-gnu/release/libupdater.a" ] - } - } } # These are in their own source_set to avoid a dependency cycle with //common/graphics @@ -369,6 +343,7 @@ if (enable_unittests) { "//flutter/common/graphics", "//flutter/display_list/testing:display_list_testing", "//flutter/shell/common:base64", + "//flutter/shell/common/shorebird:updater", "//flutter/shell/profiling:profiling_unittests", "//flutter/shell/version", "//flutter/testing:fixture_test", diff --git a/engine/src/flutter/shell/common/shell.cc b/engine/src/flutter/shell/common/shell.cc index 01d7595e2647d..857abed16d087 100644 --- a/engine/src/flutter/shell/common/shell.cc +++ b/engine/src/flutter/shell/common/shell.cc @@ -47,7 +47,7 @@ #include "third_party/skia/include/core/SkGraphics.h" #include "third_party/tonic/common/log.h" -#include "third_party/updater/library/include/updater.h" +#include "flutter/shell/common/shorebird/updater.h" namespace flutter { @@ -524,14 +524,13 @@ Shell::Shell(DartVMRef vm, is_gpu_disabled_sync_switch_(new fml::SyncSwitch(is_gpu_disabled)), weak_factory_gpu_(nullptr), weak_factory_(this) { - // FIXME: This is probably the wrong place to hook into. -#if SHOREBIRD_PLATFORM_SUPPORTED + // Report launch status to Shorebird updater for crash recovery tracking. + // On unsupported platforms, NoOpUpdater handles these calls gracefully. if (!vm_) { - shorebird_report_launch_failure(); + shorebird::Updater::Instance().ReportLaunchFailure(); } else { - shorebird_report_launch_success(); + shorebird::Updater::Instance().ReportLaunchSuccess(); } -#endif FML_CHECK(!settings.enable_software_rendering || !settings.enable_impeller) << "Software rendering is incompatible with Impeller."; if (!settings.enable_impeller && settings.warn_on_impeller_opt_out) { diff --git a/engine/src/flutter/shell/common/shell_unittests.cc b/engine/src/flutter/shell/common/shell_unittests.cc index 60ff1ecbeb213..3565fc852095f 100644 --- a/engine/src/flutter/shell/common/shell_unittests.cc +++ b/engine/src/flutter/shell/common/shell_unittests.cc @@ -53,6 +53,8 @@ #include "third_party/skia/include/gpu/ganesh/mock/GrMockTypes.h" #include "third_party/tonic/converter/dart_converter.h" +#include "flutter/shell/common/shorebird/updater.h" + #ifdef SHELL_ENABLE_VULKAN #include "flutter/vulkan/vulkan_application.h" // nogncheck #endif @@ -5184,6 +5186,66 @@ TEST_F(ShellTest, ShoulDiscardLayerTreeIfFrameIsSizedIncorrectly) { DestroyShell(std::move(shell), task_runners); } +// Test that Shell creation triggers the Shorebird Updater's ReportLaunchSuccess +// call. This is important for the crash recovery mechanism to work correctly. +TEST_F(ShellTest, ShorebirdUpdaterReportLaunchSuccessOnShellCreation) { + // Install a mock updater before creating the shell + auto mock = std::make_unique(); + auto* mock_ptr = mock.get(); + shorebird::Updater::SetInstanceForTesting(std::move(mock)); + + EXPECT_EQ(mock_ptr->launch_success_count(), 0); + EXPECT_EQ(mock_ptr->launch_failure_count(), 0); + + auto settings = CreateSettingsForFixture(); + auto task_runners = GetTaskRunnersForFixture(); + auto shell = CreateShell(settings, task_runners); + ASSERT_TRUE(shell); + + // Shell constructor should have called ReportLaunchSuccess + EXPECT_EQ(mock_ptr->launch_success_count(), 1); + EXPECT_EQ(mock_ptr->launch_failure_count(), 0); + + // Verify the call was logged + const auto& log = mock_ptr->call_log(); + EXPECT_TRUE(std::find(log.begin(), log.end(), "ReportLaunchSuccess") != + log.end()); + + DestroyShell(std::move(shell), task_runners); + + // Clean up - reset the updater instance + shorebird::Updater::ResetInstanceForTesting(); +} + +// Test that creating multiple shells only calls ReportLaunchSuccess for each +// shell. This verifies that each Shell reports its own launch status. +TEST_F(ShellTest, ShorebirdUpdaterReportLaunchSuccessForMultipleShells) { + auto mock = std::make_unique(); + auto* mock_ptr = mock.get(); + shorebird::Updater::SetInstanceForTesting(std::move(mock)); + + EXPECT_EQ(mock_ptr->launch_success_count(), 0); + + auto settings = CreateSettingsForFixture(); + + // Create first shell + auto task_runners1 = GetTaskRunnersForFixture(); + auto shell1 = CreateShell(settings, task_runners1); + ASSERT_TRUE(shell1); + EXPECT_EQ(mock_ptr->launch_success_count(), 1); + + // Create second shell + auto task_runners2 = GetTaskRunnersForFixture(); + auto shell2 = CreateShell(settings, task_runners2); + ASSERT_TRUE(shell2); + EXPECT_EQ(mock_ptr->launch_success_count(), 2); + + DestroyShell(std::move(shell1), task_runners1); + DestroyShell(std::move(shell2), task_runners2); + + shorebird::Updater::ResetInstanceForTesting(); +} + } // namespace testing } // namespace flutter diff --git a/engine/src/flutter/shell/common/shorebird/BUILD.gn b/engine/src/flutter/shell/common/shorebird/BUILD.gn index dc2fc5515bd7c..2b1e29c03512d 100644 --- a/engine/src/flutter/shell/common/shorebird/BUILD.gn +++ b/engine/src/flutter/shell/common/shorebird/BUILD.gn @@ -15,6 +15,56 @@ source_set("snapshots_data_handle") { ] } +# C++ wrapper around the Rust updater C API. +# This provides a testable abstraction layer that can be mocked for testing. +source_set("updater") { + sources = [ + "updater.cc", + "updater.h", + ] + + deps = [ "//flutter/fml" ] + + # For the Rust updater C API (shorebird_report_launch_start, etc.) + include_dirs = [ "//flutter" ] + + # Link the Rust updater static library based on target platform. + if (is_android) { + if (target_cpu == "arm") { + libs = [ "//flutter/third_party/updater/target/armv7-linux-androideabi/release/libupdater.a" ] + } else if (target_cpu == "arm64") { + libs = [ "//flutter/third_party/updater/target/aarch64-linux-android/release/libupdater.a" ] + } else if (target_cpu == "x64") { + libs = [ "//flutter/third_party/updater/target/x86_64-linux-android/release/libupdater.a" ] + } else if (target_cpu == "x86") { + libs = [ "//flutter/third_party/updater/target/i686-linux-android/release/libupdater.a" ] + } + } else if (is_ios) { + if (target_cpu == "arm64") { + libs = [ "//flutter/third_party/updater/target/aarch64-apple-ios/release/libupdater.a" ] + } else if (target_cpu == "x64") { + libs = [ "//flutter/third_party/updater/target/x86_64-apple-ios/release/libupdater.a" ] + } + } else if (is_mac) { + if (target_cpu == "arm64") { + libs = [ "//flutter/third_party/updater/target/aarch64-apple-darwin/release/libupdater.a" ] + } else if (target_cpu == "x64") { + libs = [ "//flutter/third_party/updater/target/x86_64-apple-darwin/release/libupdater.a" ] + } + } else if (is_win) { + if (target_cpu == "x64") { + libs = [ + "userenv.lib", + "//flutter/third_party/updater/target/x86_64-pc-windows-msvc/release/updater.lib", + ] + } + } else if (is_linux) { + if (target_cpu == "x64") { + libs = [ "//flutter/third_party/updater/target/x86_64-unknown-linux-gnu/release/libupdater.a" ] + } + } +} + source_set("shorebird") { sources = [ "shorebird.cc", @@ -23,14 +73,13 @@ source_set("shorebird") { deps = [ ":snapshots_data_handle", + ":updater", "//flutter/fml", "//flutter/runtime", "//flutter/runtime:libdart", "//flutter/shell/common", "//flutter/shell/platform/embedder:embedder_headers", ] - - include_dirs = [ "//flutter/updater" ] } if (enable_unittests) { @@ -45,14 +94,14 @@ if (enable_unittests) { "patch_cache_unittests.cc", "shorebird_unittests.cc", "snapshots_data_handle_unittests.cc", + "updater_unittests.cc", ] - # This only includes snapshots_data_handle and not shorebird because - # shorebird fails to link due to a missing updater lib. deps = [ ":shorebird", ":shorebird_fixtures", ":snapshots_data_handle", + ":updater", "//flutter/runtime", "//flutter/runtime/shorebird:patch_cache", "//flutter/testing", diff --git a/engine/src/flutter/shell/common/shorebird/shorebird.cc b/engine/src/flutter/shell/common/shorebird/shorebird.cc index d040e07e0285d..71d336e90f8b0 100644 --- a/engine/src/flutter/shell/common/shorebird/shorebird.cc +++ b/engine/src/flutter/shell/common/shorebird/shorebird.cc @@ -19,13 +19,12 @@ #include "flutter/runtime/dart_vm.h" #include "flutter/shell/common/shell.h" #include "flutter/shell/common/shorebird/snapshots_data_handle.h" +#include "flutter/shell/common/shorebird/updater.h" #include "flutter/shell/common/switches.h" #include "fml/logging.h" #include "shell/platform/embedder/embedder.h" #include "third_party/dart/runtime/include/dart_tools_api.h" -#include "third_party/updater/library/include/updater.h" - // Namespaced to avoid Google style warnings. namespace flutter { @@ -80,7 +79,7 @@ class FileCallbacksImpl { static void Close(void* file); }; -FileCallbacks ShorebirdFileCallbacks() { +shorebird::FileCallbacks ShorebirdFileCallbacks() { return { .open = FileCallbacksImpl::Open, .read = FileCallbacksImpl::Read, @@ -137,33 +136,22 @@ bool ConfigureShorebird(const ShorebirdConfigArgs& args, {shorebird_updater_dir_name}, fml::FilePermission::kReadWrite); - bool init_result; - // Using a block to make AppParameters lifetime explicit. - { - AppParameters app_parameters; - // Combine version and version_code into a single string. - // We could also pass these separately through to the updater if needed. - auto release_version = args.release_version.version; - if (!args.release_version.build_number.empty()) { - release_version += "+" + args.release_version.build_number; - } - - app_parameters.release_version = release_version.c_str(); - app_parameters.code_cache_dir = code_cache_dir.c_str(); - app_parameters.app_storage_dir = app_storage_dir.c_str(); + // Combine version and version_code into a single string. + // We could also pass these separately through to the updater if needed. + auto release_version = args.release_version.version; + if (!args.release_version.build_number.empty()) { + release_version += "+" + args.release_version.build_number; + } - // https://stackoverflow.com/questions/26032039/convert-vectorstring-into-char-c - std::vector c_paths{}; - c_paths.push_back(args.release_app_library_path.c_str()); - // Do not modify application_library_paths or c_strings will invalidate. + shorebird::AppConfig config; + config.release_version = release_version; + config.original_libapp_paths = {args.release_app_library_path}; + config.app_storage_dir = app_storage_dir; + config.code_cache_dir = code_cache_dir; + config.file_callbacks = ShorebirdFileCallbacks(); + config.yaml_config = args.shorebird_yaml; - app_parameters.original_libapp_paths = c_paths.data(); - app_parameters.original_libapp_paths_size = c_paths.size(); - - // shorebird_init copies from app_parameters and shorebirdYaml. - init_result = shorebird_init(&app_parameters, ShorebirdFileCallbacks(), - args.shorebird_yaml.c_str()); - } + bool init_result = shorebird::Updater::Instance().Init(config); // We do not support synchronous updates on launch, it's a terrible UX. // Users can implement custom check-for-updates using @@ -171,11 +159,10 @@ bool ConfigureShorebird(const ShorebirdConfigArgs& args, // https://github.com/shorebirdtech/shorebird/issues/950 FML_LOG(INFO) << "Checking for active patch"; - shorebird_validate_next_boot_patch(); - char* c_active_path = shorebird_next_boot_patch_path(); - if (c_active_path != NULL) { - patch_path = c_active_path; - shorebird_free_string(c_active_path); + shorebird::Updater::Instance().ValidateNextBootPatch(); + std::string active_path = shorebird::Updater::Instance().NextBootPatchPath(); + if (!active_path.empty()) { + patch_path = active_path; FML_LOG(INFO) << "Shorebird updater: patch path: " << patch_path; } else { FML_LOG(INFO) << "Shorebird updater: no active patch."; @@ -189,9 +176,9 @@ bool ConfigureShorebird(const ShorebirdConfigArgs& args, return false; } - if (shorebird_should_auto_update()) { + if (shorebird::Updater::Instance().ShouldAutoUpdate()) { FML_LOG(INFO) << "Starting Shorebird update"; - shorebird_start_update_thread(); + shorebird::Updater::Instance().StartUpdateThread(); } else { FML_LOG(INFO) << "Shorebird auto_update disabled, not checking for updates."; @@ -226,31 +213,17 @@ void ConfigureShorebird(std::string code_cache_path, {shorebird_updater_dir_name}, fml::FilePermission::kReadWrite); - bool init_result; - // Using a block to make AppParameters lifetime explicit. - { - AppParameters app_parameters; - // Combine version and version_code into a single string. - // We could also pass these separately through to the updater if needed. - auto release_version = version + "+" + version_code; - app_parameters.release_version = release_version.c_str(); - app_parameters.code_cache_dir = code_cache_dir.c_str(); - app_parameters.app_storage_dir = app_storage_dir.c_str(); - - // https://stackoverflow.com/questions/26032039/convert-vectorstring-into-char-c - std::vector c_paths{}; - for (const auto& string : settings.application_library_paths) { - c_paths.push_back(string.c_str()); - } - // Do not modify application_library_paths or c_strings will invalidate. - - app_parameters.original_libapp_paths = c_paths.data(); - app_parameters.original_libapp_paths_size = c_paths.size(); + // Combine version and version_code into a single string. + // We could also pass these separately through to the updater if needed. + shorebird::AppConfig config; + config.release_version = version + "+" + version_code; + config.original_libapp_paths = settings.application_library_paths; + config.app_storage_dir = app_storage_dir; + config.code_cache_dir = code_cache_dir; + config.file_callbacks = ShorebirdFileCallbacks(); + config.yaml_config = shorebird_yaml; - // shorebird_init copies from app_parameters and shorebirdYaml. - init_result = shorebird_init(&app_parameters, ShorebirdFileCallbacks(), - shorebird_yaml.c_str()); - } + bool init_result = shorebird::Updater::Instance().Init(config); // We do not support synchronous updates on launch, it's a terrible UX. // Users can implement custom check-for-updates using @@ -262,11 +235,9 @@ void ConfigureShorebird(std::string code_cache_path, SetBaseSnapshot(settings); #endif - shorebird_validate_next_boot_patch(); - char* c_active_path = shorebird_next_boot_patch_path(); - if (c_active_path != NULL) { - std::string active_path = c_active_path; - shorebird_free_string(c_active_path); + shorebird::Updater::Instance().ValidateNextBootPatch(); + std::string active_path = shorebird::Updater::Instance().NextBootPatchPath(); + if (!active_path.empty()) { FML_LOG(INFO) << "Shorebird updater: active path: " << active_path; #if SHOREBIRD_USE_INTERPRETER @@ -292,9 +263,9 @@ void ConfigureShorebird(std::string code_cache_path, return; } - if (shorebird_should_auto_update()) { + if (shorebird::Updater::Instance().ShouldAutoUpdate()) { FML_LOG(INFO) << "Starting Shorebird update"; - shorebird_start_update_thread(); + shorebird::Updater::Instance().StartUpdateThread(); } else { FML_LOG(INFO) << "Shorebird auto_update disabled, not checking for updates."; diff --git a/engine/src/flutter/shell/common/shorebird/updater.cc b/engine/src/flutter/shell/common/shorebird/updater.cc new file mode 100644 index 0000000000000..84de941ec2d40 --- /dev/null +++ b/engine/src/flutter/shell/common/shorebird/updater.cc @@ -0,0 +1,166 @@ +// Copyright 2013 The Flutter Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#include "flutter/shell/common/shorebird/updater.h" + +#include "flutter/fml/logging.h" + +#if SHOREBIRD_PLATFORM_SUPPORTED +#include "third_party/updater/library/include/updater.h" +#endif + +namespace flutter { +namespace shorebird { + +// Static member definitions +std::unique_ptr Updater::instance_; +std::mutex Updater::instance_mutex_; + +Updater& Updater::Instance() { + std::lock_guard lock(instance_mutex_); + if (!instance_) { +#if SHOREBIRD_PLATFORM_SUPPORTED + instance_ = std::make_unique(); +#else + instance_ = std::make_unique(); +#endif + } + return *instance_; +} + +void Updater::SetInstanceForTesting(std::unique_ptr instance) { + std::lock_guard lock(instance_mutex_); + instance_ = std::move(instance); +} + +void Updater::ResetInstanceForTesting() { + std::lock_guard lock(instance_mutex_); + instance_.reset(); +} + +#if SHOREBIRD_PLATFORM_SUPPORTED +// RealUpdater implementation - wraps the Rust C API + +bool RealUpdater::Init(const AppConfig& config) { + // Convert paths to C strings + std::vector c_paths; + c_paths.reserve(config.original_libapp_paths.size()); + for (const auto& path : config.original_libapp_paths) { + c_paths.push_back(path.c_str()); + } + + AppParameters params; + params.release_version = config.release_version.c_str(); + params.original_libapp_paths = c_paths.data(); + params.original_libapp_paths_size = static_cast(c_paths.size()); + params.app_storage_dir = config.app_storage_dir.c_str(); + params.code_cache_dir = config.code_cache_dir.c_str(); + + // Convert our FileCallbacks to the Rust struct + ::FileCallbacks rust_callbacks; + rust_callbacks.open = config.file_callbacks.open; + rust_callbacks.read = config.file_callbacks.read; + rust_callbacks.seek = config.file_callbacks.seek; + rust_callbacks.close = config.file_callbacks.close; + + return shorebird_init(¶ms, rust_callbacks, config.yaml_config.c_str()); +} + +void RealUpdater::ValidateNextBootPatch() { + shorebird_validate_next_boot_patch(); +} + +std::string RealUpdater::NextBootPatchPath() { + char* c_path = shorebird_next_boot_patch_path(); + if (c_path == nullptr) { + return ""; + } + std::string path(c_path); + shorebird_free_string(c_path); + return path; +} + +void RealUpdater::ReportLaunchStart() { + shorebird_report_launch_start(); +} + +void RealUpdater::ReportLaunchSuccess() { + shorebird_report_launch_success(); +} + +void RealUpdater::ReportLaunchFailure() { + shorebird_report_launch_failure(); +} + +bool RealUpdater::ShouldAutoUpdate() { + return shorebird_should_auto_update(); +} + +void RealUpdater::StartUpdateThread() { + shorebird_start_update_thread(); +} +#endif // SHOREBIRD_PLATFORM_SUPPORTED + +// MockUpdater implementation - for testing + +bool MockUpdater::Init(const AppConfig& config) { + init_count_++; + last_release_version_ = config.release_version; + last_yaml_config_ = config.yaml_config; + call_log_.push_back("Init"); + return init_result_; +} + +void MockUpdater::ValidateNextBootPatch() { + validate_count_++; + call_log_.push_back("ValidateNextBootPatch"); +} + +std::string MockUpdater::NextBootPatchPath() { + call_log_.push_back("NextBootPatchPath"); + return next_boot_patch_path_; +} + +void MockUpdater::ReportLaunchStart() { + launch_start_count_++; + call_log_.push_back("ReportLaunchStart"); +} + +void MockUpdater::ReportLaunchSuccess() { + launch_success_count_++; + call_log_.push_back("ReportLaunchSuccess"); +} + +void MockUpdater::ReportLaunchFailure() { + launch_failure_count_++; + call_log_.push_back("ReportLaunchFailure"); +} + +bool MockUpdater::ShouldAutoUpdate() { + call_log_.push_back("ShouldAutoUpdate"); + return should_auto_update_; +} + +void MockUpdater::StartUpdateThread() { + start_update_thread_count_++; + call_log_.push_back("StartUpdateThread"); +} + +void MockUpdater::Reset() { + init_count_ = 0; + validate_count_ = 0; + launch_start_count_ = 0; + launch_success_count_ = 0; + launch_failure_count_ = 0; + start_update_thread_count_ = 0; + init_result_ = true; + should_auto_update_ = false; + next_boot_patch_path_.clear(); + last_release_version_.clear(); + last_yaml_config_.clear(); + call_log_.clear(); +} + +} // namespace shorebird +} // namespace flutter diff --git a/engine/src/flutter/shell/common/shorebird/updater.h b/engine/src/flutter/shell/common/shorebird/updater.h new file mode 100644 index 0000000000000..f6fa5a9a2f6d9 --- /dev/null +++ b/engine/src/flutter/shell/common/shorebird/updater.h @@ -0,0 +1,191 @@ +// Copyright 2013 The Flutter Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#ifndef FLUTTER_SHELL_COMMON_SHOREBIRD_UPDATER_H_ +#define FLUTTER_SHELL_COMMON_SHOREBIRD_UPDATER_H_ + +#include +#include +#include +#include +#include +#include + +namespace flutter { +namespace shorebird { + +/// File callbacks for iOS patch loading. +/// Mirrors the FileCallbacks struct from the Rust updater. +struct FileCallbacks { + void* (*open)(void); + uintptr_t (*read)(void* file_handle, uint8_t* buffer, uintptr_t count); + int64_t (*seek)(void* file_handle, int64_t offset, int32_t whence); + void (*close)(void* file_handle); +}; + +/// Configuration for initializing the Shorebird updater. +struct AppConfig { + /// Version string for this release (e.g., "1.0.0+1"). + std::string release_version; + + /// Paths to the original AOT libraries (libapp.so on Android, App.framework + /// on iOS). + std::vector original_libapp_paths; + + /// Directory for persistent updater state (survives app updates). + std::string app_storage_dir; + + /// Directory for cached artifacts (cleared on app updates). + std::string code_cache_dir; + + /// Callbacks for iOS patch file access (can be null callbacks on Android). + FileCallbacks file_callbacks; + + /// YAML configuration from shorebird.yaml. + std::string yaml_config; +}; + +/// Abstract interface for the Shorebird updater. +/// +/// This abstraction allows for: +/// 1. Mocking in tests without requiring the real Rust library +/// 2. Future migration from Rust to C++ implementation +/// 3. Test instrumentation (call counting, logging) +class Updater { + public: + virtual ~Updater() = default; + + /// Initialize the updater with configuration. + /// @param config Configuration containing release version, paths, and + /// callbacks + /// @return true if initialization succeeded + virtual bool Init(const AppConfig& config) = 0; + + /// Validate the next boot patch. If invalid, falls back to last good state. + virtual void ValidateNextBootPatch() = 0; + + /// Get the path to the patch that will boot on next run. + /// @return Path to patch, or empty string if no patch available + virtual std::string NextBootPatchPath() = 0; + + // Boot lifecycle methods + virtual void ReportLaunchStart() = 0; + virtual void ReportLaunchSuccess() = 0; + virtual void ReportLaunchFailure() = 0; + + // Update checking + virtual bool ShouldAutoUpdate() = 0; + virtual void StartUpdateThread() = 0; + + // Singleton access + static Updater& Instance(); + + // Test support - allows injecting a mock implementation + static void SetInstanceForTesting(std::unique_ptr instance); + static void ResetInstanceForTesting(); + + protected: + Updater() = default; + + private: + static std::unique_ptr instance_; + static std::mutex instance_mutex_; +}; + +/// No-op implementation for unsupported platforms. +/// All methods are safe to call but do nothing. +class NoOpUpdater : public Updater { + public: + NoOpUpdater() = default; + ~NoOpUpdater() override = default; + + bool Init(const AppConfig& config) override { return true; } + void ValidateNextBootPatch() override {} + std::string NextBootPatchPath() override { return ""; } + void ReportLaunchStart() override {} + void ReportLaunchSuccess() override {} + void ReportLaunchFailure() override {} + bool ShouldAutoUpdate() override { return false; } + void StartUpdateThread() override {} +}; + +#if SHOREBIRD_PLATFORM_SUPPORTED +/// Production implementation that wraps the Rust updater C API. +/// Only available on supported platforms (Android, iOS, macOS, Windows, Linux). +class RealUpdater : public Updater { + public: + RealUpdater() = default; + ~RealUpdater() override = default; + + bool Init(const AppConfig& config) override; + void ValidateNextBootPatch() override; + std::string NextBootPatchPath() override; + void ReportLaunchStart() override; + void ReportLaunchSuccess() override; + void ReportLaunchFailure() override; + bool ShouldAutoUpdate() override; + void StartUpdateThread() override; +}; +#endif // SHOREBIRD_PLATFORM_SUPPORTED + +/// Mock implementation for testing. +/// Tracks call counts and can be queried to verify behavior. +class MockUpdater : public Updater { + public: + MockUpdater() = default; + ~MockUpdater() override = default; + + bool Init(const AppConfig& config) override; + void ValidateNextBootPatch() override; + std::string NextBootPatchPath() override; + void ReportLaunchStart() override; + void ReportLaunchSuccess() override; + void ReportLaunchFailure() override; + bool ShouldAutoUpdate() override; + void StartUpdateThread() override; + + // Test accessors + int init_count() const { return init_count_; } + int validate_count() const { return validate_count_; } + int launch_start_count() const { return launch_start_count_; } + int launch_success_count() const { return launch_success_count_; } + int launch_failure_count() const { return launch_failure_count_; } + int start_update_thread_count() const { return start_update_thread_count_; } + const std::vector& call_log() const { return call_log_; } + + // Last init parameters (for verification) + const std::string& last_release_version() const { + return last_release_version_; + } + const std::string& last_yaml_config() const { return last_yaml_config_; } + + // Test configuration + void set_init_result(bool value) { init_result_ = value; } + void set_should_auto_update(bool value) { should_auto_update_ = value; } + void set_next_boot_patch_path(const std::string& path) { + next_boot_patch_path_ = path; + } + + // Reset all counters and logs + void Reset(); + + private: + int init_count_ = 0; + int validate_count_ = 0; + int launch_start_count_ = 0; + int launch_success_count_ = 0; + int launch_failure_count_ = 0; + int start_update_thread_count_ = 0; + bool init_result_ = true; + bool should_auto_update_ = false; + std::string next_boot_patch_path_; + std::string last_release_version_; + std::string last_yaml_config_; + std::vector call_log_; +}; + +} // namespace shorebird +} // namespace flutter + +#endif // FLUTTER_SHELL_COMMON_SHOREBIRD_UPDATER_H_ diff --git a/engine/src/flutter/shell/common/shorebird/updater_unittests.cc b/engine/src/flutter/shell/common/shorebird/updater_unittests.cc new file mode 100644 index 0000000000000..97fb6ef2129f2 --- /dev/null +++ b/engine/src/flutter/shell/common/shorebird/updater_unittests.cc @@ -0,0 +1,127 @@ +// Copyright 2013 The Flutter Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#include "flutter/shell/common/shorebird/updater.h" + +#include + +#include "gtest/gtest.h" + +namespace flutter { +namespace shorebird { +namespace testing { + +class UpdaterTest : public ::testing::Test { + protected: + void SetUp() override { + // Install a mock for each test + auto mock = std::make_unique(); + mock_ = mock.get(); + Updater::SetInstanceForTesting(std::move(mock)); + } + + void TearDown() override { + mock_ = nullptr; + Updater::ResetInstanceForTesting(); + } + + MockUpdater* mock_ = nullptr; +}; + +TEST_F(UpdaterTest, MockUpdaterTracksLaunchStartCalls) { + EXPECT_EQ(mock_->launch_start_count(), 0); + + Updater::Instance().ReportLaunchStart(); + EXPECT_EQ(mock_->launch_start_count(), 1); + + Updater::Instance().ReportLaunchStart(); + EXPECT_EQ(mock_->launch_start_count(), 2); +} + +TEST_F(UpdaterTest, MockUpdaterTracksLaunchSuccessCalls) { + EXPECT_EQ(mock_->launch_success_count(), 0); + + Updater::Instance().ReportLaunchSuccess(); + EXPECT_EQ(mock_->launch_success_count(), 1); +} + +TEST_F(UpdaterTest, MockUpdaterTracksLaunchFailureCalls) { + EXPECT_EQ(mock_->launch_failure_count(), 0); + + Updater::Instance().ReportLaunchFailure(); + EXPECT_EQ(mock_->launch_failure_count(), 1); +} + +TEST_F(UpdaterTest, MockUpdaterTracksShouldAutoUpdate) { + mock_->set_should_auto_update(false); + EXPECT_FALSE(Updater::Instance().ShouldAutoUpdate()); + + mock_->set_should_auto_update(true); + EXPECT_TRUE(Updater::Instance().ShouldAutoUpdate()); +} + +TEST_F(UpdaterTest, MockUpdaterTracksStartUpdateThreadCalls) { + EXPECT_EQ(mock_->start_update_thread_count(), 0); + + Updater::Instance().StartUpdateThread(); + EXPECT_EQ(mock_->start_update_thread_count(), 1); +} + +TEST_F(UpdaterTest, MockUpdaterCallLogRecordsSequence) { + EXPECT_TRUE(mock_->call_log().empty()); + + Updater::Instance().ReportLaunchStart(); + Updater::Instance().ShouldAutoUpdate(); + Updater::Instance().ReportLaunchSuccess(); + + const auto& log = mock_->call_log(); + ASSERT_EQ(log.size(), 3u); + EXPECT_EQ(log[0], "ReportLaunchStart"); + EXPECT_EQ(log[1], "ShouldAutoUpdate"); + EXPECT_EQ(log[2], "ReportLaunchSuccess"); +} + +TEST_F(UpdaterTest, MockUpdaterResetClearsState) { + Updater::Instance().ReportLaunchStart(); + Updater::Instance().ReportLaunchSuccess(); + mock_->set_should_auto_update(true); + + EXPECT_EQ(mock_->launch_start_count(), 1); + EXPECT_EQ(mock_->launch_success_count(), 1); + EXPECT_TRUE(mock_->ShouldAutoUpdate()); + + mock_->Reset(); + + EXPECT_EQ(mock_->launch_start_count(), 0); + EXPECT_EQ(mock_->launch_success_count(), 0); + // Check call_log before ShouldAutoUpdate() since the method adds to call_log + EXPECT_TRUE(mock_->call_log().empty()); + EXPECT_FALSE(mock_->ShouldAutoUpdate()); +} + +// Test that demonstrates the std::once_flag pattern works correctly. +// This is the same pattern used in TryLoadFromPatch. +TEST_F(UpdaterTest, OncePerProcessPatternOnlyCallsOnce) { + static std::once_flag test_flag; + int call_count = 0; + + auto simulate_patch_load = [&]() { + std::call_once(test_flag, [&]() { + call_count++; + Updater::Instance().ReportLaunchStart(); + }); + }; + + // Simulate multiple engines loading patches + simulate_patch_load(); // Engine 1 + simulate_patch_load(); // Engine 2 + simulate_patch_load(); // Engine 3 + + EXPECT_EQ(call_count, 1); + EXPECT_EQ(mock_->launch_start_count(), 1); +} + +} // namespace testing +} // namespace shorebird +} // namespace flutter diff --git a/engine/src/flutter/shell/platform/android/BUILD.gn b/engine/src/flutter/shell/platform/android/BUILD.gn index 3b41ad6dd3b7a..5909efa260b4f 100644 --- a/engine/src/flutter/shell/platform/android/BUILD.gn +++ b/engine/src/flutter/shell/platform/android/BUILD.gn @@ -188,8 +188,6 @@ source_set("flutter_shell_native_src") { public_configs = [ "//flutter:config" ] - include_dirs = [ "//flutter/updater" ] - defines = [] libs = [ @@ -197,17 +195,6 @@ source_set("flutter_shell_native_src") { "EGL", "GLESv2", ] - if (target_cpu == "arm") { - libs += [ "//flutter/third_party/updater/target/armv7-linux-androideabi/release/libupdater.a" ] - } else if (target_cpu == "arm64") { - libs += [ "//flutter/third_party/updater/target/aarch64-linux-android/release/libupdater.a" ] - } else if (target_cpu == "x64") { - libs += [ "//flutter/third_party/updater/target/x86_64-linux-android/release/libupdater.a" ] - } else if (target_cpu == "x86") { - libs += [ "//flutter/third_party/updater/target/i686-linux-android/release/libupdater.a" ] - } else { - assert(false, "Unsupported target_cpu") - } } action("gen_android_build_config_java") { diff --git a/engine/src/flutter/shell/platform/android/flutter_main.cc b/engine/src/flutter/shell/platform/android/flutter_main.cc index d881ca39efd3f..45e369f5b3a9d 100644 --- a/engine/src/flutter/shell/platform/android/flutter_main.cc +++ b/engine/src/flutter/shell/platform/android/flutter_main.cc @@ -30,8 +30,6 @@ #include "impeller/toolkit/android/proc_table.h" #include "txt/platform.h" -#include "third_party/updater/library/include/updater.h" - namespace flutter { constexpr int kMinimumAndroidApiLevelForImpeller = 29; diff --git a/engine/src/flutter/shell/platform/darwin/ios/BUILD.gn b/engine/src/flutter/shell/platform/darwin/ios/BUILD.gn index c4548b572c8cd..5b3fa88938f11 100644 --- a/engine/src/flutter/shell/platform/darwin/ios/BUILD.gn +++ b/engine/src/flutter/shell/platform/darwin/ios/BUILD.gn @@ -88,14 +88,6 @@ source_set("flutter_framework_source") { "//build/config/ios:ios_application_extension", ] - if (target_cpu == "arm64") { - libs = [ "//flutter/third_party/updater/target/aarch64-apple-ios/release/libupdater.a" ] - } else if (target_cpu == "x64") { - libs = [ "//flutter/third_party/updater/target/x86_64-apple-ios/release/libupdater.a" ] - } else { - assert(false, "Unsupported target_cpu") - } - sources = [ "framework/Source/FlutterAppDelegate.mm", "framework/Source/FlutterAppDelegate_Internal.h", From 550f16bb1c8c82f9d160a2e45d467d1853895746 Mon Sep 17 00:00:00 2001 From: Eric Seidel Date: Fri, 30 Jan 2026 18:41:39 -0800 Subject: [PATCH 20/95] fix: start/success reporting (#103) Previously we stopped reporting start on android by accident. This fixes that. I also removed the once-per-process guard since it's not necessary. This should be correctly reporting once-per-shell and let the rust code only handle the first of the calls. Fixes https://github.com/shorebirdtech/shorebird/issues/3488 --- engine/src/flutter/runtime/dart_snapshot.cc | 6 +++ .../flutter/runtime/shorebird/patch_cache.cc | 15 ++----- .../flutter/shell/common/shell_unittests.cc | 42 +++++++++-------- .../common/shorebird/updater_unittests.cc | 45 ++++++++++--------- 4 files changed, 53 insertions(+), 55 deletions(-) diff --git a/engine/src/flutter/runtime/dart_snapshot.cc b/engine/src/flutter/runtime/dart_snapshot.cc index de35132f4544c..93fec8548c8a8 100644 --- a/engine/src/flutter/runtime/dart_snapshot.cc +++ b/engine/src/flutter/runtime/dart_snapshot.cc @@ -17,6 +17,7 @@ #if SHOREBIRD_USE_INTERPRETER #include "flutter/runtime/shorebird/patch_cache.h" // nogncheck #endif +#include "flutter/shell/common/shorebird/updater.h" // nogncheck namespace flutter { @@ -150,6 +151,11 @@ static std::shared_ptr ResolveIsolateData( true // dontneed_safe ); #else // DART_SNAPSHOT_STATIC_LINK + // Report launch start to pair with ReportLaunchSuccess/ReportLaunchFailure + // in Shell::Shell. Called once per engine, matching the per-engine success/ + // failure calls. The Rust updater no-ops when no patch is booting. + FML_LOG(INFO) << "Reporting launch start for patch"; + shorebird::Updater::Instance().ReportLaunchStart(); #if SHOREBIRD_USE_INTERPRETER // Try loading from a Shorebird patch first. if (auto mapping = TryLoadFromPatch(settings.application_library_paths, diff --git a/engine/src/flutter/runtime/shorebird/patch_cache.cc b/engine/src/flutter/runtime/shorebird/patch_cache.cc index b835d01b98639..8c5cc3a83f182 100644 --- a/engine/src/flutter/runtime/shorebird/patch_cache.cc +++ b/engine/src/flutter/runtime/shorebird/patch_cache.cc @@ -9,7 +9,6 @@ #include "flutter/fml/logging.h" #include "flutter/fml/mapping.h" #include "flutter/runtime/shorebird/patch_mapping.h" -#include "flutter/shell/common/shorebird/updater.h" #include "third_party/dart/runtime/include/dart_api.h" namespace flutter { @@ -152,18 +151,10 @@ std::shared_ptr TryLoadFromPatch( FML_LOG(INFO) << "Loading symbol from patch: " << symbol_name; - // Report launch_start when we're actually about to use a patch. - // This is called at exactly the right time - right before the patched - // snapshot is loaded. We use std::once_flag to ensure it's only called - // once per process, and only for the first symbol (isolate data). - // This fixes the FlutterEngineGroup issue where report_launch_start was - // called too early (in ConfigureShorebird) before any patch was used. - static std::once_flag launch_start_flag; + // ReportLaunchStart is now called from ResolveIsolateData in + // dart_snapshot.cc, which runs before TryLoadFromPatch on all platforms. + if (symbol == kIsolateDataSymbol) { - std::call_once(launch_start_flag, []() { - FML_LOG(INFO) << "Reporting launch start for patch"; - shorebird::Updater::Instance().ReportLaunchStart(); - }); return PatchMapping::CreateIsolateData(cache_entry); } else { FML_CHECK(symbol == kIsolateInstructionsSymbol); diff --git a/engine/src/flutter/shell/common/shell_unittests.cc b/engine/src/flutter/shell/common/shell_unittests.cc index 3565fc852095f..21e4f45314617 100644 --- a/engine/src/flutter/shell/common/shell_unittests.cc +++ b/engine/src/flutter/shell/common/shell_unittests.cc @@ -5186,60 +5186,58 @@ TEST_F(ShellTest, ShoulDiscardLayerTreeIfFrameIsSizedIncorrectly) { DestroyShell(std::move(shell), task_runners); } -// Test that Shell creation triggers the Shorebird Updater's ReportLaunchSuccess -// call. This is important for the crash recovery mechanism to work correctly. -TEST_F(ShellTest, ShorebirdUpdaterReportLaunchSuccessOnShellCreation) { - // Install a mock updater before creating the shell +// Test the full boot flow: ReportLaunchStart is called from +// ResolveIsolateData, then ReportLaunchSuccess from the Shell constructor. +// Each shell gets a paired Start+Success. +TEST_F(ShellTest, ShorebirdBootFlowCallsLaunchStartThenSuccess) { auto mock = std::make_unique(); auto* mock_ptr = mock.get(); shorebird::Updater::SetInstanceForTesting(std::move(mock)); - EXPECT_EQ(mock_ptr->launch_success_count(), 0); - EXPECT_EQ(mock_ptr->launch_failure_count(), 0); - auto settings = CreateSettingsForFixture(); auto task_runners = GetTaskRunnersForFixture(); auto shell = CreateShell(settings, task_runners); ASSERT_TRUE(shell); - // Shell constructor should have called ReportLaunchSuccess - EXPECT_EQ(mock_ptr->launch_success_count(), 1); - EXPECT_EQ(mock_ptr->launch_failure_count(), 0); - - // Verify the call was logged const auto& log = mock_ptr->call_log(); - EXPECT_TRUE(std::find(log.begin(), log.end(), "ReportLaunchSuccess") != - log.end()); + ASSERT_EQ(log.size(), 2u); + EXPECT_EQ(log[0], "ReportLaunchStart"); + EXPECT_EQ(log[1], "ReportLaunchSuccess"); DestroyShell(std::move(shell), task_runners); - - // Clean up - reset the updater instance shorebird::Updater::ResetInstanceForTesting(); } -// Test that creating multiple shells only calls ReportLaunchSuccess for each -// shell. This verifies that each Shell reports its own launch status. +// Test that each shell gets a paired ReportLaunchStart + ReportLaunchSuccess. TEST_F(ShellTest, ShorebirdUpdaterReportLaunchSuccessForMultipleShells) { auto mock = std::make_unique(); auto* mock_ptr = mock.get(); shorebird::Updater::SetInstanceForTesting(std::move(mock)); - EXPECT_EQ(mock_ptr->launch_success_count(), 0); - auto settings = CreateSettingsForFixture(); - // Create first shell + // Create first shell โ€” gets Start + Success auto task_runners1 = GetTaskRunnersForFixture(); auto shell1 = CreateShell(settings, task_runners1); ASSERT_TRUE(shell1); + EXPECT_EQ(mock_ptr->launch_start_count(), 1); EXPECT_EQ(mock_ptr->launch_success_count(), 1); - // Create second shell + // Create second shell โ€” also gets Start + Success auto task_runners2 = GetTaskRunnersForFixture(); auto shell2 = CreateShell(settings, task_runners2); ASSERT_TRUE(shell2); + EXPECT_EQ(mock_ptr->launch_start_count(), 2); EXPECT_EQ(mock_ptr->launch_success_count(), 2); + // Full call log: Start+Success per shell + const auto& log = mock_ptr->call_log(); + ASSERT_EQ(log.size(), 4u); + EXPECT_EQ(log[0], "ReportLaunchStart"); + EXPECT_EQ(log[1], "ReportLaunchSuccess"); + EXPECT_EQ(log[2], "ReportLaunchStart"); + EXPECT_EQ(log[3], "ReportLaunchSuccess"); + DestroyShell(std::move(shell1), task_runners1); DestroyShell(std::move(shell2), task_runners2); diff --git a/engine/src/flutter/shell/common/shorebird/updater_unittests.cc b/engine/src/flutter/shell/common/shorebird/updater_unittests.cc index 97fb6ef2129f2..569673a9e5b1c 100644 --- a/engine/src/flutter/shell/common/shorebird/updater_unittests.cc +++ b/engine/src/flutter/shell/common/shorebird/updater_unittests.cc @@ -4,8 +4,6 @@ #include "flutter/shell/common/shorebird/updater.h" -#include - #include "gtest/gtest.h" namespace flutter { @@ -100,26 +98,31 @@ TEST_F(UpdaterTest, MockUpdaterResetClearsState) { EXPECT_FALSE(mock_->ShouldAutoUpdate()); } -// Test that demonstrates the std::once_flag pattern works correctly. -// This is the same pattern used in TryLoadFromPatch. -TEST_F(UpdaterTest, OncePerProcessPatternOnlyCallsOnce) { - static std::once_flag test_flag; - int call_count = 0; - - auto simulate_patch_load = [&]() { - std::call_once(test_flag, [&]() { - call_count++; - Updater::Instance().ReportLaunchStart(); - }); - }; - - // Simulate multiple engines loading patches - simulate_patch_load(); // Engine 1 - simulate_patch_load(); // Engine 2 - simulate_patch_load(); // Engine 3 - - EXPECT_EQ(call_count, 1); +// ReportLaunchStart and ReportLaunchSuccess are always paired per shell. +// The Rust updater no-ops both when no patch is booting. +TEST_F(UpdaterTest, LaunchStartAndSuccessAreAlwaysPaired) { + Updater::Instance().ReportLaunchStart(); + Updater::Instance().ReportLaunchSuccess(); + + EXPECT_EQ(mock_->launch_start_count(), 1); + EXPECT_EQ(mock_->launch_success_count(), 1); + const auto& log = mock_->call_log(); + ASSERT_EQ(log.size(), 2u); + EXPECT_EQ(log[0], "ReportLaunchStart"); + EXPECT_EQ(log[1], "ReportLaunchSuccess"); +} + +// ReportLaunchStart and ReportLaunchFailure are paired on failed boots. +TEST_F(UpdaterTest, LaunchStartAndFailureAreAlwaysPaired) { + Updater::Instance().ReportLaunchStart(); + Updater::Instance().ReportLaunchFailure(); + EXPECT_EQ(mock_->launch_start_count(), 1); + EXPECT_EQ(mock_->launch_failure_count(), 1); + const auto& log = mock_->call_log(); + ASSERT_EQ(log.size(), 2u); + EXPECT_EQ(log[0], "ReportLaunchStart"); + EXPECT_EQ(log[1], "ReportLaunchFailure"); } } // namespace testing From 0142f7e08fd197ad4242f51e735d02fa76917acb Mon Sep 17 00:00:00 2001 From: Eric Seidel Date: Mon, 2 Feb 2026 16:17:02 -0800 Subject: [PATCH 21/95] fix: only call shorebird_report_start once (#105) As part of our previous fix for FlutterEngineGroup, we introduced a new bug whereby report_launch_start could be called more than once in a multi-engine scenerio. That would cause confusion about what the current boot patch is, since the current patch is updated as part of report_launch_start. report_launch_start should only be called once per processs, which this change fixes. We still need more end-to-end testing at this layer to prevent bugs like this from sneaking in. --- engine/src/flutter/runtime/dart_snapshot.cc | 9 +-- engine/src/flutter/shell/common/shell.cc | 7 +- .../flutter/shell/common/shell_unittests.cc | 26 ++++--- .../flutter/shell/common/shorebird/updater.cc | 48 +++++++++++-- .../flutter/shell/common/shorebird/updater.h | 65 +++++++++++++---- .../common/shorebird/updater_unittests.cc | 71 ++++++++++++++++--- 6 files changed, 183 insertions(+), 43 deletions(-) diff --git a/engine/src/flutter/runtime/dart_snapshot.cc b/engine/src/flutter/runtime/dart_snapshot.cc index 93fec8548c8a8..d3c6be4373745 100644 --- a/engine/src/flutter/runtime/dart_snapshot.cc +++ b/engine/src/flutter/runtime/dart_snapshot.cc @@ -151,10 +151,11 @@ static std::shared_ptr ResolveIsolateData( true // dontneed_safe ); #else // DART_SNAPSHOT_STATIC_LINK - // Report launch start to pair with ReportLaunchSuccess/ReportLaunchFailure - // in Shell::Shell. Called once per engine, matching the per-engine success/ - // failure calls. The Rust updater no-ops when no patch is booting. - FML_LOG(INFO) << "Reporting launch start for patch"; + // Tell the Rust updater we're booting from whatever patch it selected. + // This copies next_boot โ†’ current_boot in the Rust state. The call is + // guarded inside Updater to execute at most once per process โ€” see the + // Updater class comment for why this matters in add-to-app and + // FlutterEngineGroup scenarios. shorebird::Updater::Instance().ReportLaunchStart(); #if SHOREBIRD_USE_INTERPRETER // Try loading from a Shorebird patch first. diff --git a/engine/src/flutter/shell/common/shell.cc b/engine/src/flutter/shell/common/shell.cc index 857abed16d087..d01e5e03bb6de 100644 --- a/engine/src/flutter/shell/common/shell.cc +++ b/engine/src/flutter/shell/common/shell.cc @@ -524,7 +524,12 @@ Shell::Shell(DartVMRef vm, is_gpu_disabled_sync_switch_(new fml::SyncSwitch(is_gpu_disabled)), weak_factory_gpu_(nullptr), weak_factory_(this) { - // Report launch status to Shorebird updater for crash recovery tracking. + // Report launch outcome to the Shorebird updater for crash recovery. + // If the VM failed to start, we report failure so the updater can roll + // back the patch. These calls are guarded inside Updater to execute at + // most once per process โ€” only the first Shell's outcome is reported. + // In add-to-app, subsequent engines are silently ignored since they + // boot from the same snapshot that was already reported on. // On unsupported platforms, NoOpUpdater handles these calls gracefully. if (!vm_) { shorebird::Updater::Instance().ReportLaunchFailure(); diff --git a/engine/src/flutter/shell/common/shell_unittests.cc b/engine/src/flutter/shell/common/shell_unittests.cc index 21e4f45314617..300277877f404 100644 --- a/engine/src/flutter/shell/common/shell_unittests.cc +++ b/engine/src/flutter/shell/common/shell_unittests.cc @@ -5188,11 +5188,12 @@ TEST_F(ShellTest, ShoulDiscardLayerTreeIfFrameIsSizedIncorrectly) { // Test the full boot flow: ReportLaunchStart is called from // ResolveIsolateData, then ReportLaunchSuccess from the Shell constructor. -// Each shell gets a paired Start+Success. +// Both are guarded to run at most once per process. TEST_F(ShellTest, ShorebirdBootFlowCallsLaunchStartThenSuccess) { auto mock = std::make_unique(); auto* mock_ptr = mock.get(); shorebird::Updater::SetInstanceForTesting(std::move(mock)); + shorebird::Updater::ResetLaunchStateForTesting(); auto settings = CreateSettingsForFixture(); auto task_runners = GetTaskRunnersForFixture(); @@ -5205,14 +5206,20 @@ TEST_F(ShellTest, ShorebirdBootFlowCallsLaunchStartThenSuccess) { EXPECT_EQ(log[1], "ReportLaunchSuccess"); DestroyShell(std::move(shell), task_runners); + shorebird::Updater::ResetLaunchStateForTesting(); shorebird::Updater::ResetInstanceForTesting(); } -// Test that each shell gets a paired ReportLaunchStart + ReportLaunchSuccess. -TEST_F(ShellTest, ShorebirdUpdaterReportLaunchSuccessForMultipleShells) { +// In add-to-app, multiple engines may be created within a single process. +// Only the first engine should report launch start/success to the Rust +// updater. This prevents the updater from promoting a newly-downloaded patch +// to "current_boot" when subsequent engines are still running the original +// snapshot that was selected at process init time. +TEST_F(ShellTest, ShorebirdUpdaterReportsOnlyOnceForMultipleShells) { auto mock = std::make_unique(); auto* mock_ptr = mock.get(); shorebird::Updater::SetInstanceForTesting(std::move(mock)); + shorebird::Updater::ResetLaunchStateForTesting(); auto settings = CreateSettingsForFixture(); @@ -5223,24 +5230,23 @@ TEST_F(ShellTest, ShorebirdUpdaterReportLaunchSuccessForMultipleShells) { EXPECT_EQ(mock_ptr->launch_start_count(), 1); EXPECT_EQ(mock_ptr->launch_success_count(), 1); - // Create second shell โ€” also gets Start + Success + // Create second shell โ€” guarded, no additional Start or Success calls. auto task_runners2 = GetTaskRunnersForFixture(); auto shell2 = CreateShell(settings, task_runners2); ASSERT_TRUE(shell2); - EXPECT_EQ(mock_ptr->launch_start_count(), 2); - EXPECT_EQ(mock_ptr->launch_success_count(), 2); + EXPECT_EQ(mock_ptr->launch_start_count(), 1); + EXPECT_EQ(mock_ptr->launch_success_count(), 1); - // Full call log: Start+Success per shell + // Only one Start+Success pair in the call log. const auto& log = mock_ptr->call_log(); - ASSERT_EQ(log.size(), 4u); + ASSERT_EQ(log.size(), 2u); EXPECT_EQ(log[0], "ReportLaunchStart"); EXPECT_EQ(log[1], "ReportLaunchSuccess"); - EXPECT_EQ(log[2], "ReportLaunchStart"); - EXPECT_EQ(log[3], "ReportLaunchSuccess"); DestroyShell(std::move(shell1), task_runners1); DestroyShell(std::move(shell2), task_runners2); + shorebird::Updater::ResetLaunchStateForTesting(); shorebird::Updater::ResetInstanceForTesting(); } diff --git a/engine/src/flutter/shell/common/shorebird/updater.cc b/engine/src/flutter/shell/common/shorebird/updater.cc index 84de941ec2d40..63d993613c3f9 100644 --- a/engine/src/flutter/shell/common/shorebird/updater.cc +++ b/engine/src/flutter/shell/common/shorebird/updater.cc @@ -16,6 +16,8 @@ namespace shorebird { // Static member definitions std::unique_ptr Updater::instance_; std::mutex Updater::instance_mutex_; +std::atomic Updater::launch_started_{false}; +std::atomic Updater::launch_completed_{false}; Updater& Updater::Instance() { std::lock_guard lock(instance_mutex_); @@ -39,6 +41,40 @@ void Updater::ResetInstanceForTesting() { instance_.reset(); } +void Updater::ResetLaunchStateForTesting() { + launch_started_.store(false); + launch_completed_.store(false); +} + +void Updater::ReportLaunchStart() { + // Guard: only the first engine in a process should promote next_boot โ†’ + // current_boot in the Rust updater. See class-level comment for rationale. + bool expected = false; + if (!launch_started_.compare_exchange_strong(expected, true)) { + return; + } + DoReportLaunchStart(); +} + +void Updater::ReportLaunchSuccess() { + // Guard: only report success once per process. Subsequent engines reuse + // the same patch and don't need to re-confirm the boot. + bool expected = false; + if (!launch_completed_.compare_exchange_strong(expected, true)) { + return; + } + DoReportLaunchSuccess(); +} + +void Updater::ReportLaunchFailure() { + // Guard: only report failure once per process. + bool expected = false; + if (!launch_completed_.compare_exchange_strong(expected, true)) { + return; + } + DoReportLaunchFailure(); +} + #if SHOREBIRD_PLATFORM_SUPPORTED // RealUpdater implementation - wraps the Rust C API @@ -81,15 +117,15 @@ std::string RealUpdater::NextBootPatchPath() { return path; } -void RealUpdater::ReportLaunchStart() { +void RealUpdater::DoReportLaunchStart() { shorebird_report_launch_start(); } -void RealUpdater::ReportLaunchSuccess() { +void RealUpdater::DoReportLaunchSuccess() { shorebird_report_launch_success(); } -void RealUpdater::ReportLaunchFailure() { +void RealUpdater::DoReportLaunchFailure() { shorebird_report_launch_failure(); } @@ -122,17 +158,17 @@ std::string MockUpdater::NextBootPatchPath() { return next_boot_patch_path_; } -void MockUpdater::ReportLaunchStart() { +void MockUpdater::DoReportLaunchStart() { launch_start_count_++; call_log_.push_back("ReportLaunchStart"); } -void MockUpdater::ReportLaunchSuccess() { +void MockUpdater::DoReportLaunchSuccess() { launch_success_count_++; call_log_.push_back("ReportLaunchSuccess"); } -void MockUpdater::ReportLaunchFailure() { +void MockUpdater::DoReportLaunchFailure() { launch_failure_count_++; call_log_.push_back("ReportLaunchFailure"); } diff --git a/engine/src/flutter/shell/common/shorebird/updater.h b/engine/src/flutter/shell/common/shorebird/updater.h index f6fa5a9a2f6d9..aeb2d1f2d90cc 100644 --- a/engine/src/flutter/shell/common/shorebird/updater.h +++ b/engine/src/flutter/shell/common/shorebird/updater.h @@ -5,6 +5,7 @@ #ifndef FLUTTER_SHELL_COMMON_SHOREBIRD_UPDATER_H_ #define FLUTTER_SHELL_COMMON_SHOREBIRD_UPDATER_H_ +#include #include #include #include @@ -52,6 +53,29 @@ struct AppConfig { /// 1. Mocking in tests without requiring the real Rust library /// 2. Future migration from Rust to C++ implementation /// 3. Test instrumentation (call counting, logging) +/// +/// ## Launch lifecycle (start/success/failure) +/// +/// The Rust updater uses a start/success/failure protocol to detect crashes: +/// - `ReportLaunchStart` copies `next_boot` โ†’ `current_boot` in the Rust +/// state. If the app crashes before `ReportLaunchSuccess`, the updater +/// assumes the patch caused the crash and rolls back on the next launch. +/// +/// These calls are guarded to execute at most once per process because: +/// 1. The Rust updater is a process-global singleton โ€” calling +/// `report_launch_start` multiple times would repeatedly copy `next_boot` +/// โ†’ `current_boot`, which could promote a newly-downloaded (but not yet +/// booted) patch to "current" even though the running engine loaded the +/// old snapshot. +/// 2. In add-to-app, multiple FlutterEngines may be created and destroyed +/// within a single process. Each engine creation resolves snapshots and +/// constructs a Shell, but we must only report launch start/success once +/// โ€” for the first engine that actually boots. Without this guard, a +/// background update that completes between engine creations would get +/// promoted to "current" by the second engine's `ReportLaunchStart`, +/// even though that engine is still running the old snapshot. +/// +/// Tests can call `ResetLaunchStateForTesting()` to re-enable the guards. class Updater { public: virtual ~Updater() = default; @@ -69,10 +93,12 @@ class Updater { /// @return Path to patch, or empty string if no patch available virtual std::string NextBootPatchPath() = 0; - // Boot lifecycle methods - virtual void ReportLaunchStart() = 0; - virtual void ReportLaunchSuccess() = 0; - virtual void ReportLaunchFailure() = 0; + // Boot lifecycle methods โ€” guarded to run at most once per process. + // Callers may call these freely; subsequent calls after the first are + // silently ignored. + void ReportLaunchStart(); + void ReportLaunchSuccess(); + void ReportLaunchFailure(); // Update checking virtual bool ShouldAutoUpdate() = 0; @@ -85,12 +111,25 @@ class Updater { static void SetInstanceForTesting(std::unique_ptr instance); static void ResetInstanceForTesting(); + /// Resets the once-per-process launch guards so tests can verify + /// start/success/failure calls on fresh Updater instances. + static void ResetLaunchStateForTesting(); + protected: Updater() = default; + // Subclass hooks โ€” called by the public guarded methods above. + virtual void DoReportLaunchStart() = 0; + virtual void DoReportLaunchSuccess() = 0; + virtual void DoReportLaunchFailure() = 0; + private: static std::unique_ptr instance_; static std::mutex instance_mutex_; + + // Once-per-process guards for launch lifecycle. + static std::atomic launch_started_; + static std::atomic launch_completed_; }; /// No-op implementation for unsupported platforms. @@ -103,9 +142,9 @@ class NoOpUpdater : public Updater { bool Init(const AppConfig& config) override { return true; } void ValidateNextBootPatch() override {} std::string NextBootPatchPath() override { return ""; } - void ReportLaunchStart() override {} - void ReportLaunchSuccess() override {} - void ReportLaunchFailure() override {} + void DoReportLaunchStart() override {} + void DoReportLaunchSuccess() override {} + void DoReportLaunchFailure() override {} bool ShouldAutoUpdate() override { return false; } void StartUpdateThread() override {} }; @@ -121,9 +160,9 @@ class RealUpdater : public Updater { bool Init(const AppConfig& config) override; void ValidateNextBootPatch() override; std::string NextBootPatchPath() override; - void ReportLaunchStart() override; - void ReportLaunchSuccess() override; - void ReportLaunchFailure() override; + void DoReportLaunchStart() override; + void DoReportLaunchSuccess() override; + void DoReportLaunchFailure() override; bool ShouldAutoUpdate() override; void StartUpdateThread() override; }; @@ -139,9 +178,9 @@ class MockUpdater : public Updater { bool Init(const AppConfig& config) override; void ValidateNextBootPatch() override; std::string NextBootPatchPath() override; - void ReportLaunchStart() override; - void ReportLaunchSuccess() override; - void ReportLaunchFailure() override; + void DoReportLaunchStart() override; + void DoReportLaunchSuccess() override; + void DoReportLaunchFailure() override; bool ShouldAutoUpdate() override; void StartUpdateThread() override; diff --git a/engine/src/flutter/shell/common/shorebird/updater_unittests.cc b/engine/src/flutter/shell/common/shorebird/updater_unittests.cc index 569673a9e5b1c..def93050e885d 100644 --- a/engine/src/flutter/shell/common/shorebird/updater_unittests.cc +++ b/engine/src/flutter/shell/common/shorebird/updater_unittests.cc @@ -13,42 +13,56 @@ namespace testing { class UpdaterTest : public ::testing::Test { protected: void SetUp() override { - // Install a mock for each test + // Install a mock for each test and reset the once-per-process guards + // so each test starts with a clean slate. auto mock = std::make_unique(); mock_ = mock.get(); Updater::SetInstanceForTesting(std::move(mock)); + Updater::ResetLaunchStateForTesting(); } void TearDown() override { mock_ = nullptr; Updater::ResetInstanceForTesting(); + Updater::ResetLaunchStateForTesting(); } MockUpdater* mock_ = nullptr; }; -TEST_F(UpdaterTest, MockUpdaterTracksLaunchStartCalls) { +// ReportLaunchStart is guarded to run at most once per process. +// The second call should be silently ignored. +TEST_F(UpdaterTest, ReportLaunchStartOnlyCallsOnce) { EXPECT_EQ(mock_->launch_start_count(), 0); Updater::Instance().ReportLaunchStart(); EXPECT_EQ(mock_->launch_start_count(), 1); + // Second call is a no-op due to the once-per-process guard. Updater::Instance().ReportLaunchStart(); - EXPECT_EQ(mock_->launch_start_count(), 2); + EXPECT_EQ(mock_->launch_start_count(), 1); } -TEST_F(UpdaterTest, MockUpdaterTracksLaunchSuccessCalls) { +TEST_F(UpdaterTest, ReportLaunchSuccessOnlyCallsOnce) { EXPECT_EQ(mock_->launch_success_count(), 0); Updater::Instance().ReportLaunchSuccess(); EXPECT_EQ(mock_->launch_success_count(), 1); + + // Second call is a no-op. + Updater::Instance().ReportLaunchSuccess(); + EXPECT_EQ(mock_->launch_success_count(), 1); } -TEST_F(UpdaterTest, MockUpdaterTracksLaunchFailureCalls) { +TEST_F(UpdaterTest, ReportLaunchFailureOnlyCallsOnce) { EXPECT_EQ(mock_->launch_failure_count(), 0); Updater::Instance().ReportLaunchFailure(); EXPECT_EQ(mock_->launch_failure_count(), 1); + + // Second call is a no-op. + Updater::Instance().ReportLaunchFailure(); + EXPECT_EQ(mock_->launch_failure_count(), 1); } TEST_F(UpdaterTest, MockUpdaterTracksShouldAutoUpdate) { @@ -98,9 +112,9 @@ TEST_F(UpdaterTest, MockUpdaterResetClearsState) { EXPECT_FALSE(mock_->ShouldAutoUpdate()); } -// ReportLaunchStart and ReportLaunchSuccess are always paired per shell. +// ReportLaunchStart and ReportLaunchSuccess are paired once per process. // The Rust updater no-ops both when no patch is booting. -TEST_F(UpdaterTest, LaunchStartAndSuccessAreAlwaysPaired) { +TEST_F(UpdaterTest, LaunchStartAndSuccessArePairedOncePerProcess) { Updater::Instance().ReportLaunchStart(); Updater::Instance().ReportLaunchSuccess(); @@ -112,8 +126,8 @@ TEST_F(UpdaterTest, LaunchStartAndSuccessAreAlwaysPaired) { EXPECT_EQ(log[1], "ReportLaunchSuccess"); } -// ReportLaunchStart and ReportLaunchFailure are paired on failed boots. -TEST_F(UpdaterTest, LaunchStartAndFailureAreAlwaysPaired) { +// ReportLaunchStart and ReportLaunchFailure are paired once per process. +TEST_F(UpdaterTest, LaunchStartAndFailureArePairedOncePerProcess) { Updater::Instance().ReportLaunchStart(); Updater::Instance().ReportLaunchFailure(); @@ -125,6 +139,45 @@ TEST_F(UpdaterTest, LaunchStartAndFailureAreAlwaysPaired) { EXPECT_EQ(log[1], "ReportLaunchFailure"); } +// Simulates the add-to-app scenario: multiple engines call ReportLaunchStart +// and ReportLaunchSuccess, but only the first should actually reach the +// updater. This prevents the Rust updater from promoting a newly-downloaded +// patch to "current_boot" when subsequent engines are still running the +// original snapshot. +TEST_F(UpdaterTest, MultipleEnginesOnlyReportOnce) { + // First engine boots. + Updater::Instance().ReportLaunchStart(); + Updater::Instance().ReportLaunchSuccess(); + + // Second engine boots โ€” these should be no-ops. + Updater::Instance().ReportLaunchStart(); + Updater::Instance().ReportLaunchSuccess(); + + EXPECT_EQ(mock_->launch_start_count(), 1); + EXPECT_EQ(mock_->launch_success_count(), 1); + + const auto& log = mock_->call_log(); + ASSERT_EQ(log.size(), 2u); + EXPECT_EQ(log[0], "ReportLaunchStart"); + EXPECT_EQ(log[1], "ReportLaunchSuccess"); +} + +// ResetLaunchStateForTesting re-enables the guards, allowing tests to +// verify launch calls on a fresh state. +TEST_F(UpdaterTest, ResetLaunchStateReenablesGuards) { + Updater::Instance().ReportLaunchStart(); + Updater::Instance().ReportLaunchSuccess(); + EXPECT_EQ(mock_->launch_start_count(), 1); + EXPECT_EQ(mock_->launch_success_count(), 1); + + Updater::ResetLaunchStateForTesting(); + + Updater::Instance().ReportLaunchStart(); + Updater::Instance().ReportLaunchSuccess(); + EXPECT_EQ(mock_->launch_start_count(), 2); + EXPECT_EQ(mock_->launch_success_count(), 2); +} + } // namespace testing } // namespace shorebird } // namespace flutter From ec91878eeea7802e15802fa09a05e0738fc07d51 Mon Sep 17 00:00:00 2001 From: Eric Seidel Date: Mon, 2 Feb 2026 16:56:02 -0800 Subject: [PATCH 22/95] feat: show logs when running shorebird_tests (#106) --- .../shorebird_tests/test/shorebird_tests.dart | 46 ++++++++++++++++++- 1 file changed, 44 insertions(+), 2 deletions(-) diff --git a/packages/shorebird_tests/test/shorebird_tests.dart b/packages/shorebird_tests/test/shorebird_tests.dart index 4bead2de4503e..e332d8714d34a 100644 --- a/packages/shorebird_tests/test/shorebird_tests.dart +++ b/packages/shorebird_tests/test/shorebird_tests.dart @@ -1,4 +1,5 @@ import 'dart:async'; +import 'dart:convert'; import 'dart:io'; import 'package:archive/archive_io.dart'; @@ -23,12 +24,19 @@ File get _flutterBinaryFile => File( ); /// Runs a flutter command using the correct binary ([_flutterBinaryFile]) with the given arguments. +/// +/// Streams stdout and stderr to the test output in real time so that +/// CI logs show progress even if the process hangs or times out. Future _runFlutterCommand( List arguments, { required Directory workingDirectory, Map? environment, -}) { - return Process.run( +}) async { + final String command = 'flutter ${arguments.join(' ')}'; + print('[$command] starting...'); + final stopwatch = Stopwatch()..start(); + + final Process process = await Process.start( _flutterBinaryFile.absolute.path, arguments, workingDirectory: workingDirectory.path, @@ -37,6 +45,40 @@ Future _runFlutterCommand( if (environment != null) ...environment, }, ); + + final StringBuffer stdoutBuffer = StringBuffer(); + final StringBuffer stderrBuffer = StringBuffer(); + + process.stdout.transform(utf8.decoder).listen((String data) { + stdoutBuffer.write(data); + // Print each line with a prefix so it's easy to identify in CI logs. + for (final String line in data.split('\n')) { + if (line.isNotEmpty) { + print(' [$command] $line'); + } + } + }); + + process.stderr.transform(utf8.decoder).listen((String data) { + stderrBuffer.write(data); + for (final String line in data.split('\n')) { + if (line.isNotEmpty) { + print(' [$command] (stderr) $line'); + } + } + }); + + final int exitCode = await process.exitCode; + stopwatch.stop(); + print('[$command] completed in ${stopwatch.elapsed} ' + '(exit code $exitCode)'); + + return ProcessResult( + process.pid, + exitCode, + stdoutBuffer.toString(), + stderrBuffer.toString(), + ); } Future _createFlutterProject(Directory projectDirectory) async { From f6033015885e6e41bdb63bd8673478221f3384f1 Mon Sep 17 00:00:00 2001 From: Eric Seidel Date: Mon, 2 Feb 2026 21:07:55 -0800 Subject: [PATCH 23/95] perf: speed up shorebird_tests with template project and Gradle cache (#108) - Create a template Flutter project once in setUpAll and copy it per test, avoiding repeated `flutter create` calls - Run a warm-up `flutter build apk` in setUpAll (outside per-test timeout) to prime the Gradle cache - Add actions/cache for ~/.gradle so subsequent CI runs start warm - Add VERBOSE env var and failure output logging from #107 --- .github/workflows/shorebird_ci.yml | 11 ++ .../shorebird_tests/test/android_test.dart | 2 + packages/shorebird_tests/test/base_test.dart | 2 + packages/shorebird_tests/test/ios_test.dart | 2 + .../shorebird_tests/test/shorebird_tests.dart | 108 ++++++++++++++---- 5 files changed, 100 insertions(+), 25 deletions(-) diff --git a/.github/workflows/shorebird_ci.yml b/.github/workflows/shorebird_ci.yml index 9597d6a599344..690db437da585 100644 --- a/.github/workflows/shorebird_ci.yml +++ b/.github/workflows/shorebird_ci.yml @@ -38,6 +38,17 @@ jobs: distribution: "zulu" java-version: "17" + - name: ๐Ÿ“ฆ Cache Gradle + if: ${{ matrix.os == 'ubuntu-latest' || matrix.os == 'macos-latest' }} + uses: actions/cache@v4 + with: + path: | + ~/.gradle/caches + ~/.gradle/wrapper + key: ${{ runner.os }}-gradle-${{ hashFiles('**/*.gradle*', '**/gradle-wrapper.properties') }} + restore-keys: | + ${{ runner.os }}-gradle- + - name: ๐Ÿฆ Run Flutter Tools Tests # TODO(eseidel): Find a nice way to run this on windows. if: ${{ matrix.os == 'ubuntu-latest' || matrix.os == 'macos-latest' }} diff --git a/packages/shorebird_tests/test/android_test.dart b/packages/shorebird_tests/test/android_test.dart index b24cce0ee9bf6..1e0a6da4f3824 100644 --- a/packages/shorebird_tests/test/android_test.dart +++ b/packages/shorebird_tests/test/android_test.dart @@ -3,6 +3,8 @@ import 'package:test/test.dart'; import 'shorebird_tests.dart'; void main() { + setUpAll(warmUpTemplateProject); + group('shorebird android projects', () { testWithShorebirdProject('can build an apk', (projectDirectory) async { await projectDirectory.runFlutterBuildApk(); diff --git a/packages/shorebird_tests/test/base_test.dart b/packages/shorebird_tests/test/base_test.dart index 14dba7ca6cc85..ce9b0564b1396 100644 --- a/packages/shorebird_tests/test/base_test.dart +++ b/packages/shorebird_tests/test/base_test.dart @@ -3,6 +3,8 @@ import 'package:test/test.dart'; import 'shorebird_tests.dart'; void main() { + setUpAll(warmUpTemplateProject); + group('shorebird helpers', () { testWithShorebirdProject('can build a base project', (projectDirectory) async { diff --git a/packages/shorebird_tests/test/ios_test.dart b/packages/shorebird_tests/test/ios_test.dart index 3fe6e1652c3e5..bb694405ae946 100644 --- a/packages/shorebird_tests/test/ios_test.dart +++ b/packages/shorebird_tests/test/ios_test.dart @@ -3,6 +3,8 @@ import 'package:test/test.dart'; import 'shorebird_tests.dart'; void main() { + setUpAll(warmUpTemplateProject); + group( 'shorebird ios projects', () { diff --git a/packages/shorebird_tests/test/shorebird_tests.dart b/packages/shorebird_tests/test/shorebird_tests.dart index e332d8714d34a..4100c6c5aca50 100644 --- a/packages/shorebird_tests/test/shorebird_tests.dart +++ b/packages/shorebird_tests/test/shorebird_tests.dart @@ -91,46 +91,104 @@ Future _createFlutterProject(Directory projectDirectory) async { } } -@isTest -Future testWithShorebirdProject(String name, - FutureOr Function(Directory projectDirectory) testFn) async { - test( - name, - () async { - final parentDirectory = Directory.systemTemp.createTempSync(); - final projectDirectory = Directory( - path.join( - parentDirectory.path, - 'shorebird_test', - ), - )..createSync(); +/// Cached template project directory, created once and reused across tests. +/// +/// This avoids running `flutter create` for every test, which saves +/// significant time (especially the first Gradle/SDK download). +Directory? _templateProject; - try { - await _createFlutterProject(projectDirectory); +/// Creates (or returns the cached) template Flutter project with +/// shorebird.yaml configured. The first call runs `flutter create` and +/// `flutter build apk` to warm up Gradle caches. +/// +/// Call this from `setUpAll` so the expensive setup runs outside per-test +/// timeouts. +Future warmUpTemplateProject() => _getTemplateProject(); + +Future _getTemplateProject() async { + if (_templateProject != null) { + return _templateProject!; + } + + final Directory templateDir = Directory( + path.join(Directory.systemTemp.createTempSync().path, 'shorebird_template'), + )..createSync(); - projectDirectory.pubspecFile.writeAsStringSync(''' -${projectDirectory.pubspecFile.readAsStringSync()} + await _createFlutterProject(templateDir); + + templateDir.pubspecFile.writeAsStringSync(''' +${templateDir.pubspecFile.readAsStringSync()} assets: - shorebird.yaml '''); - File( - path.join( - projectDirectory.path, - 'shorebird.yaml', - ), - ).writeAsStringSync(''' + File( + path.join(templateDir.path, 'shorebird.yaml'), + ).writeAsStringSync(''' app_id: "123" '''); + // Warm up the Gradle cache with a throwaway build so subsequent + // per-test builds are fast and don't hit the per-test timeout. + // Skip if Gradle cache is already populated (e.g., from GHA cache restore). + final Directory gradleCache = Directory( + path.join(Platform.environment['HOME'] ?? '', '.gradle', 'caches'), + ); + final bool hasGradleCache = + gradleCache.existsSync() && gradleCache.listSync().isNotEmpty; + if (hasGradleCache) { + print('[warmup] Gradle cache exists, skipping warm-up build'); + } else { + await _runFlutterCommand( + ['build', 'apk'], + workingDirectory: templateDir, + ); + } + + _templateProject = templateDir; + return templateDir; +} + +/// Copies the template project to a fresh directory for test isolation. +Future _copyTemplateProject() async { + final Directory template = await _getTemplateProject(); + final Directory testDir = Directory( + path.join(Directory.systemTemp.createTempSync().path, 'shorebird_test'), + ); + + // Use platform copy to preserve the full directory tree efficiently. + if (Platform.isWindows) { + await Process.run('xcopy', [ + template.path, + testDir.path, + '/E', + '/I', + '/Q', + ]); + } else { + await Process.run('cp', ['-R', template.path, testDir.path]); + } + + return testDir; +} + +@isTest +Future testWithShorebirdProject(String name, + FutureOr Function(Directory projectDirectory) testFn) async { + test( + name, + () async { + final Directory projectDirectory = await _copyTemplateProject(); + + try { await testFn(projectDirectory); } finally { projectDirectory.deleteSync(recursive: true); } }, timeout: Timeout( - // These tests usually run flutter create, flutter build, etc, which can take a while, - // specially in CI, so setting from the default of 30 seconds to 6 minutes. + // Per-test timeout can be shorter now since the template project + // creation and Gradle warm-up happen outside the test timeout. Duration(minutes: 6), ), ); From a086528d36e22790d47c5b75661f7807cd60eadc Mon Sep 17 00:00:00 2001 From: Eric Seidel Date: Mon, 2 Feb 2026 21:31:24 -0800 Subject: [PATCH 24/95] fix: throw on copy failure in _copyTemplateProject (#109) --- packages/shorebird_tests/test/shorebird_tests.dart | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/packages/shorebird_tests/test/shorebird_tests.dart b/packages/shorebird_tests/test/shorebird_tests.dart index 4100c6c5aca50..409f3963e95e1 100644 --- a/packages/shorebird_tests/test/shorebird_tests.dart +++ b/packages/shorebird_tests/test/shorebird_tests.dart @@ -157,8 +157,9 @@ Future _copyTemplateProject() async { ); // Use platform copy to preserve the full directory tree efficiently. + final ProcessResult result; if (Platform.isWindows) { - await Process.run('xcopy', [ + result = await Process.run('xcopy', [ template.path, testDir.path, '/E', @@ -166,7 +167,10 @@ Future _copyTemplateProject() async { '/Q', ]); } else { - await Process.run('cp', ['-R', template.path, testDir.path]); + result = await Process.run('cp', ['-R', template.path, testDir.path]); + } + if (result.exitCode != 0) { + throw Exception('Failed to copy template project: ${result.stderr}'); } return testDir; From f5d0ed88beabedb8471aaf7244490fb38fc963bd Mon Sep 17 00:00:00 2001 From: Eric Seidel Date: Mon, 2 Feb 2026 22:28:57 -0800 Subject: [PATCH 25/95] chore: split CI into parallel jobs (#110) * chore: split CI into parallel jobs Split the single CI job into three parallel jobs: 1. flutter-tools-tests: Runs on ubuntu + macOS (unchanged) 2. shorebird-android-tests: Runs on Ubuntu only (faster runners) 3. shorebird-ios-tests: Runs on macOS only (requires Xcode) This improves CI performance by: - Running all jobs in parallel instead of sequentially - Moving Android tests off macOS to faster Ubuntu runners - Removing Windows from the matrix (nothing was running there anyway) Expected speedup on macOS: ~5 minutes (no longer runs Android tests) * Add Android smoke test on macOS Run a single "can build an apk" test on macOS to catch any platform-specific issues with Android builds on macOS. * Add comment explaining why Windows is excluded --- .github/workflows/shorebird_ci.yml | 83 ++++++++++++++++++++++++------ 1 file changed, 66 insertions(+), 17 deletions(-) diff --git a/.github/workflows/shorebird_ci.yml b/.github/workflows/shorebird_ci.yml index 690db437da585..7dbedfa88a2fa 100644 --- a/.github/workflows/shorebird_ci.yml +++ b/.github/workflows/shorebird_ci.yml @@ -11,25 +11,45 @@ on: - shorebird/dev jobs: - test: + # NOTE: Windows is not included because shorebird_tests depends on + # flutter_flavorizr which requires Xcode. Additionally, flutter_tools + # tests on Windows would need additional setup work. + + flutter-tools-tests: strategy: fail-fast: false matrix: - os: [ubuntu-latest, macos-latest, windows-latest] + os: [ubuntu-latest, macos-latest] runs-on: ${{ matrix.os }} - name: ๐Ÿฆ Shorebird Test + name: ๐Ÿ› ๏ธ Flutter Tools Tests (${{ matrix.os }}) + + steps: + - name: ๐Ÿ“š Git Checkout + uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - name: ๐ŸŽฏ Setup Dart + uses: dart-lang/setup-dart@v1 + + - name: ๐Ÿฆ Run Flutter Tools Tests + run: ../../bin/flutter test test/general.shard + working-directory: packages/flutter_tools + + shorebird-android-tests: + # Android tests run on Ubuntu (faster runners, no macOS needed) + runs-on: ubuntu-latest + + name: ๐Ÿค– Shorebird Android Tests steps: - name: ๐Ÿ“š Git Checkout uses: actions/checkout@v4 with: - # Fetch all branches and tags to ensure that Flutter can determine its version fetch-depth: 0 - # TODO(eseidel): shorebird_tests seems to assume flutter is available - # yet it doesn't seem to set it up here? - name: ๐ŸŽฏ Setup Dart uses: dart-lang/setup-dart@v1 @@ -39,7 +59,6 @@ jobs: java-version: "17" - name: ๐Ÿ“ฆ Cache Gradle - if: ${{ matrix.os == 'ubuntu-latest' || matrix.os == 'macos-latest' }} uses: actions/cache@v4 with: path: | @@ -49,15 +68,45 @@ jobs: restore-keys: | ${{ runner.os }}-gradle- - - name: ๐Ÿฆ Run Flutter Tools Tests - # TODO(eseidel): Find a nice way to run this on windows. - if: ${{ matrix.os == 'ubuntu-latest' || matrix.os == 'macos-latest' }} - run: ../../bin/flutter test test/general.shard - working-directory: packages/flutter_tools + - name: ๐Ÿค– Run Android Tests + run: dart test test/base_test.dart test/android_test.dart + working-directory: packages/shorebird_tests + + shorebird-ios-tests: + # iOS tests require macOS for Xcode + runs-on: macos-latest + + name: ๐ŸŽ Shorebird iOS + Android Smoke Tests + + steps: + - name: ๐Ÿ“š Git Checkout + uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - name: ๐ŸŽฏ Setup Dart + uses: dart-lang/setup-dart@v1 + + - uses: actions/setup-java@v4 + with: + distribution: "zulu" + java-version: "17" + + - name: ๐Ÿ“ฆ Cache Gradle + uses: actions/cache@v4 + with: + path: | + ~/.gradle/caches + ~/.gradle/wrapper + key: ${{ runner.os }}-gradle-${{ hashFiles('**/*.gradle*', '**/gradle-wrapper.properties') }} + restore-keys: | + ${{ runner.os }}-gradle- + + - name: ๐ŸŽ Run iOS Tests + run: dart test test/base_test.dart test/ios_test.dart + working-directory: packages/shorebird_tests - - name: ๐Ÿฆ Run Shorebird Tests - # TODO(felangel): These tests have a dependency on pkg:flutter_flavorizr which - # requires XCode -- therefore they don't work on Windows. - if: ${{ matrix.os == 'ubuntu-latest' || matrix.os == 'macos-latest' }} - run: dart test + - name: ๐Ÿค– Run Android Smoke Test (macOS) + # Quick sanity check that Android builds work on macOS too + run: dart test test/android_test.dart --name "can build an apk" working-directory: packages/shorebird_tests From 257a23b6011f8ee021bc4fb80578e6533d804495 Mon Sep 17 00:00:00 2001 From: Eric Seidel Date: Tue, 3 Feb 2026 20:40:35 -0800 Subject: [PATCH 26/95] feat: add sharded CI build runner (#111) * feat: add sharded CI build runner Adds a Dart-based shard runner for parallel engine builds: - JSON configs for Linux, macOS, Windows shards - run_shard.dart: executes gn/ninja/rust builds - compose.dart: assembles iOS/macOS frameworks - GCS upload/download for artifact staging * refactor: parse compose.json into typed objects Move from reading compose.json directly as Map to using proper ComposeConfig and ComposeDef model classes. This follows a more idiomatic Dart pattern. * feat: add finalize.dart for manifest generation and uploads Implements the finalize job logic that: - Downloads artifacts from GCS staging - Generates artifacts_manifest.yaml - Uploads to production GCS bucket (download.shorebird.dev) Ports upload logic from linux_upload.sh, mac_upload.sh, and generate_manifest.sh into a single Dart script. * fix: correct cargo-ndk invocation for Android Rust builds - Use --target flag (not -t) with Rust target triples - Set ANDROID_NDK_HOME to engine's hermetic NDK - Build all Android targets in a single cargo ndk command - Remove incorrect _androidArch helper function This matches the behavior of linux_build.sh. * test: add unit tests for config parsing Tests cover: - PlatformConfig: single-step shards, multi-step shards, compose_input - BuildStep: gn_ninja and rust step types - ComposeConfig: compose definitions, requires, script, args - Error handling for unknown shards/compose names * feat: add artifacts field to shard configs Define artifacts declaratively in JSON configs instead of hardcoding upload paths in Dart. Each artifact specifies: - src: source path relative to out/ - dst: destination path with $engine placeholder - zip: whether to zip before upload (for directories like dart-sdk) - content_hash: whether to also upload to content-hash path (for Dart SDK) This makes the config self-describing and aligns with Flutter's ci/builders/*.json pattern of explicit artifact declarations. * refactor: read shard names from JSON configs instead of hardcoding Load PlatformConfig for each platform to get shard names dynamically, rather than maintaining a duplicate list in finalize.dart. * feat(ci): add manifest generation and bucket configuration - Extract generateManifest to lib/manifest.dart with tests - Refactor finalize.dart to use artifacts from JSON configs - Add --bucket flag for test uploads to alternate buckets - Add compare_buckets.dart for validating uploads against production * chore: add pubspec.lock for shard_runner * chore: allow shard_runner pubspec.lock in gitignore * chore: use local .gitignore for shard_runner pubspec.lock * refactor: load manifest from template file Move manifest content to artifacts_manifest.template.yaml and update generateManifest to load from template file with sync IO. * fix: fail finalize on download errors instead of continuing A missing shard download means incomplete artifacts. Better to fail loudly than silently upload an incomplete build. * fix: fail on gsutil/zip errors instead of warning Upload and zip failures should halt the build, not silently continue with missing artifacts. * refactor: clean up shard_runner CLI and config parsing - Add shared runChecked() helper to eliminate duplicated Process.run + exit code check patterns across config.dart, gcs.dart, finalize.dart, and compose.dart - Add @immutable annotations to all data classes (via package:meta) - Remove implicit single-step shard shorthand; all shards now use explicit steps arrays in JSON configs - Convert all async file IO to sync equivalents (existsSync, etc.) - Make --engine-src and --run-id mandatory CLI args, removing hidden defaults and GITHUB_RUN_ID env var fallback - Restructure compose.json to use explicit flags/path_args instead of a single args list that guessed flag vs path semantics - Collect outDirs from config upfront rather than accumulating during execution * ci: add shard_runner tests to shorebird_ci workflow - Add analysis_options.yaml (package:lints/recommended with strict mode) - Add shard-runner-tests job with format, analyze, and test steps - Fix stale await on sync PlatformConfig.load in compare_buckets.dart - Reformat all files to Dart standard (80 char width) --- .github/workflows/shorebird_ci.yml | 28 ++ shorebird/ci/artifacts_manifest.template.yaml | 65 +++ shorebird/ci/compose.json | 30 ++ shorebird/ci/shard_runner/.gitignore | 2 + .../ci/shard_runner/analysis_options.yaml | 7 + .../ci/shard_runner/bin/compare_buckets.dart | 158 +++++++ shorebird/ci/shard_runner/bin/compose.dart | 104 +++++ shorebird/ci/shard_runner/bin/finalize.dart | 207 ++++++++++ shorebird/ci/shard_runner/bin/run_shard.dart | 84 ++++ shorebird/ci/shard_runner/lib/cli.dart | 83 ++++ .../ci/shard_runner/lib/compose_config.dart | 76 ++++ shorebird/ci/shard_runner/lib/config.dart | 252 ++++++++++++ shorebird/ci/shard_runner/lib/gcs.dart | 112 +++++ shorebird/ci/shard_runner/lib/manifest.dart | 26 ++ shorebird/ci/shard_runner/lib/process.dart | 36 ++ shorebird/ci/shard_runner/pubspec.lock | 389 ++++++++++++++++++ shorebird/ci/shard_runner/pubspec.yaml | 17 + .../test/compose_config_test.dart | 119 ++++++ .../ci/shard_runner/test/config_test.dart | 264 ++++++++++++ .../ci/shard_runner/test/manifest_test.dart | 208 ++++++++++ shorebird/ci/shards/linux.json | 92 +++++ shorebird/ci/shards/macos.json | 145 +++++++ shorebird/ci/shards/windows.json | 66 +++ 23 files changed, 2570 insertions(+) create mode 100644 shorebird/ci/artifacts_manifest.template.yaml create mode 100644 shorebird/ci/compose.json create mode 100644 shorebird/ci/shard_runner/.gitignore create mode 100644 shorebird/ci/shard_runner/analysis_options.yaml create mode 100644 shorebird/ci/shard_runner/bin/compare_buckets.dart create mode 100644 shorebird/ci/shard_runner/bin/compose.dart create mode 100644 shorebird/ci/shard_runner/bin/finalize.dart create mode 100644 shorebird/ci/shard_runner/bin/run_shard.dart create mode 100644 shorebird/ci/shard_runner/lib/cli.dart create mode 100644 shorebird/ci/shard_runner/lib/compose_config.dart create mode 100644 shorebird/ci/shard_runner/lib/config.dart create mode 100644 shorebird/ci/shard_runner/lib/gcs.dart create mode 100644 shorebird/ci/shard_runner/lib/manifest.dart create mode 100644 shorebird/ci/shard_runner/lib/process.dart create mode 100644 shorebird/ci/shard_runner/pubspec.lock create mode 100644 shorebird/ci/shard_runner/pubspec.yaml create mode 100644 shorebird/ci/shard_runner/test/compose_config_test.dart create mode 100644 shorebird/ci/shard_runner/test/config_test.dart create mode 100644 shorebird/ci/shard_runner/test/manifest_test.dart create mode 100644 shorebird/ci/shards/linux.json create mode 100644 shorebird/ci/shards/macos.json create mode 100644 shorebird/ci/shards/windows.json diff --git a/.github/workflows/shorebird_ci.yml b/.github/workflows/shorebird_ci.yml index 7dbedfa88a2fa..83e1511d102de 100644 --- a/.github/workflows/shorebird_ci.yml +++ b/.github/workflows/shorebird_ci.yml @@ -38,6 +38,34 @@ jobs: run: ../../bin/flutter test test/general.shard working-directory: packages/flutter_tools + shard-runner-tests: + runs-on: ubuntu-latest + + name: ๐Ÿงฉ Shard Runner Tests + + defaults: + run: + working-directory: shorebird/ci/shard_runner + + steps: + - name: ๐Ÿ“š Git Checkout + uses: actions/checkout@v4 + + - name: ๐ŸŽฏ Setup Dart + uses: dart-lang/setup-dart@v1 + + - name: ๐Ÿ“ฆ Install Dependencies + run: dart pub get + + - name: ๐Ÿ” Check Formatting + run: dart format --output=none --set-exit-if-changed . + + - name: ๐Ÿ”Ž Analyze + run: dart analyze --fatal-infos + + - name: ๐Ÿงช Run Tests + run: dart test + shorebird-android-tests: # Android tests run on Ubuntu (faster runners, no macOS needed) runs-on: ubuntu-latest diff --git a/shorebird/ci/artifacts_manifest.template.yaml b/shorebird/ci/artifacts_manifest.template.yaml new file mode 100644 index 0000000000000..4b5c8a7f0809d --- /dev/null +++ b/shorebird/ci/artifacts_manifest.template.yaml @@ -0,0 +1,65 @@ +# Template for artifacts_manifest.yaml +# This file is processed by shard_runner:finalize +# Variable substitution: {{flutter_engine_revision}} is replaced at generation time +# The $engine placeholder is kept as-is for Shorebird's artifact proxy + +flutter_engine_revision: {{flutter_engine_revision}} +storage_bucket: download.shorebird.dev +artifact_overrides: + # Android release artifacts + - flutter_infra_release/flutter/$engine/android-arm64-release/artifacts.zip + - flutter_infra_release/flutter/$engine/android-arm64-release/darwin-x64.zip + - flutter_infra_release/flutter/$engine/android-arm64-release/linux-x64.zip + - flutter_infra_release/flutter/$engine/android-arm64-release/symbols.zip + - flutter_infra_release/flutter/$engine/android-arm64-release/windows-x64.zip + + - flutter_infra_release/flutter/$engine/android-arm-release/artifacts.zip + - flutter_infra_release/flutter/$engine/android-arm-release/darwin-x64.zip + - flutter_infra_release/flutter/$engine/android-arm-release/linux-x64.zip + - flutter_infra_release/flutter/$engine/android-arm-release/symbols.zip + - flutter_infra_release/flutter/$engine/android-arm-release/windows-x64.zip + + - flutter_infra_release/flutter/$engine/android-x64-release/artifacts.zip + - flutter_infra_release/flutter/$engine/android-x64-release/darwin-x64.zip + - flutter_infra_release/flutter/$engine/android-x64-release/linux-x64.zip + - flutter_infra_release/flutter/$engine/android-x64-release/symbols.zip + - flutter_infra_release/flutter/$engine/android-x64-release/windows-x64.zip + + # engine_stamp.json + - flutter_infra_release/flutter/$engine/engine_stamp.json + + # Dart SDK + - flutter_infra_release/flutter/$engine/dart-sdk-darwin-arm64.zip + - flutter_infra_release/flutter/$engine/dart-sdk-darwin-x64.zip + - flutter_infra_release/flutter/$engine/dart-sdk-linux-x64.zip + - flutter_infra_release/flutter/$engine/dart-sdk-windows-x64.zip + + # Maven artifacts + - download.flutter.io/io/flutter/flutter_embedding_release/1.0.0-$engine/flutter_embedding_release-1.0.0-$engine.pom + - download.flutter.io/io/flutter/flutter_embedding_release/1.0.0-$engine/flutter_embedding_release-1.0.0-$engine.jar + - download.flutter.io/io/flutter/arm64_v8a_release/1.0.0-$engine/arm64_v8a_release-1.0.0-$engine.pom + - download.flutter.io/io/flutter/arm64_v8a_release/1.0.0-$engine/arm64_v8a_release-1.0.0-$engine.jar + - download.flutter.io/io/flutter/armeabi_v7a_release/1.0.0-$engine/armeabi_v7a_release-1.0.0-$engine.pom + - download.flutter.io/io/flutter/armeabi_v7a_release/1.0.0-$engine/armeabi_v7a_release-1.0.0-$engine.jar + - download.flutter.io/io/flutter/x86_64_release/1.0.0-$engine/x86_64_release-1.0.0-$engine.pom + - download.flutter.io/io/flutter/x86_64_release/1.0.0-$engine/x86_64_release-1.0.0-$engine.jar + + # Common release artifacts + - flutter_infra_release/flutter/$engine/flutter_patched_sdk_product.zip + + # iOS release artifacts + - flutter_infra_release/flutter/$engine/ios-release/artifacts.zip + - flutter_infra_release/flutter/$engine/ios-release/Flutter.dSYM.zip + + # Linux release artifacts + - flutter_infra_release/flutter/$engine/linux-x64/artifacts.zip + - flutter_infra_release/flutter/$engine/linux-x64-release/linux-x64-flutter-gtk.zip + + # macOS release artifacts + - flutter_infra_release/flutter/$engine/darwin-x64-release/artifacts.zip + - flutter_infra_release/flutter/$engine/darwin-x64-release/framework.zip + - flutter_infra_release/flutter/$engine/darwin-x64-release/gen_snapshot.zip + + # Windows release artifacts + - flutter_infra_release/flutter/$engine/windows-x64/artifacts.zip + - flutter_infra_release/flutter/$engine/windows-x64-release/windows-x64-flutter.zip diff --git a/shorebird/ci/compose.json b/shorebird/ci/compose.json new file mode 100644 index 0000000000000..1f6ca744d1e5d --- /dev/null +++ b/shorebird/ci/compose.json @@ -0,0 +1,30 @@ +{ + "ios-framework": { + "requires": ["ios-release", "ios-release-ext", "ios-sim-x64", "ios-sim-x64-ext", "ios-sim-arm64", "ios-sim-arm64-ext"], + "script": "flutter/sky/tools/create_ios_framework.py", + "flags": ["--dsym", "--strip"], + "path_args": { + "--arm64-out-dir": "ios_release", + "--simulator-x64-out-dir": "ios_debug_sim", + "--simulator-arm64-out-dir": "ios_debug_sim_arm64" + } + }, + "macos-framework": { + "requires": ["mac-arm64", "mac-x64"], + "script": "flutter/sky/tools/create_macos_framework.py", + "flags": ["--dsym", "--strip", "--zip"], + "path_args": { + "--arm64-out-dir": "mac_release_arm64", + "--x64-out-dir": "mac_release" + } + }, + "macos-gen-snapshot": { + "requires": ["mac-arm64", "mac-x64"], + "script": "flutter/sky/tools/create_macos_gen_snapshots.py", + "flags": ["--zip"], + "path_args": { + "--arm64-path": "mac_release_arm64/universal/gen_snapshot_arm64", + "--x64-path": "mac_release/universal/gen_snapshot_x64" + } + } +} diff --git a/shorebird/ci/shard_runner/.gitignore b/shorebird/ci/shard_runner/.gitignore new file mode 100644 index 0000000000000..929f3d7e632c0 --- /dev/null +++ b/shorebird/ci/shard_runner/.gitignore @@ -0,0 +1,2 @@ +# Override root .gitignore to track our lockfile +!pubspec.lock diff --git a/shorebird/ci/shard_runner/analysis_options.yaml b/shorebird/ci/shard_runner/analysis_options.yaml new file mode 100644 index 0000000000000..d04adaf90938a --- /dev/null +++ b/shorebird/ci/shard_runner/analysis_options.yaml @@ -0,0 +1,7 @@ +include: package:lints/recommended.yaml + +analyzer: + language: + strict-casts: true + strict-inference: true + strict-raw-types: true diff --git a/shorebird/ci/shard_runner/bin/compare_buckets.dart b/shorebird/ci/shard_runner/bin/compare_buckets.dart new file mode 100644 index 0000000000000..958c238bb0230 --- /dev/null +++ b/shorebird/ci/shard_runner/bin/compare_buckets.dart @@ -0,0 +1,158 @@ +import 'dart:io'; + +import 'package:args/args.dart'; +import 'package:path/path.dart' as p; +import 'package:shard_runner/config.dart'; + +/// Compares artifacts between two GCS buckets for a given engine revision. +/// +/// Usage: dart run shard_runner:compare_buckets [options] +/// +/// Example: +/// dart run shard_runner:compare_buckets \ +/// --engine-revision abc123 \ +/// --test-bucket shorebird-build-test \ +/// --production-bucket download.shorebird.dev +Future main(List args) async { + final ArgParser parser = ArgParser() + ..addOption('engine-revision', + abbr: 'r', help: 'Engine revision (git hash)', mandatory: true) + ..addOption('test-bucket', + abbr: 't', help: 'Test bucket to compare', mandatory: true) + ..addOption('production-bucket', + abbr: 'p', + help: 'Production bucket (default: download.shorebird.dev)', + defaultsTo: 'download.shorebird.dev') + ..addOption('config-dir', + abbr: 'c', help: 'Config directory containing shards/*.json') + ..addFlag('verbose', abbr: 'v', help: 'Show detailed output') + ..addFlag('help', abbr: 'h', negatable: false, help: 'Show this help'); + + final ArgResults results = parser.parse(args); + + if (results['help'] as bool) { + print('Usage: dart run shard_runner:compare_buckets [options]'); + print(''); + print('Compares artifacts between test and production GCS buckets.'); + print('Uses gsutil hash to compare file checksums (MD5 + CRC32C).'); + print(''); + print(parser.usage); + exit(0); + } + + final String engineRevision = results['engine-revision'] as String; + final String testBucket = results['test-bucket'] as String; + final String productionBucket = results['production-bucket'] as String; + final String? configDirPath = results['config-dir'] as String?; + final bool verbose = results['verbose'] as bool; + + // Find config directory + final String configDir = configDirPath ?? + p.join(p.dirname(p.dirname(Platform.script.toFilePath())), 'shards'); + + print('=' * 60); + print('Compare Buckets'); + print('=' * 60); + print('Engine revision: $engineRevision'); + print('Test bucket: $testBucket'); + print('Production bucket: $productionBucket'); + print('Config dir: $configDir'); + print(''); + + // Load configs to get artifact paths + const List platforms = ['linux', 'macos', 'windows']; + final Map configs = {}; + for (final String platform in platforms) { + configs[platform] = PlatformConfig.load(platform, configDir); + } + + // Collect all artifact paths + final List artifacts = []; + for (final PlatformConfig config in configs.values) { + for (final ShardDef shard in config.shards.values) { + for (final ArtifactDef artifact in shard.artifacts) { + final String dstPath = + artifact.dst.replaceAll(r'$engine', engineRevision); + artifacts.add(dstPath); + } + } + } + + // Add manifest + artifacts.add('shorebird/$engineRevision/artifacts_manifest.yaml'); + + print('Comparing ${artifacts.length} artifacts...\n'); + + int matches = 0; + int mismatches = 0; + int missing = 0; + + for (final String artifact in artifacts) { + final String testUri = 'gs://$testBucket/$artifact'; + final String prodUri = 'gs://$productionBucket/$artifact'; + + // Get hash from test bucket + final String? testHash = await _getHash(testUri); + if (testHash == null) { + if (verbose) print('[MISSING] $artifact (not in test bucket)'); + missing++; + continue; + } + + // Get hash from production bucket + final String? prodHash = await _getHash(prodUri); + if (prodHash == null) { + if (verbose) print('[MISSING] $artifact (not in production bucket)'); + missing++; + continue; + } + + // Compare hashes + if (testHash == prodHash) { + if (verbose) print('[OK] $artifact'); + matches++; + } else { + print('[MISMATCH] $artifact'); + print(' Test: $testHash'); + print(' Prod: $prodHash'); + mismatches++; + } + } + + print(''); + print('=' * 60); + print('Results:'); + print(' Matches: $matches'); + print(' Mismatches: $mismatches'); + print(' Missing: $missing'); + print('=' * 60); + + if (mismatches > 0) { + print('\nWARNING: Found $mismatches mismatched artifacts!'); + exit(1); + } else if (missing > 0) { + print('\nWARNING: Found $missing missing artifacts.'); + exit(2); + } else { + print('\nSUCCESS: All artifacts match!'); + } +} + +/// Gets the MD5 hash of a GCS object using gsutil hash. +Future _getHash(String uri) async { + final ProcessResult result = + await Process.run('gsutil', ['hash', uri]); + if (result.exitCode != 0) { + return null; + } + + // Parse output for MD5 hash + // Example output: + // Hashes [hex] for gs://bucket/path: + // Hash (crc32c): abc123== + // Hash (md5): xyz789== + final String output = result.stdout as String; + final RegExpMatch? md5Match = + RegExp(r'Hash \(md5\):\s+(\S+)').firstMatch(output); + return md5Match?.group(1); +} diff --git a/shorebird/ci/shard_runner/bin/compose.dart b/shorebird/ci/shard_runner/bin/compose.dart new file mode 100644 index 0000000000000..73cdb44393bb7 --- /dev/null +++ b/shorebird/ci/shard_runner/bin/compose.dart @@ -0,0 +1,104 @@ +import 'dart:io'; + +import 'package:args/args.dart'; +import 'package:path/path.dart' as p; +import 'package:shard_runner/cli.dart'; +import 'package:shard_runner/compose_config.dart'; +import 'package:shard_runner/gcs.dart'; +import 'package:shard_runner/process.dart'; + +/// Composes artifacts from multiple shards into final outputs. +/// +/// Usage: dart run shard_runner:compose [options] +/// +/// Example: +/// dart run shard_runner:compose ios-framework --engine-src ~/.engine_checkout/engine/src +Future main(List args) async { + final ArgParser parser = ArgParser(); + CliConfig.addCommonOptions(parser, includeUpload: false); + parser.addFlag('download', + defaultsTo: true, help: 'Download artifacts from GCS staging'); + + final ArgResults results = parser.parse(args); + + if (results['help'] as bool || results.rest.isEmpty) { + print('Usage: dart run shard_runner:compose [options]'); + print(''); + print('Compose names: ios-framework, macos-framework, macos-gen-snapshot'); + print(''); + print(parser.usage); + exit(results['help'] as bool ? 0 : 1); + } + + final String composeName = results.rest[0]; + final CliConfig cli = + CliConfig.fromArgs(results, scriptPath: Platform.script.toFilePath()); + final bool shouldDownload = results['download'] as bool; + + cli.printHeader('Compose Runner', { + 'Compose:': composeName, + 'Download:': shouldDownload.toString(), + }); + + // Load compose config + final ComposeConfig config; + try { + config = ComposeConfig.load(cli.configDir); + } on FileSystemException catch (e) { + print('Error: ${e.message} at ${e.path}'); + exit(1); + } + + final ComposeDef composeDef; + try { + composeDef = config.getCompose(composeName); + } on ArgumentError catch (e) { + print('Error: ${e.message}'); + exit(1); + } + + print('\n[Compose] Requires shards: ${composeDef.requires.join(', ')}'); + + // Download artifacts from each required shard + if (shouldDownload) { + for (final String shard in composeDef.requires) { + print('\n[Download] Fetching $shard artifacts...'); + await downloadFromStaging( + runId: cli.runId, + platform: 'macos', // Compose only runs for macOS currently + shard: shard, + destDir: p.join(cli.engineSrc, 'out'), + ); + } + } + + // Build script arguments + final String outDir = p.join(cli.engineSrc, 'out', 'release'); + + // Ensure output directory exists + Directory(outDir).createSync(recursive: true); + + // Build script arguments: expand path_args to absolute paths, pass flags as-is. + final List expandedArgs = ['--dst', outDir]; + for (final MapEntry entry in composeDef.pathArgs.entries) { + expandedArgs + .addAll([entry.key, p.join(cli.engineSrc, 'out', entry.value)]); + } + expandedArgs.addAll(composeDef.flags); + + // Run the composition script + print('\n[Compose] Running ${composeDef.script}...'); + print('[Compose] Args: ${expandedArgs.join(' ')}'); + + await runChecked( + 'python3', + [p.join(cli.engineSrc, composeDef.script), ...expandedArgs], + workingDirectory: cli.engineSrc, + description: 'Compose ($composeName)', + ); + + print('[Compose] Complete'); + print('\n${'='.padRight(60, '=')}'); + print('Compose $composeName completed successfully'); + print('='.padRight(60, '=')); +} diff --git a/shorebird/ci/shard_runner/bin/finalize.dart b/shorebird/ci/shard_runner/bin/finalize.dart new file mode 100644 index 0000000000000..5a6dae8a923ee --- /dev/null +++ b/shorebird/ci/shard_runner/bin/finalize.dart @@ -0,0 +1,207 @@ +import 'dart:io'; + +import 'package:args/args.dart'; +import 'package:path/path.dart' as p; +import 'package:shard_runner/cli.dart'; +import 'package:shard_runner/config.dart'; +import 'package:shard_runner/gcs.dart'; +import 'package:shard_runner/manifest.dart'; +import 'package:shard_runner/process.dart'; + +/// Finalizes a sharded build by generating manifest and uploading artifacts. +/// +/// Usage: dart run shard_runner:finalize [options] +/// +/// Example: +/// dart run shard_runner:finalize --engine-revision abc123 +Future main(List args) async { + final ArgParser parser = ArgParser() + ..addOption('engine-revision', + abbr: 'r', help: 'Engine revision (git hash)', mandatory: true) + ..addOption('base-engine-revision', + help: 'Base Flutter engine revision for manifest') + ..addOption('content-hash', help: 'Content-aware hash for Dart SDK') + ..addOption('bucket', + abbr: 'b', + help: 'GCS bucket for uploads (default: download.shorebird.dev)', + defaultsTo: 'download.shorebird.dev') + ..addFlag('download', + defaultsTo: true, help: 'Download artifacts from GCS staging') + ..addFlag('upload', + defaultsTo: true, help: 'Upload artifacts to GCS bucket') + ..addFlag('help', abbr: 'h', negatable: false, help: 'Show this help'); + + CliConfig.addCommonOptions(parser, includeUpload: false); + + final ArgResults results = parser.parse(args); + + if (results['help'] as bool) { + print('Usage: dart run shard_runner:finalize [options]'); + print(''); + print(parser.usage); + exit(0); + } + + final CliConfig cli = + CliConfig.fromArgs(results, scriptPath: Platform.script.toFilePath()); + final String engineRevision = results['engine-revision'] as String; + final String baseEngineRevision = + results['base-engine-revision'] as String? ?? engineRevision; + final String? contentHash = results['content-hash'] as String?; + final String bucket = results['bucket'] as String; + final bool shouldDownload = results['download'] as bool; + final bool shouldUpload = results['upload'] as bool; + + cli.printHeader('Finalize Build', { + 'Engine:': engineRevision, + 'Base Engine:': baseEngineRevision, + 'Bucket:': bucket, + 'Download:': shouldDownload.toString(), + 'Upload:': shouldUpload.toString(), + }); + + // Load shard configs for each platform + const List platforms = ['linux', 'macos', 'windows']; + final Map configs = {}; + for (final String platform in platforms) { + configs[platform] = PlatformConfig.load(platform, cli.configDir); + } + + // Download all artifacts from staging + if (shouldDownload) { + final String outDir = p.join(cli.engineSrc, 'out'); + Directory(outDir).createSync(recursive: true); + + for (final String platform in platforms) { + final PlatformConfig config = configs[platform]!; + for (final String shardName in config.shards.keys) { + print('\n[Download] Fetching $platform/$shardName...'); + await downloadFromStaging( + runId: cli.runId, + platform: platform, + shard: shardName, + destDir: outDir, + ); + } + } + } + + // Generate manifest + print('\n[Manifest] Generating artifacts_manifest.yaml...'); + final String manifest = + generateManifest(baseEngineRevision, configDir: cli.configDir); + final File manifestFile = + File(p.join(cli.engineSrc, 'artifacts_manifest.yaml')); + manifestFile.writeAsStringSync(manifest); + print('[Manifest] Written to ${manifestFile.path}'); + + // Upload to production + if (shouldUpload) { + print('\n[Upload] Uploading to $bucket...'); + await uploadToProduction( + engineSrc: cli.engineSrc, + engineRevision: engineRevision, + contentHash: contentHash, + configs: configs, + bucket: bucket, + ); + } + + print('\n${'=' * 60}'); + print('Finalize completed successfully'); + print('=' * 60); +} + +/// Production storage bucket name (without gs:// prefix). +const String productionBucket = 'download.shorebird.dev'; + +/// Uploads artifacts to a GCS bucket based on config definitions. +Future uploadToProduction({ + required String engineSrc, + required String engineRevision, + required String? contentHash, + required Map configs, + required String bucket, +}) async { + final String outDir = p.join(engineSrc, 'out'); + final String bucketUri = 'gs://$bucket'; + + // Helper to run gsutil cp + Future gscp(String src, String dest) async { + print('[Upload] $src -> $dest'); + await runChecked('gsutil', ['cp', src, dest], + description: 'gsutil cp $src'); + } + + // Helper to zip a directory and upload + Future zipAndUpload(String srcPath, String dest) async { + final String tempZip = '$srcPath.zip'; + print('[Zip] Creating $tempZip...'); + await runChecked( + 'zip', + ['-r', tempZip, '.'], + workingDirectory: srcPath, + description: 'zip $tempZip', + ); + await gscp(tempZip, dest); + File(tempZip).deleteSync(); + } + + // Process artifacts from all configs + for (final MapEntry entry in configs.entries) { + final String platform = entry.key; + final PlatformConfig config = entry.value; + + for (final MapEntry shardEntry in config.shards.entries) { + final String shardName = shardEntry.key; + final ShardDef shard = shardEntry.value; + + print('\n[Upload] Processing $platform/$shardName...'); + + for (final ArtifactDef artifact in shard.artifacts) { + // Resolve source path + final String srcPath = p.join(outDir, artifact.src); + + // Resolve destination path (replace $engine with actual revision) + final String dstPath = + artifact.dst.replaceAll(r'$engine', engineRevision); + final String fullDest = '$bucketUri/$dstPath'; + + // Check if source exists + final File srcFile = File(srcPath); + final Directory srcDir = Directory(srcPath); + final bool srcExists = srcFile.existsSync() || srcDir.existsSync(); + + if (!srcExists) { + print('[Skip] $srcPath (not found)'); + continue; + } + + // Handle zip flag + if (artifact.zip && srcDir.existsSync()) { + await zipAndUpload(srcPath, fullDest); + } else { + await gscp(srcPath, fullDest); + } + + // Handle content-hash uploads (for Dart SDK) + if (artifact.contentHash && contentHash != null) { + final String contentDstPath = + artifact.dst.replaceAll(r'$engine', contentHash); + final String contentFullDest = '$bucketUri/$contentDstPath'; + await gscp(srcPath, contentFullDest); + } + } + } + } + + // Upload manifest + final String manifestFile = p.join(engineSrc, 'artifacts_manifest.yaml'); + if (File(manifestFile).existsSync()) { + final String manifestDest = + '$bucketUri/shorebird/$engineRevision/artifacts_manifest.yaml'; + await gscp(manifestFile, manifestDest); + } + + print('\n[Upload] Production upload complete'); +} diff --git a/shorebird/ci/shard_runner/bin/run_shard.dart b/shorebird/ci/shard_runner/bin/run_shard.dart new file mode 100644 index 0000000000000..ed8c4cddac435 --- /dev/null +++ b/shorebird/ci/shard_runner/bin/run_shard.dart @@ -0,0 +1,84 @@ +import 'dart:io'; + +import 'package:args/args.dart'; +import 'package:shard_runner/cli.dart'; +import 'package:shard_runner/config.dart'; +import 'package:shard_runner/gcs.dart'; + +/// Runs a single build shard. +/// +/// Usage: dart run shard_runner:run_shard [options] +/// +/// Example: +/// dart run shard_runner:run_shard linux android-arm64 --engine-src ~/.engine_checkout/engine/src +Future main(List args) async { + final ArgParser parser = ArgParser(); + CliConfig.addCommonOptions(parser); + + final ArgResults results = parser.parse(args); + + if (results['help'] as bool || results.rest.length < 2) { + print( + 'Usage: dart run shard_runner:run_shard [options]'); + print(''); + print('Platforms: linux, macos, windows'); + print(''); + print(parser.usage); + exit(results['help'] as bool ? 0 : 1); + } + + final String platform = results.rest[0]; + final String shard = results.rest[1]; + final CliConfig cli = + CliConfig.fromArgs(results, scriptPath: Platform.script.toFilePath()); + + cli.printHeader('Shard Runner', { + 'Platform:': platform, + 'Shard:': shard, + 'Upload:': cli.shouldUpload.toString(), + }); + + cli.verifyEngineSrc(); + + // Load config + print('\n[Config] Loading $platform.json...'); + final PlatformConfig config = PlatformConfig.load(platform, cli.configDir); + final ShardDef shardDef = config.getShard(shard); + + print('[Config] Found ${shardDef.steps.length} step(s)'); + + // Collect output directories from GnNinja steps for upload + final List outDirs = [ + for (final BuildStep step in shardDef.steps) + if (step is GnNinjaStep) step.outDir, + ]; + + // Execute steps + final Stopwatch stopwatch = Stopwatch()..start(); + + for (int i = 0; i < shardDef.steps.length; i++) { + final BuildStep step = shardDef.steps[i]; + print( + '\n[${'Step ${i + 1}/${shardDef.steps.length}'}] ${step.runtimeType}'); + + await step.execute(cli.engineSrc); + } + + stopwatch.stop(); + print('\n[Build] Complete in ${stopwatch.elapsed}'); + + // Upload to GCS staging + if (cli.shouldUpload && outDirs.isNotEmpty) { + await uploadToStaging( + runId: cli.runId, + platform: platform, + shard: shard, + engineSrc: cli.engineSrc, + outDirs: outDirs, + ); + } + + print('\n${'='.padRight(60, '=')}'); + print('Shard $platform/$shard completed successfully'); + print('='.padRight(60, '=')); +} diff --git a/shorebird/ci/shard_runner/lib/cli.dart b/shorebird/ci/shard_runner/lib/cli.dart new file mode 100644 index 0000000000000..23a2a0b99a5a3 --- /dev/null +++ b/shorebird/ci/shard_runner/lib/cli.dart @@ -0,0 +1,83 @@ +import 'dart:io'; + +import 'package:args/args.dart'; +import 'package:meta/meta.dart'; +import 'package:path/path.dart' as p; + +/// Common CLI configuration used by shard runner scripts. +@immutable +class CliConfig { + CliConfig({ + required this.engineSrc, + required this.configDir, + required this.runId, + required this.shouldUpload, + }); + + /// Parses common options from ArgResults. + /// + /// [scriptPath] should be Platform.script.toFilePath() from the calling script. + factory CliConfig.fromArgs(ArgResults results, {required String scriptPath}) { + final String engineSrc = p.canonicalize(results['engine-src'] as String); + + // Config directory defaults to shorebird/ci (grandparent of bin/*.dart) + final String configDir = results['config-dir'] as String? ?? + p.dirname(p.dirname(p.dirname(scriptPath))); + + final String runId = results['run-id'] as String; + + final bool shouldUpload = + !results.options.contains('upload') || results['upload'] as bool; + + return CliConfig( + engineSrc: engineSrc, + configDir: configDir, + runId: runId, + shouldUpload: shouldUpload, + ); + } + final String engineSrc; + final String configDir; + final String runId; + final bool shouldUpload; + + /// Creates common argument parser options. + static void addCommonOptions(ArgParser parser, {bool includeUpload = true}) { + parser + ..addOption('engine-src', + abbr: 'e', help: 'Path to engine/src directory', mandatory: true) + ..addOption('run-id', + help: 'Build run identifier (use "local" for local development)', + mandatory: true) + ..addOption('config-dir', + abbr: 'c', help: 'Path to config directory (shorebird/ci)') + ..addFlag('help', abbr: 'h', negatable: false, help: 'Show this help'); + + if (includeUpload) { + parser.addFlag('upload', + defaultsTo: true, help: 'Upload artifacts to GCS staging'); + } + } + + /// Prints a standard header with configuration info. + void printHeader(String title, Map extra) { + print('='.padRight(60, '=')); + print(title); + print('='.padRight(60, '=')); + print('Engine: $engineSrc'); + print('Config: $configDir'); + print('Run ID: $runId'); + for (final MapEntry entry in extra.entries) { + print('${entry.key.padRight(12)}${entry.value}'); + } + print('='.padRight(60, '=')); + } + + /// Verifies that the engine source directory exists. + void verifyEngineSrc() { + if (!Directory(engineSrc).existsSync()) { + print('Error: Engine source not found at $engineSrc'); + exit(1); + } + } +} diff --git a/shorebird/ci/shard_runner/lib/compose_config.dart b/shorebird/ci/shard_runner/lib/compose_config.dart new file mode 100644 index 0000000000000..927748d326484 --- /dev/null +++ b/shorebird/ci/shard_runner/lib/compose_config.dart @@ -0,0 +1,76 @@ +import 'dart:convert'; +import 'dart:io'; + +import 'package:meta/meta.dart'; +import 'package:path/path.dart' as p; + +/// Configuration for all compose operations. +@immutable +class ComposeConfig { + ComposeConfig({required this.composes}); + + factory ComposeConfig.fromJson(Map json) { + return ComposeConfig( + composes: json.map( + (String key, value) => + MapEntry(key, ComposeDef.fromJson(value as Map)), + ), + ); + } + final Map composes; + + static ComposeConfig load(String configDir) { + final File file = File(p.join(configDir, 'compose.json')); + if (!file.existsSync()) { + throw FileSystemException('compose.json not found', file.path); + } + final String content = file.readAsStringSync(); + final Map json = + jsonDecode(content) as Map; + return ComposeConfig.fromJson(json); + } + + ComposeDef getCompose(String name) { + final ComposeDef? compose = composes[name]; + if (compose == null) { + throw ArgumentError( + 'Unknown compose: $name. Available: ${composes.keys.join(', ')}'); + } + return compose; + } +} + +/// Definition of a single compose operation. +@immutable +class ComposeDef { + ComposeDef({ + required this.requires, + required this.script, + this.flags = const [], + this.pathArgs = const {}, + }); + + factory ComposeDef.fromJson(Map json) { + return ComposeDef( + requires: (json['requires'] as List).cast(), + script: json['script'] as String, + flags: (json['flags'] as List?)?.cast() ?? [], + pathArgs: (json['path_args'] as Map?) + ?.cast() ?? + {}, + ); + } + + /// Shards that must complete before this compose can run. + final List requires; + + /// Path to the Python script to execute (relative to engine/src). + final String script; + + /// Boolean flags to pass to the script (e.g., --dsym, --strip, --zip). + final List flags; + + /// Arguments whose values are paths relative to out/ (e.g., --arm64-out-dir: ios_release). + /// These are expanded to absolute paths at runtime. + final Map pathArgs; +} diff --git a/shorebird/ci/shard_runner/lib/config.dart b/shorebird/ci/shard_runner/lib/config.dart new file mode 100644 index 0000000000000..378e463f96712 --- /dev/null +++ b/shorebird/ci/shard_runner/lib/config.dart @@ -0,0 +1,252 @@ +import 'dart:convert'; +import 'dart:io'; + +import 'package:meta/meta.dart'; +import 'package:path/path.dart' as p; +import 'package:shard_runner/process.dart'; + +/// Configuration for all shards on a platform. +@immutable +class PlatformConfig { + PlatformConfig({required this.shards}); + + factory PlatformConfig.fromJson(Map json) { + return PlatformConfig( + shards: json.map( + (String key, value) => + MapEntry(key, ShardDef.fromJson(value as Map)), + ), + ); + } + final Map shards; + + ShardDef getShard(String name) { + final ShardDef? shard = shards[name]; + if (shard == null) { + throw ArgumentError( + 'Unknown shard: $name. Available: ${shards.keys.join(', ')}'); + } + return shard; + } + + static PlatformConfig load(String platform, String configDir) { + final File file = File(p.join(configDir, 'shards', '$platform.json')); + if (!file.existsSync()) { + throw FileSystemException('Config file not found', file.path); + } + final String content = file.readAsStringSync(); + final Map json = + jsonDecode(content) as Map; + return PlatformConfig.fromJson(json); + } +} + +/// Definition of a single build shard. +@immutable +class ShardDef { + ShardDef( + {required this.steps, + this.composeInput, + this.artifacts = const []}); + + factory ShardDef.fromJson(Map json) { + final List steps = (json['steps'] as List) + .map((s) => BuildStep.fromJson(s as Map)) + .toList(); + + final List artifacts = (json['artifacts'] as List?) + ?.map((a) => ArtifactDef.fromJson(a as Map)) + .toList() ?? + []; + + return ShardDef( + steps: steps, + composeInput: json['compose_input'] as String?, + artifacts: artifacts, + ); + } + + /// Build steps to execute. For simple shards, this is a single GnNinja step. + final List steps; + + /// If set, this shard contributes to a compose operation. + final String? composeInput; + + /// Artifacts produced by this shard (paths relative to out_dir). + /// Used by finalize to know what to upload. + final List artifacts; +} + +/// Definition of an artifact to upload. +@immutable +class ArtifactDef { + ArtifactDef({ + required this.src, + required this.dst, + this.zip = false, + this.contentHash = false, + }); + + factory ArtifactDef.fromJson(Map json) { + return ArtifactDef( + src: json['src'] as String, + dst: json['dst'] as String, + zip: json['zip'] as bool? ?? false, + contentHash: json['content_hash'] as bool? ?? false, + ); + } + + /// Source path relative to out/ (or out// for single-step shards) + final String src; + + /// Destination path (relative to storage bucket root). + /// Supports placeholders: $engine (engine hash) + final String dst; + + /// If true, zip the source directory before uploading. + final bool zip; + + /// If true, also upload to content-hash path (for Dart SDK). + final bool contentHash; +} + +/// Base class for build steps. +sealed class BuildStep { + factory BuildStep.fromJson(Map json) { + final String type = json['type'] as String; + return switch (type) { + 'gn_ninja' => GnNinjaStep.fromJson(json), + 'rust' => RustStep.fromJson(json), + _ => throw ArgumentError('Unknown step type: $type'), + }; + } + Future execute(String engineSrc); +} + +/// A GN + Ninja build step. +@immutable +class GnNinjaStep implements BuildStep { + GnNinjaStep({ + required this.gnArgs, + required this.ninjaTargets, + required this.outDir, + }); + + factory GnNinjaStep.fromJson(Map json) { + return GnNinjaStep( + gnArgs: (json['gn_args'] as List).cast(), + ninjaTargets: (json['ninja_targets'] as List).cast(), + outDir: json['out_dir'] as String, + ); + } + final List gnArgs; + final List ninjaTargets; + final String outDir; + + @override + Future execute(String engineSrc) async { + // Import gn.dart functions + await _runGn(engineSrc, gnArgs, outDir); + await _runNinja(engineSrc, outDir, ninjaTargets); + } +} + +/// A Rust/Cargo build step. +@immutable +class RustStep implements BuildStep { + RustStep({required this.targets}); + + factory RustStep.fromJson(Map json) { + return RustStep( + targets: (json['targets'] as List).cast(), + ); + } + final List targets; + + @override + Future execute(String engineSrc) async { + await _runRust(engineSrc, targets); + } +} + +// Internal execution functions (to be moved to separate files) +Future _runGn(String engineSrc, List args, String outDir) async { + print('[GN] Building $outDir with args: ${args.join(' ')}'); + await runChecked( + 'python3', + [ + p.join(engineSrc, 'flutter', 'tools', 'gn'), + '--no-rbe', + '--no-enable-unittests', + '--target-dir', + outDir, + ...args, + ], + workingDirectory: engineSrc, + description: 'GN ($outDir)', + ); + print('[GN] Complete'); +} + +Future _runNinja( + String engineSrc, String outDir, List targets) async { + print('[Ninja] Building ${targets.join(' ')} in out/$outDir'); + await runChecked( + 'ninja', + [ + '-C', + p.join(engineSrc, 'out', outDir), + ...targets, + ], + workingDirectory: engineSrc, + description: 'Ninja ($outDir)', + ); + print('[Ninja] Complete'); +} + +Future _runRust(String engineSrc, List targets) async { + final String updaterPath = + p.join(engineSrc, 'flutter', 'third_party', 'updater', 'library'); + + // Separate Android and non-Android targets + final List androidTargets = + targets.where((String t) => t.contains('android')).toList(); + final List otherTargets = + targets.where((String t) => !t.contains('android')).toList(); + + // Build all Android targets together with cargo-ndk + if (androidTargets.isNotEmpty) { + print('[Rust] Building Android targets: ${androidTargets.join(', ')}'); + + final List args = ['ndk']; + for (final String target in androidTargets) { + args.addAll(['--target', target]); + } + args.addAll(['build', '--release']); + + await runChecked( + 'cargo', + args, + workingDirectory: updaterPath, + environment: { + 'ANDROID_NDK_HOME': + p.join(engineSrc, 'flutter', 'third_party', 'android_tools', 'ndk'), + }, + description: 'Cargo ndk (${androidTargets.join(', ')})', + ); + } + + // Build non-Android targets individually + for (final String target in otherTargets) { + print('[Rust] Building for target: $target'); + + await runChecked( + 'cargo', + ['build', '--release', '--target', target], + workingDirectory: updaterPath, + description: 'Cargo ($target)', + ); + } + + print('[Rust] Complete'); +} diff --git a/shorebird/ci/shard_runner/lib/gcs.dart b/shorebird/ci/shard_runner/lib/gcs.dart new file mode 100644 index 0000000000000..c4263658bc275 --- /dev/null +++ b/shorebird/ci/shard_runner/lib/gcs.dart @@ -0,0 +1,112 @@ +import 'dart:io'; + +import 'package:path/path.dart' as p; +import 'package:shard_runner/process.dart'; + +/// Staging bucket for intermediate artifacts. +const String stagingBucket = 'gs://shorebird-build-staging'; + +/// Uploads shard artifacts to GCS staging bucket. +/// +/// Artifacts are uploaded to: +/// gs://shorebird-build-staging/builds/{runId}/{platform}/{shard}/ +Future uploadToStaging({ + required String runId, + required String platform, + required String shard, + required String engineSrc, + required List outDirs, +}) async { + final String stagingRoot = '$stagingBucket/builds/$runId/$platform/$shard'; + + print('[GCS] Uploading to $stagingRoot'); + + for (final String outDir in outDirs) { + final String outPath = p.join(engineSrc, 'out', outDir); + if (!Directory(outPath).existsSync()) { + print('[GCS] Skipping $outDir (not found)'); + continue; + } + + // Create a tarball of the out directory + final String tarFile = '$outDir.tar.gz'; + print('[GCS] Creating $tarFile...'); + + await runChecked( + 'tar', + ['-czf', tarFile, '-C', p.join(engineSrc, 'out'), outDir], + workingDirectory: engineSrc, + description: 'tar create $tarFile', + ); + + // Upload to GCS + print('[GCS] Uploading $tarFile...'); + await runChecked( + 'gsutil', + ['-m', 'cp', p.join(engineSrc, tarFile), '$stagingRoot/'], + description: 'gsutil upload $tarFile', + ); + + // Clean up local tarball + File(p.join(engineSrc, tarFile)).deleteSync(); + } + + // Upload status file + final File statusFile = File(p.join(engineSrc, 'status.json')); + statusFile.writeAsStringSync('{"status": "success", "shard": "$shard"}'); + await runChecked('gsutil', ['cp', statusFile.path, '$stagingRoot/'], + description: 'gsutil upload status.json'); + statusFile.deleteSync(); + + print('[GCS] Upload complete'); +} + +/// Downloads artifacts from GCS staging bucket. +Future downloadFromStaging({ + required String runId, + required String platform, + required String shard, + required String destDir, +}) async { + final String stagingRoot = '$stagingBucket/builds/$runId/$platform/$shard'; + + print('[GCS] Downloading from $stagingRoot'); + + // List files in the staging location + final ProcessResult lsResult = await runChecked( + 'gsutil', + ['ls', stagingRoot], + description: 'gsutil ls $stagingRoot', + ); + + final List files = (lsResult.stdout as String) + .split('\n') + .where((String f) => f.endsWith('.tar.gz')) + .toList(); + + for (final String file in files) { + final String fileName = p.basename(file); + print('[GCS] Downloading $fileName...'); + + // Download + await runChecked( + 'gsutil', + ['cp', file, p.join(destDir, fileName)], + description: 'gsutil download $fileName', + ); + + // Extract + print('[GCS] Extracting $fileName...'); + await runChecked( + 'tar', + ['-xzf', fileName], + workingDirectory: destDir, + description: 'tar extract $fileName', + ); + + // Clean up tarball + File(p.join(destDir, fileName)).deleteSync(); + } + + print('[GCS] Download complete'); +} diff --git a/shorebird/ci/shard_runner/lib/manifest.dart b/shorebird/ci/shard_runner/lib/manifest.dart new file mode 100644 index 0000000000000..db5c3a5789248 --- /dev/null +++ b/shorebird/ci/shard_runner/lib/manifest.dart @@ -0,0 +1,26 @@ +import 'dart:io'; + +import 'package:path/path.dart' as p; + +/// Generates the artifacts manifest YAML from template. +/// +/// The manifest maps a Shorebird engine revision to a Flutter engine revision +/// and lists all artifact paths that should be proxied. +/// +/// [flutterEngineRevision] is the base Flutter engine revision this build is based on. +/// [configDir] is the path to the ci/ directory containing the template. +String generateManifest( + String flutterEngineRevision, { + required String configDir, +}) { + final templatePath = p.join(configDir, 'artifacts_manifest.template.yaml'); + final templateFile = File(templatePath); + + if (!templateFile.existsSync()) { + throw ArgumentError('Manifest template not found: $templatePath'); + } + + final template = templateFile.readAsStringSync(); + return template.replaceAll( + '{{flutter_engine_revision}}', flutterEngineRevision); +} diff --git a/shorebird/ci/shard_runner/lib/process.dart b/shorebird/ci/shard_runner/lib/process.dart new file mode 100644 index 0000000000000..897714d0d3d16 --- /dev/null +++ b/shorebird/ci/shard_runner/lib/process.dart @@ -0,0 +1,36 @@ +import 'dart:io'; + +/// Runs a process and throws if it exits with a non-zero code. +/// +/// Returns the [ProcessResult] for callers that need stdout/stderr. +Future runChecked( + String executable, + List arguments, { + String? workingDirectory, + Map? environment, + String? description, +}) async { + final ProcessResult result = await Process.run( + executable, + arguments, + workingDirectory: workingDirectory, + environment: environment, + ); + + if (result.exitCode != 0) { + final String desc = description ?? '$executable ${arguments.join(' ')}'; + final String stderr = (result.stderr as String).trim(); + final String stdout = (result.stdout as String).trim(); + final StringBuffer message = + StringBuffer('$desc failed (exit ${result.exitCode})'); + if (stdout.isNotEmpty) { + message.write('\nSTDOUT: $stdout'); + } + if (stderr.isNotEmpty) { + message.write('\nSTDERR: $stderr'); + } + throw Exception(message.toString()); + } + + return result; +} diff --git a/shorebird/ci/shard_runner/pubspec.lock b/shorebird/ci/shard_runner/pubspec.lock new file mode 100644 index 0000000000000..14ad0cf1c27c0 --- /dev/null +++ b/shorebird/ci/shard_runner/pubspec.lock @@ -0,0 +1,389 @@ +# Generated by pub +# See https://dart.dev/tools/pub/glossary#lockfile +packages: + _fe_analyzer_shared: + dependency: transitive + description: + name: _fe_analyzer_shared + sha256: "796d97d925add7ffcdf5595f33a2066a6e3cee97971e6dbef09b76b7880fd760" + url: "https://pub.dev" + source: hosted + version: "94.0.0" + analyzer: + dependency: transitive + description: + name: analyzer + sha256: "9c8ebb304d72c0a0c8764344627529d9503fc83d7d73e43ed727dc532f822e4b" + url: "https://pub.dev" + source: hosted + version: "10.0.2" + args: + dependency: "direct main" + description: + name: args + sha256: d0481093c50b1da8910eb0bb301626d4d8eb7284aa739614d2b394ee09e3ea04 + url: "https://pub.dev" + source: hosted + version: "2.7.0" + async: + dependency: transitive + description: + name: async + sha256: "758e6d74e971c3e5aceb4110bfd6698efc7f501675bcfe0c775459a8140750eb" + url: "https://pub.dev" + source: hosted + version: "2.13.0" + boolean_selector: + dependency: transitive + description: + name: boolean_selector + sha256: "8aab1771e1243a5063b8b0ff68042d67334e3feab9e95b9490f9a6ebf73b42ea" + url: "https://pub.dev" + source: hosted + version: "2.1.2" + cli_config: + dependency: transitive + description: + name: cli_config + sha256: ac20a183a07002b700f0c25e61b7ee46b23c309d76ab7b7640a028f18e4d99ec + url: "https://pub.dev" + source: hosted + version: "0.2.0" + collection: + dependency: "direct main" + description: + name: collection + sha256: "2f5709ae4d3d59dd8f7cd309b4e023046b57d8a6c82130785d2b0e5868084e76" + url: "https://pub.dev" + source: hosted + version: "1.19.1" + convert: + dependency: transitive + description: + name: convert + sha256: b30acd5944035672bc15c6b7a8b47d773e41e2f17de064350988c5d02adb1c68 + url: "https://pub.dev" + source: hosted + version: "3.1.2" + coverage: + dependency: transitive + description: + name: coverage + sha256: "5da775aa218eaf2151c721b16c01c7676fbfdd99cebba2bf64e8b807a28ff94d" + url: "https://pub.dev" + source: hosted + version: "1.15.0" + crypto: + dependency: transitive + description: + name: crypto + sha256: c8ea0233063ba03258fbcf2ca4d6dadfefe14f02fab57702265467a19f27fadf + url: "https://pub.dev" + source: hosted + version: "3.0.7" + file: + dependency: transitive + description: + name: file + sha256: a3b4f84adafef897088c160faf7dfffb7696046cb13ae90b508c2cbc95d3b8d4 + url: "https://pub.dev" + source: hosted + version: "7.0.1" + frontend_server_client: + dependency: transitive + description: + name: frontend_server_client + sha256: f64a0333a82f30b0cca061bc3d143813a486dc086b574bfb233b7c1372427694 + url: "https://pub.dev" + source: hosted + version: "4.0.0" + glob: + dependency: transitive + description: + name: glob + sha256: c3f1ee72c96f8f78935e18aa8cecced9ab132419e8625dc187e1c2408efc20de + url: "https://pub.dev" + source: hosted + version: "2.1.3" + http_multi_server: + dependency: transitive + description: + name: http_multi_server + sha256: aa6199f908078bb1c5efb8d8638d4ae191aac11b311132c3ef48ce352fb52ef8 + url: "https://pub.dev" + source: hosted + version: "3.2.2" + http_parser: + dependency: transitive + description: + name: http_parser + sha256: "178d74305e7866013777bab2c3d8726205dc5a4dd935297175b19a23a2e66571" + url: "https://pub.dev" + source: hosted + version: "4.1.2" + io: + dependency: transitive + description: + name: io + sha256: dfd5a80599cf0165756e3181807ed3e77daf6dd4137caaad72d0b7931597650b + url: "https://pub.dev" + source: hosted + version: "1.0.5" + lints: + dependency: "direct dev" + description: + name: lints + sha256: cbf8d4b858bb0134ef3ef87841abdf8d63bfc255c266b7bf6b39daa1085c4290 + url: "https://pub.dev" + source: hosted + version: "3.0.0" + logging: + dependency: transitive + description: + name: logging + sha256: c8245ada5f1717ed44271ed1c26b8ce85ca3228fd2ffdb75468ab01979309d61 + url: "https://pub.dev" + source: hosted + version: "1.3.0" + matcher: + dependency: transitive + description: + name: matcher + sha256: "12956d0ad8390bbcc63ca2e1469c0619946ccb52809807067a7020d57e647aa6" + url: "https://pub.dev" + source: hosted + version: "0.12.18" + meta: + dependency: "direct main" + description: + name: meta + sha256: "9f29b9bcc8ee287b1a31e0d01be0eae99a930dbffdaecf04b3f3d82a969f296f" + url: "https://pub.dev" + source: hosted + version: "1.18.1" + mime: + dependency: transitive + description: + name: mime + sha256: "41a20518f0cb1256669420fdba0cd90d21561e560ac240f26ef8322e45bb7ed6" + url: "https://pub.dev" + source: hosted + version: "2.0.0" + node_preamble: + dependency: transitive + description: + name: node_preamble + sha256: "6e7eac89047ab8a8d26cf16127b5ed26de65209847630400f9aefd7cd5c730db" + url: "https://pub.dev" + source: hosted + version: "2.0.2" + package_config: + dependency: transitive + description: + name: package_config + sha256: f096c55ebb7deb7e384101542bfba8c52696c1b56fca2eb62827989ef2353bbc + url: "https://pub.dev" + source: hosted + version: "2.2.0" + path: + dependency: "direct main" + description: + name: path + sha256: "75cca69d1490965be98c73ceaea117e8a04dd21217b37b292c9ddbec0d955bc5" + url: "https://pub.dev" + source: hosted + version: "1.9.1" + pool: + dependency: transitive + description: + name: pool + sha256: "978783255c543aa3586a1b3c21f6e9d720eb315376a915872c61ef8b5c20177d" + url: "https://pub.dev" + source: hosted + version: "1.5.2" + pub_semver: + dependency: transitive + description: + name: pub_semver + sha256: "5bfcf68ca79ef689f8990d1160781b4bad40a3bd5e5218ad4076ddb7f4081585" + url: "https://pub.dev" + source: hosted + version: "2.2.0" + shelf: + dependency: transitive + description: + name: shelf + sha256: e7dd780a7ffb623c57850b33f43309312fc863fb6aa3d276a754bb299839ef12 + url: "https://pub.dev" + source: hosted + version: "1.4.2" + shelf_packages_handler: + dependency: transitive + description: + name: shelf_packages_handler + sha256: "89f967eca29607c933ba9571d838be31d67f53f6e4ee15147d5dc2934fee1b1e" + url: "https://pub.dev" + source: hosted + version: "3.0.2" + shelf_static: + dependency: transitive + description: + name: shelf_static + sha256: c87c3875f91262785dade62d135760c2c69cb217ac759485334c5857ad89f6e3 + url: "https://pub.dev" + source: hosted + version: "1.1.3" + shelf_web_socket: + dependency: transitive + description: + name: shelf_web_socket + sha256: "3632775c8e90d6c9712f883e633716432a27758216dfb61bd86a8321c0580925" + url: "https://pub.dev" + source: hosted + version: "3.0.0" + source_map_stack_trace: + dependency: transitive + description: + name: source_map_stack_trace + sha256: c0713a43e323c3302c2abe2a1cc89aa057a387101ebd280371d6a6c9fa68516b + url: "https://pub.dev" + source: hosted + version: "2.1.2" + source_maps: + dependency: transitive + description: + name: source_maps + sha256: "190222579a448b03896e0ca6eca5998fa810fda630c1d65e2f78b3f638f54812" + url: "https://pub.dev" + source: hosted + version: "0.10.13" + source_span: + dependency: transitive + description: + name: source_span + sha256: "254ee5351d6cb365c859e20ee823c3bb479bf4a293c22d17a9f1bf144ce86f7c" + url: "https://pub.dev" + source: hosted + version: "1.10.1" + stack_trace: + dependency: transitive + description: + name: stack_trace + sha256: "8b27215b45d22309b5cddda1aa2b19bdfec9df0e765f2de506401c071d38d1b1" + url: "https://pub.dev" + source: hosted + version: "1.12.1" + stream_channel: + dependency: transitive + description: + name: stream_channel + sha256: "969e04c80b8bcdf826f8f16579c7b14d780458bd97f56d107d3950fdbeef059d" + url: "https://pub.dev" + source: hosted + version: "2.1.4" + string_scanner: + dependency: transitive + description: + name: string_scanner + sha256: "921cd31725b72fe181906c6a94d987c78e3b98c2e205b397ea399d4054872b43" + url: "https://pub.dev" + source: hosted + version: "1.4.1" + term_glyph: + dependency: transitive + description: + name: term_glyph + sha256: "7f554798625ea768a7518313e58f83891c7f5024f88e46e7182a4558850a4b8e" + url: "https://pub.dev" + source: hosted + version: "1.2.2" + test: + dependency: "direct dev" + description: + name: test + sha256: "54c516bbb7cee2754d327ad4fca637f78abfc3cbcc5ace83b3eda117e42cd71a" + url: "https://pub.dev" + source: hosted + version: "1.29.0" + test_api: + dependency: transitive + description: + name: test_api + sha256: "93167629bfc610f71560ab9312acdda4959de4df6fac7492c89ff0d3886f6636" + url: "https://pub.dev" + source: hosted + version: "0.7.9" + test_core: + dependency: transitive + description: + name: test_core + sha256: "394f07d21f0f2255ec9e3989f21e54d3c7dc0e6e9dbce160e5a9c1a6be0e2943" + url: "https://pub.dev" + source: hosted + version: "0.6.15" + typed_data: + dependency: transitive + description: + name: typed_data + sha256: f9049c039ebfeb4cf7a7104a675823cd72dba8297f264b6637062516699fa006 + url: "https://pub.dev" + source: hosted + version: "1.4.0" + vm_service: + dependency: transitive + description: + name: vm_service + sha256: "45caa6c5917fa127b5dbcfbd1fa60b14e583afdc08bfc96dda38886ca252eb60" + url: "https://pub.dev" + source: hosted + version: "15.0.2" + watcher: + dependency: transitive + description: + name: watcher + sha256: "1398c9f081a753f9226febe8900fce8f7d0a67163334e1c94a2438339d79d635" + url: "https://pub.dev" + source: hosted + version: "1.2.1" + web: + dependency: transitive + description: + name: web + sha256: "868d88a33d8a87b18ffc05f9f030ba328ffefba92d6c127917a2ba740f9cfe4a" + url: "https://pub.dev" + source: hosted + version: "1.1.1" + web_socket: + dependency: transitive + description: + name: web_socket + sha256: "34d64019aa8e36bf9842ac014bb5d2f5586ca73df5e4d9bf5c936975cae6982c" + url: "https://pub.dev" + source: hosted + version: "1.0.1" + web_socket_channel: + dependency: transitive + description: + name: web_socket_channel + sha256: d645757fb0f4773d602444000a8131ff5d48c9e47adfe9772652dd1a4f2d45c8 + url: "https://pub.dev" + source: hosted + version: "3.0.3" + webkit_inspection_protocol: + dependency: transitive + description: + name: webkit_inspection_protocol + sha256: "87d3f2333bb240704cd3f1c6b5b7acd8a10e7f0bc28c28dcf14e782014f4a572" + url: "https://pub.dev" + source: hosted + version: "1.2.1" + yaml: + dependency: transitive + description: + name: yaml + sha256: b9da305ac7c39faa3f030eccd175340f968459dae4af175130b3fc47e40d76ce + url: "https://pub.dev" + source: hosted + version: "3.1.3" +sdks: + dart: ">=3.9.0 <4.0.0" diff --git a/shorebird/ci/shard_runner/pubspec.yaml b/shorebird/ci/shard_runner/pubspec.yaml new file mode 100644 index 0000000000000..8e0834004a88b --- /dev/null +++ b/shorebird/ci/shard_runner/pubspec.yaml @@ -0,0 +1,17 @@ +name: shard_runner +description: Dart-based build shard runner for Shorebird CI +version: 0.1.0 +publish_to: none + +environment: + sdk: ^3.0.0 + +dependencies: + args: ^2.4.0 + collection: ^1.18.0 + meta: ^1.11.0 + path: ^1.8.0 + +dev_dependencies: + lints: ^3.0.0 + test: ^1.24.0 diff --git a/shorebird/ci/shard_runner/test/compose_config_test.dart b/shorebird/ci/shard_runner/test/compose_config_test.dart new file mode 100644 index 0000000000000..24868a6affcc5 --- /dev/null +++ b/shorebird/ci/shard_runner/test/compose_config_test.dart @@ -0,0 +1,119 @@ +import 'package:test/test.dart'; +import 'package:shard_runner/compose_config.dart'; + +void main() { + group('ComposeConfig', () { + test('parses compose definitions', () { + final Map json = { + 'ios-framework': { + 'requires': ['ios-release', 'ios-sim-x64', 'ios-sim-arm64'], + 'script': 'flutter/sky/tools/create_ios_framework.py', + 'flags': ['--dsym', '--strip'], + 'path_args': { + '--arm64-out-dir': 'ios_release', + }, + }, + 'macos-framework': { + 'requires': ['mac-arm64', 'mac-x64'], + 'script': 'flutter/sky/tools/create_macos_framework.py', + 'flags': ['--zip'], + }, + }; + + final ComposeConfig config = ComposeConfig.fromJson(json); + + expect(config.composes.length, 2); + expect(config.composes.containsKey('ios-framework'), true); + expect(config.composes.containsKey('macos-framework'), true); + }); + + test('getCompose returns correct definition', () { + final Map json = { + 'ios-framework': { + 'requires': ['ios-release'], + 'script': 'create_ios_framework.py', + }, + }; + + final ComposeConfig config = ComposeConfig.fromJson(json); + final ComposeDef compose = config.getCompose('ios-framework'); + + expect(compose.requires, ['ios-release']); + expect(compose.script, 'create_ios_framework.py'); + }); + + test('getCompose throws for unknown name', () { + final ComposeConfig config = + ComposeConfig(composes: {}); + + expect( + () => config.getCompose('nonexistent'), + throwsA(isA()), + ); + }); + }); + + group('ComposeDef', () { + test('parses all fields', () { + final Map json = { + 'requires': ['shard-a', 'shard-b'], + 'script': 'path/to/script.py', + 'flags': ['--dsym', '--strip'], + 'path_args': { + '--arm64-out-dir': 'ios_release', + '--x64-out-dir': 'ios_debug_sim', + }, + }; + + final ComposeDef compose = ComposeDef.fromJson(json); + + expect(compose.requires, ['shard-a', 'shard-b']); + expect(compose.script, 'path/to/script.py'); + expect(compose.flags, ['--dsym', '--strip']); + expect(compose.pathArgs, { + '--arm64-out-dir': 'ios_release', + '--x64-out-dir': 'ios_debug_sim', + }); + }); + + test('defaults flags and path_args when missing', () { + final Map json = { + 'requires': ['shard-a'], + 'script': 'script.py', + }; + + final ComposeDef compose = ComposeDef.fromJson(json); + + expect(compose.flags, isEmpty); + expect(compose.pathArgs, isEmpty); + }); + + test('parses ios-framework config correctly', () { + final Map json = { + 'requires': [ + 'ios-release', + 'ios-release-ext', + 'ios-sim-x64', + 'ios-sim-x64-ext', + 'ios-sim-arm64', + 'ios-sim-arm64-ext', + ], + 'script': 'flutter/sky/tools/create_ios_framework.py', + 'flags': ['--dsym', '--strip'], + 'path_args': { + '--arm64-out-dir': 'ios_release', + '--simulator-x64-out-dir': 'ios_debug_sim', + '--simulator-arm64-out-dir': 'ios_debug_sim_arm64', + }, + }; + + final ComposeDef compose = ComposeDef.fromJson(json); + + expect(compose.requires.length, 6); + expect(compose.script, 'flutter/sky/tools/create_ios_framework.py'); + expect(compose.flags, ['--dsym', '--strip']); + expect(compose.pathArgs.keys, contains('--arm64-out-dir')); + expect(compose.pathArgs['--arm64-out-dir'], 'ios_release'); + }); + }); +} diff --git a/shorebird/ci/shard_runner/test/config_test.dart b/shorebird/ci/shard_runner/test/config_test.dart new file mode 100644 index 0000000000000..ec3d508864758 --- /dev/null +++ b/shorebird/ci/shard_runner/test/config_test.dart @@ -0,0 +1,264 @@ +import 'package:test/test.dart'; +import 'package:shard_runner/config.dart'; + +void main() { + group('PlatformConfig', () { + test('parses single-step shard', () { + final json = { + 'android-arm64': { + 'steps': [ + { + 'type': 'gn_ninja', + 'gn_args': [ + '--android', + '--android-cpu=arm64', + '--runtime-mode=release' + ], + 'ninja_targets': ['default', 'gen_snapshot'], + 'out_dir': 'android_release_arm64', + }, + ], + }, + }; + + final config = PlatformConfig.fromJson(json); + + expect(config.shards.length, 1); + expect(config.shards.containsKey('android-arm64'), true); + + final shard = config.getShard('android-arm64'); + expect(shard.steps.length, 1); + expect(shard.steps.first, isA()); + expect(shard.artifacts, isEmpty); + + final step = shard.steps.first as GnNinjaStep; + expect(step.gnArgs, + ['--android', '--android-cpu=arm64', '--runtime-mode=release']); + expect(step.ninjaTargets, ['default', 'gen_snapshot']); + expect(step.outDir, 'android_release_arm64'); + }); + + test('parses shard with artifacts', () { + final json = { + 'android-arm64': { + 'steps': [ + { + 'type': 'gn_ninja', + 'gn_args': ['--android'], + 'ninja_targets': ['default'], + 'out_dir': 'android_release_arm64', + }, + ], + 'artifacts': [ + { + 'src': 'zip_archives/artifacts.zip', + 'dst': 'flutter_infra/\$engine/artifacts.zip' + }, + {'src': 'maven.pom', 'dst': 'maven/\$engine/maven.pom'}, + ], + }, + }; + + final config = PlatformConfig.fromJson(json); + final shard = config.getShard('android-arm64'); + + expect(shard.artifacts.length, 2); + expect(shard.artifacts[0].src, 'zip_archives/artifacts.zip'); + expect(shard.artifacts[0].dst, 'flutter_infra/\$engine/artifacts.zip'); + expect(shard.artifacts[1].src, 'maven.pom'); + }); + + test('parses multi-step shard', () { + final json = { + 'host': { + 'steps': [ + { + 'type': 'rust', + 'targets': ['aarch64-linux-android', 'x86_64-unknown-linux-gnu'], + }, + { + 'type': 'gn_ninja', + 'gn_args': ['--runtime-mode=release'], + 'ninja_targets': ['dart_sdk'], + 'out_dir': 'host_release', + }, + ], + }, + }; + + final config = PlatformConfig.fromJson(json); + final shard = config.getShard('host'); + + expect(shard.steps.length, 2); + expect(shard.steps[0], isA()); + expect(shard.steps[1], isA()); + + final rustStep = shard.steps[0] as RustStep; + expect(rustStep.targets, + ['aarch64-linux-android', 'x86_64-unknown-linux-gnu']); + + final gnStep = shard.steps[1] as GnNinjaStep; + expect(gnStep.outDir, 'host_release'); + }); + + test('parses compose_input', () { + final json = { + 'ios-release': { + 'steps': [ + { + 'type': 'gn_ninja', + 'gn_args': ['--ios', '--runtime-mode=release'], + 'ninja_targets': ['flutter_framework'], + 'out_dir': 'ios_release', + }, + ], + 'compose_input': 'ios-framework', + }, + }; + + final config = PlatformConfig.fromJson(json); + final shard = config.getShard('ios-release'); + + expect(shard.composeInput, 'ios-framework'); + }); + + test('getShard throws for unknown shard', () { + final config = PlatformConfig(shards: {}); + + expect( + () => config.getShard('nonexistent'), + throwsA(isA()), + ); + }); + }); + + group('BuildStep.fromJson', () { + test('parses gn_ninja type', () { + final json = { + 'type': 'gn_ninja', + 'gn_args': ['--android'], + 'ninja_targets': ['default'], + 'out_dir': 'out_dir', + }; + + final step = BuildStep.fromJson(json); + expect(step, isA()); + }); + + test('parses rust type', () { + final json = { + 'type': 'rust', + 'targets': ['x86_64-unknown-linux-gnu'], + }; + + final step = BuildStep.fromJson(json); + expect(step, isA()); + }); + + test('throws for unknown type', () { + final json = { + 'type': 'unknown', + }; + + expect( + () => BuildStep.fromJson(json), + throwsA(isA()), + ); + }); + }); + + group('GnNinjaStep', () { + test('fromJson parses all fields', () { + final json = { + 'type': 'gn_ninja', + 'gn_args': ['--android', '--runtime-mode=release'], + 'ninja_targets': ['default', 'gen_snapshot'], + 'out_dir': 'android_release', + }; + + final step = GnNinjaStep.fromJson(json); + + expect(step.gnArgs, ['--android', '--runtime-mode=release']); + expect(step.ninjaTargets, ['default', 'gen_snapshot']); + expect(step.outDir, 'android_release'); + }); + }); + + group('RustStep', () { + test('fromJson parses targets', () { + final json = { + 'type': 'rust', + 'targets': [ + 'aarch64-linux-android', + 'armv7-linux-androideabi', + 'x86_64-unknown-linux-gnu', + ], + }; + + final step = RustStep.fromJson(json); + + expect(step.targets, [ + 'aarch64-linux-android', + 'armv7-linux-androideabi', + 'x86_64-unknown-linux-gnu', + ]); + }); + }); + + group('ArtifactDef', () { + test('fromJson parses src and dst', () { + final json = { + 'src': 'zip_archives/artifacts.zip', + 'dst': 'flutter_infra/\$engine/artifacts.zip', + }; + + final artifact = ArtifactDef.fromJson(json); + + expect(artifact.src, 'zip_archives/artifacts.zip'); + expect(artifact.dst, 'flutter_infra/\$engine/artifacts.zip'); + expect(artifact.zip, false); + expect(artifact.contentHash, false); + }); + + test('fromJson parses zip flag', () { + final json = { + 'src': 'dart-sdk', + 'dst': 'flutter_infra/dart-sdk.zip', + 'zip': true, + }; + + final artifact = ArtifactDef.fromJson(json); + + expect(artifact.zip, true); + }); + + test('fromJson parses content_hash flag', () { + final json = { + 'src': 'dart-sdk', + 'dst': 'flutter_infra/dart-sdk.zip', + 'content_hash': true, + }; + + final artifact = ArtifactDef.fromJson(json); + + expect(artifact.contentHash, true); + }); + + test('fromJson parses all flags together', () { + final json = { + 'src': 'host_release/dart-sdk', + 'dst': 'flutter_infra/flutter/\$engine/dart-sdk-linux-x64.zip', + 'zip': true, + 'content_hash': true, + }; + + final artifact = ArtifactDef.fromJson(json); + + expect(artifact.src, 'host_release/dart-sdk'); + expect(artifact.dst, + 'flutter_infra/flutter/\$engine/dart-sdk-linux-x64.zip'); + expect(artifact.zip, true); + expect(artifact.contentHash, true); + }); + }); +} diff --git a/shorebird/ci/shard_runner/test/manifest_test.dart b/shorebird/ci/shard_runner/test/manifest_test.dart new file mode 100644 index 0000000000000..1e8e930eb2458 --- /dev/null +++ b/shorebird/ci/shard_runner/test/manifest_test.dart @@ -0,0 +1,208 @@ +import 'dart:io'; + +import 'package:path/path.dart' as p; +import 'package:test/test.dart'; +import 'package:shard_runner/manifest.dart'; + +void main() { + group('generateManifest', () { + // Path to the ci/ directory where the template lives + // Tests run from shard_runner/, so go up one level to ci/ + final String configDir = p.normalize(p.join(Directory.current.path, '..')); + + test('generates valid YAML structure', () { + final manifest = generateManifest('abc123def456', configDir: configDir); + + expect(manifest, contains('flutter_engine_revision: abc123def456')); + expect(manifest, contains('storage_bucket: download.shorebird.dev')); + expect(manifest, contains('artifact_overrides:')); + }); + + test('includes Android release artifacts', () { + final manifest = generateManifest('test-hash', configDir: configDir); + + // Android arm64 + expect( + manifest, + contains( + r'flutter_infra_release/flutter/$engine/android-arm64-release/artifacts.zip')); + expect( + manifest, + contains( + r'flutter_infra_release/flutter/$engine/android-arm64-release/linux-x64.zip')); + expect( + manifest, + contains( + r'flutter_infra_release/flutter/$engine/android-arm64-release/darwin-x64.zip')); + expect( + manifest, + contains( + r'flutter_infra_release/flutter/$engine/android-arm64-release/windows-x64.zip')); + expect( + manifest, + contains( + r'flutter_infra_release/flutter/$engine/android-arm64-release/symbols.zip')); + + // Android arm32 + expect( + manifest, + contains( + r'flutter_infra_release/flutter/$engine/android-arm-release/artifacts.zip')); + + // Android x64 + expect( + manifest, + contains( + r'flutter_infra_release/flutter/$engine/android-x64-release/artifacts.zip')); + }); + + test('includes Dart SDK for all platforms', () { + final manifest = generateManifest('test-hash', configDir: configDir); + + expect( + manifest, + contains( + r'flutter_infra_release/flutter/$engine/dart-sdk-darwin-arm64.zip')); + expect( + manifest, + contains( + r'flutter_infra_release/flutter/$engine/dart-sdk-darwin-x64.zip')); + expect( + manifest, + contains( + r'flutter_infra_release/flutter/$engine/dart-sdk-linux-x64.zip')); + expect( + manifest, + contains( + r'flutter_infra_release/flutter/$engine/dart-sdk-windows-x64.zip')); + }); + + test('includes Maven artifacts', () { + final manifest = generateManifest('test-hash', configDir: configDir); + + // flutter_embedding_release + expect( + manifest, + contains( + r'download.flutter.io/io/flutter/flutter_embedding_release/1.0.0-$engine/flutter_embedding_release-1.0.0-$engine.pom')); + expect( + manifest, + contains( + r'download.flutter.io/io/flutter/flutter_embedding_release/1.0.0-$engine/flutter_embedding_release-1.0.0-$engine.jar')); + + // arm64_v8a_release + expect( + manifest, + contains( + r'download.flutter.io/io/flutter/arm64_v8a_release/1.0.0-$engine/arm64_v8a_release-1.0.0-$engine.pom')); + expect( + manifest, + contains( + r'download.flutter.io/io/flutter/arm64_v8a_release/1.0.0-$engine/arm64_v8a_release-1.0.0-$engine.jar')); + + // armeabi_v7a_release + expect( + manifest, + contains( + r'download.flutter.io/io/flutter/armeabi_v7a_release/1.0.0-$engine/armeabi_v7a_release-1.0.0-$engine.pom')); + + // x86_64_release + expect( + manifest, + contains( + r'download.flutter.io/io/flutter/x86_64_release/1.0.0-$engine/x86_64_release-1.0.0-$engine.pom')); + }); + + test('includes iOS release artifacts', () { + final manifest = generateManifest('test-hash', configDir: configDir); + + expect( + manifest, + contains( + r'flutter_infra_release/flutter/$engine/ios-release/artifacts.zip')); + expect( + manifest, + contains( + r'flutter_infra_release/flutter/$engine/ios-release/Flutter.dSYM.zip')); + }); + + test('includes Linux release artifacts', () { + final manifest = generateManifest('test-hash', configDir: configDir); + + expect( + manifest, + contains( + r'flutter_infra_release/flutter/$engine/linux-x64/artifacts.zip')); + expect( + manifest, + contains( + r'flutter_infra_release/flutter/$engine/linux-x64-release/linux-x64-flutter-gtk.zip')); + }); + + test('includes macOS release artifacts', () { + final manifest = generateManifest('test-hash', configDir: configDir); + + expect( + manifest, + contains( + r'flutter_infra_release/flutter/$engine/darwin-x64-release/artifacts.zip')); + expect( + manifest, + contains( + r'flutter_infra_release/flutter/$engine/darwin-x64-release/framework.zip')); + expect( + manifest, + contains( + r'flutter_infra_release/flutter/$engine/darwin-x64-release/gen_snapshot.zip')); + }); + + test('includes Windows release artifacts', () { + final manifest = generateManifest('test-hash', configDir: configDir); + + expect( + manifest, + contains( + r'flutter_infra_release/flutter/$engine/windows-x64/artifacts.zip')); + expect( + manifest, + contains( + r'flutter_infra_release/flutter/$engine/windows-x64-release/windows-x64-flutter.zip')); + }); + + test('includes engine_stamp.json', () { + final manifest = generateManifest('test-hash', configDir: configDir); + + expect(manifest, + contains(r'flutter_infra_release/flutter/$engine/engine_stamp.json')); + }); + + test('includes flutter_patched_sdk_product', () { + final manifest = generateManifest('test-hash', configDir: configDir); + + expect( + manifest, + contains( + r'flutter_infra_release/flutter/$engine/flutter_patched_sdk_product.zip')); + }); + + test('uses \$engine placeholder (not hardcoded hash)', () { + final manifest = generateManifest('abc123', configDir: configDir); + + // The flutter_engine_revision should use the actual hash + expect(manifest, contains('flutter_engine_revision: abc123')); + + // But artifact paths should use $engine placeholder + expect(manifest, contains(r'$engine')); + // Should NOT contain the actual hash in artifact paths + expect(manifest.split('flutter_engine_revision:')[1], + isNot(contains('abc123/'))); + }); + + test('throws when template file not found', () { + expect( + () => generateManifest('test-hash', configDir: '/nonexistent'), + throwsA(isA()), + ); + }); + }); +} diff --git a/shorebird/ci/shards/linux.json b/shorebird/ci/shards/linux.json new file mode 100644 index 0000000000000..453ede1e851bc --- /dev/null +++ b/shorebird/ci/shards/linux.json @@ -0,0 +1,92 @@ +{ + "android-arm64": { + "steps": [ + { + "type": "gn_ninja", + "gn_args": ["--android", "--android-cpu=arm64", "--runtime-mode=release"], + "ninja_targets": ["default", "gen_snapshot"], + "out_dir": "android_release_arm64" + } + ], + "artifacts": [ + {"src": "zip_archives/android-arm64-release/artifacts.zip", "dst": "flutter_infra_release/flutter/$engine/android-arm64-release/artifacts.zip"}, + {"src": "zip_archives/android-arm64-release/linux-x64.zip", "dst": "flutter_infra_release/flutter/$engine/android-arm64-release/linux-x64.zip"}, + {"src": "zip_archives/android-arm64-release/symbols.zip", "dst": "flutter_infra_release/flutter/$engine/android-arm64-release/symbols.zip"}, + {"src": "arm64_v8a_release.pom", "dst": "download.flutter.io/io/flutter/arm64_v8a_release/1.0.0-$engine/arm64_v8a_release-1.0.0-$engine.pom"}, + {"src": "arm64_v8a_release.jar", "dst": "download.flutter.io/io/flutter/arm64_v8a_release/1.0.0-$engine/arm64_v8a_release-1.0.0-$engine.jar"}, + {"src": "arm64_v8a_release.maven-metadata.xml", "dst": "download.flutter.io/io/flutter/arm64_v8a_release/1.0.0-$engine/arm64_v8a_release-1.0.0-$engine.maven-metadata.xml"} + ] + }, + "android-arm32": { + "steps": [ + { + "type": "gn_ninja", + "gn_args": ["--android", "--runtime-mode=release"], + "ninja_targets": ["default", "gen_snapshot"], + "out_dir": "android_release" + } + ], + "artifacts": [ + {"src": "zip_archives/android-arm-release/artifacts.zip", "dst": "flutter_infra_release/flutter/$engine/android-arm-release/artifacts.zip"}, + {"src": "zip_archives/android-arm-release/linux-x64.zip", "dst": "flutter_infra_release/flutter/$engine/android-arm-release/linux-x64.zip"}, + {"src": "zip_archives/android-arm-release/symbols.zip", "dst": "flutter_infra_release/flutter/$engine/android-arm-release/symbols.zip"}, + {"src": "armeabi_v7a_release.pom", "dst": "download.flutter.io/io/flutter/armeabi_v7a_release/1.0.0-$engine/armeabi_v7a_release-1.0.0-$engine.pom"}, + {"src": "armeabi_v7a_release.jar", "dst": "download.flutter.io/io/flutter/armeabi_v7a_release/1.0.0-$engine/armeabi_v7a_release-1.0.0-$engine.jar"}, + {"src": "armeabi_v7a_release.maven-metadata.xml", "dst": "download.flutter.io/io/flutter/armeabi_v7a_release/1.0.0-$engine/armeabi_v7a_release-1.0.0-$engine.maven-metadata.xml"}, + {"src": "flutter_embedding_release.pom", "dst": "download.flutter.io/io/flutter/flutter_embedding_release/1.0.0-$engine/flutter_embedding_release-1.0.0-$engine.pom"}, + {"src": "flutter_embedding_release.jar", "dst": "download.flutter.io/io/flutter/flutter_embedding_release/1.0.0-$engine/flutter_embedding_release-1.0.0-$engine.jar"}, + {"src": "flutter_embedding_release.maven-metadata.xml", "dst": "download.flutter.io/io/flutter/flutter_embedding_release/1.0.0-$engine/flutter_embedding_release-1.0.0-$engine.maven-metadata.xml"} + ] + }, + "android-x64": { + "steps": [ + { + "type": "gn_ninja", + "gn_args": ["--android", "--android-cpu=x64", "--runtime-mode=release"], + "ninja_targets": ["default", "gen_snapshot"], + "out_dir": "android_release_x64" + } + ], + "artifacts": [ + {"src": "zip_archives/android-x64-release/artifacts.zip", "dst": "flutter_infra_release/flutter/$engine/android-x64-release/artifacts.zip"}, + {"src": "zip_archives/android-x64-release/linux-x64.zip", "dst": "flutter_infra_release/flutter/$engine/android-x64-release/linux-x64.zip"}, + {"src": "zip_archives/android-x64-release/symbols.zip", "dst": "flutter_infra_release/flutter/$engine/android-x64-release/symbols.zip"}, + {"src": "x86_64_release.pom", "dst": "download.flutter.io/io/flutter/x86_64_release/1.0.0-$engine/x86_64_release-1.0.0-$engine.pom"}, + {"src": "x86_64_release.jar", "dst": "download.flutter.io/io/flutter/x86_64_release/1.0.0-$engine/x86_64_release-1.0.0-$engine.jar"}, + {"src": "x86_64_release.maven-metadata.xml", "dst": "download.flutter.io/io/flutter/x86_64_release/1.0.0-$engine/x86_64_release-1.0.0-$engine.maven-metadata.xml"} + ] + }, + "host": { + "steps": [ + { + "type": "rust", + "targets": [ + "armv7-linux-androideabi", + "aarch64-linux-android", + "i686-linux-android", + "x86_64-linux-android", + "x86_64-unknown-linux-gnu" + ] + }, + { + "type": "gn_ninja", + "gn_args": ["--runtime-mode=release", "--no-prebuilt-dart-sdk"], + "ninja_targets": ["dart_sdk", "flutter/shell/platform/linux:flutter_gtk", "flutter/build/archives:flutter_patched_sdk"], + "out_dir": "host_release" + }, + { + "type": "gn_ninja", + "gn_args": ["--no-prebuilt-dart-sdk"], + "ninja_targets": ["flutter/build/archives:artifacts"], + "out_dir": "host_debug" + } + ], + "artifacts": [ + {"src": "host_release/dart-sdk", "dst": "flutter_infra_release/flutter/$engine/dart-sdk-linux-x64.zip", "zip": true, "content_hash": true}, + {"src": "host_release/zip_archives/flutter_patched_sdk_product.zip", "dst": "flutter_infra_release/flutter/$engine/flutter_patched_sdk_product.zip"}, + {"src": "host_release/zip_archives/linux-x64-release/linux-x64-flutter-gtk.zip", "dst": "flutter_infra_release/flutter/$engine/linux-x64-release/linux-x64-flutter-gtk.zip"}, + {"src": "host_debug/zip_archives/linux-x64/artifacts.zip", "dst": "flutter_infra_release/flutter/$engine/linux-x64/artifacts.zip"}, + {"src": "host_release/aot_tools/aot-tools.dill", "dst": "shorebird/$engine/aot-tools.dill"} + ] + } +} diff --git a/shorebird/ci/shards/macos.json b/shorebird/ci/shards/macos.json new file mode 100644 index 0000000000000..a1ebf2d50a26e --- /dev/null +++ b/shorebird/ci/shards/macos.json @@ -0,0 +1,145 @@ +{ + "android": { + "steps": [ + { + "type": "rust", + "targets": [ + "aarch64-apple-ios", + "x86_64-apple-ios", + "aarch64-apple-darwin", + "x86_64-apple-darwin" + ] + }, + { + "type": "gn_ninja", + "gn_args": ["--android", "--android-cpu=arm64", "--runtime-mode=release", "--gn-args=host_cpu=\"x64\""], + "ninja_targets": ["flutter/shell/platform/android:gen_snapshot"], + "out_dir": "android_release_arm64" + }, + { + "type": "gn_ninja", + "gn_args": ["--android", "--runtime-mode=release", "--gn-args=host_cpu=\"x64\""], + "ninja_targets": ["flutter/shell/platform/android:gen_snapshot"], + "out_dir": "android_release" + }, + { + "type": "gn_ninja", + "gn_args": ["--android", "--android-cpu=x64", "--runtime-mode=release", "--gn-args=host_cpu=\"x64\""], + "ninja_targets": ["flutter/shell/platform/android:gen_snapshot"], + "out_dir": "android_release_x64" + } + ], + "artifacts": [ + {"src": "android_release_arm64/zip_archives/android-arm64-release/darwin-x64.zip", "dst": "flutter_infra_release/flutter/$engine/android-arm64-release/darwin-x64.zip"}, + {"src": "android_release/zip_archives/android-arm-release/darwin-x64.zip", "dst": "flutter_infra_release/flutter/$engine/android-arm-release/darwin-x64.zip"}, + {"src": "android_release_x64/zip_archives/android-x64-release/darwin-x64.zip", "dst": "flutter_infra_release/flutter/$engine/android-x64-release/darwin-x64.zip"} + ] + }, + "ios-release": { + "steps": [ + { + "type": "gn_ninja", + "gn_args": ["--ios", "--runtime-mode=release", "--gn-arg=shorebird_runtime=true"], + "ninja_targets": ["flutter/shell/platform/darwin/ios:flutter_framework", "flutter/lib/snapshot:generate_snapshot_bins"], + "out_dir": "ios_release" + } + ], + "compose_input": "ios-framework" + }, + "ios-release-ext": { + "steps": [ + { + "type": "gn_ninja", + "gn_args": ["--ios", "--runtime-mode=release", "--darwin-extension-safe", "--xcode-symlinks", "--gn-arg=shorebird_runtime=true"], + "ninja_targets": ["flutter/shell/platform/darwin/ios:flutter_framework", "flutter/lib/snapshot:generate_snapshot_bins"], + "out_dir": "ios_release_extension_safe" + } + ], + "compose_input": "ios-framework" + }, + "ios-sim-x64": { + "steps": [ + { + "type": "gn_ninja", + "gn_args": ["--ios", "--runtime-mode=debug", "--simulator"], + "ninja_targets": ["flutter/shell/platform/darwin/ios:flutter_framework", "flutter/lib/snapshot:generate_snapshot_bins"], + "out_dir": "ios_debug_sim" + } + ], + "compose_input": "ios-framework" + }, + "ios-sim-x64-ext": { + "steps": [ + { + "type": "gn_ninja", + "gn_args": ["--ios", "--runtime-mode=debug", "--darwin-extension-safe", "--simulator"], + "ninja_targets": ["flutter/shell/platform/darwin/ios:flutter_framework", "flutter/lib/snapshot:generate_snapshot_bins"], + "out_dir": "ios_debug_sim_extension_safe" + } + ], + "compose_input": "ios-framework" + }, + "ios-sim-arm64": { + "steps": [ + { + "type": "gn_ninja", + "gn_args": ["--ios", "--runtime-mode=debug", "--simulator", "--simulator-cpu=arm64"], + "ninja_targets": ["flutter/shell/platform/darwin/ios:flutter_framework", "flutter/lib/snapshot:generate_snapshot_bins"], + "out_dir": "ios_debug_sim_arm64" + } + ], + "compose_input": "ios-framework" + }, + "ios-sim-arm64-ext": { + "steps": [ + { + "type": "gn_ninja", + "gn_args": ["--ios", "--runtime-mode=debug", "--darwin-extension-safe", "--simulator", "--simulator-cpu=arm64"], + "ninja_targets": ["flutter/shell/platform/darwin/ios:flutter_framework", "flutter/lib/snapshot:generate_snapshot_bins"], + "out_dir": "ios_debug_sim_arm64_extension_safe" + } + ], + "compose_input": "ios-framework" + }, + "mac-arm64": { + "steps": [ + { + "type": "gn_ninja", + "gn_args": ["--runtime-mode=release", "--mac-cpu=arm64", "--no-prebuilt-dart-sdk"], + "ninja_targets": ["dart_sdk"], + "out_dir": "host_release_arm64" + }, + { + "type": "gn_ninja", + "gn_args": ["--runtime-mode=release", "--mac", "--mac-cpu=arm64"], + "ninja_targets": ["flutter/shell/platform/darwin/macos:zip_macos_flutter_framework", "flutter/lib/snapshot:generate_snapshot_bins", "flutter/build/archives:artifacts"], + "out_dir": "mac_release_arm64" + } + ], + "compose_input": "macos-framework", + "artifacts": [ + {"src": "host_release_arm64/dart-sdk", "dst": "flutter_infra_release/flutter/$engine/dart-sdk-darwin-arm64.zip", "zip": true, "content_hash": true} + ] + }, + "mac-x64": { + "steps": [ + { + "type": "gn_ninja", + "gn_args": ["--runtime-mode=release", "--mac-cpu=x64", "--no-prebuilt-dart-sdk"], + "ninja_targets": ["dart_sdk"], + "out_dir": "host_release" + }, + { + "type": "gn_ninja", + "gn_args": ["--runtime-mode=release", "--mac", "--mac-cpu=x64"], + "ninja_targets": ["flutter/shell/platform/darwin/macos:zip_macos_flutter_framework", "flutter/lib/snapshot:generate_snapshot_bins", "flutter/build/archives:artifacts"], + "out_dir": "mac_release" + } + ], + "compose_input": "macos-framework", + "artifacts": [ + {"src": "host_release/dart-sdk", "dst": "flutter_infra_release/flutter/$engine/dart-sdk-darwin-x64.zip", "zip": true, "content_hash": true}, + {"src": "engine_stamp.json", "dst": "flutter_infra_release/flutter/$engine/engine_stamp.json"} + ] + } +} diff --git a/shorebird/ci/shards/windows.json b/shorebird/ci/shards/windows.json new file mode 100644 index 0000000000000..5775e5d0a1ad0 --- /dev/null +++ b/shorebird/ci/shards/windows.json @@ -0,0 +1,66 @@ +{ + "android-arm64": { + "steps": [ + { + "type": "gn_ninja", + "gn_args": ["--android", "--android-cpu=arm64", "--runtime-mode=release"], + "ninja_targets": ["archive_win_gen_snapshot"], + "out_dir": "android_release_arm64" + } + ], + "artifacts": [ + {"src": "zip_archives/android-arm64-release/windows-x64.zip", "dst": "flutter_infra_release/flutter/$engine/android-arm64-release/windows-x64.zip"} + ] + }, + "android-arm32": { + "steps": [ + { + "type": "gn_ninja", + "gn_args": ["--android", "--runtime-mode=release"], + "ninja_targets": ["archive_win_gen_snapshot"], + "out_dir": "android_release" + } + ], + "artifacts": [ + {"src": "zip_archives/android-arm-release/windows-x64.zip", "dst": "flutter_infra_release/flutter/$engine/android-arm-release/windows-x64.zip"} + ] + }, + "android-x64": { + "steps": [ + { + "type": "gn_ninja", + "gn_args": ["--android", "--android-cpu=x64", "--runtime-mode=release"], + "ninja_targets": ["archive_win_gen_snapshot"], + "out_dir": "android_release_x64" + } + ], + "artifacts": [ + {"src": "zip_archives/android-x64-release/windows-x64.zip", "dst": "flutter_infra_release/flutter/$engine/android-x64-release/windows-x64.zip"} + ] + }, + "host": { + "steps": [ + { + "type": "rust", + "targets": ["x86_64-pc-windows-msvc"] + }, + { + "type": "gn_ninja", + "gn_args": ["--runtime-mode=release", "--no-prebuilt-dart-sdk"], + "ninja_targets": ["dart_sdk", "flutter/build/archives:windows_flutter", "gen_snapshot", "windows", "flutter/build/archives:artifacts"], + "out_dir": "host_release" + }, + { + "type": "gn_ninja", + "gn_args": ["--no-prebuilt-dart-sdk"], + "ninja_targets": ["flutter/build/archives:artifacts"], + "out_dir": "host_debug" + } + ], + "artifacts": [ + {"src": "host_release/dart-sdk", "dst": "flutter_infra_release/flutter/$engine/dart-sdk-windows-x64.zip", "zip": true, "content_hash": true}, + {"src": "host_release/zip_archives/windows-x64-release/windows-x64-flutter.zip", "dst": "flutter_infra_release/flutter/$engine/windows-x64-release/windows-x64-flutter.zip"}, + {"src": "host_debug/zip_archives/windows-x64/artifacts.zip", "dst": "flutter_infra_release/flutter/$engine/windows-x64/artifacts.zip"} + ] + } +} From fd8657f64329b7cb6dfb4af970ee2114f0c0e6a0 Mon Sep 17 00:00:00 2001 From: Eric Seidel Date: Tue, 3 Feb 2026 22:40:28 -0800 Subject: [PATCH 27/95] fix: add Rust build steps to each shard that needs libupdater (#112) Each shard runs on a separate machine, so it needs its own Rust build step for the updater library. Previously only the host/android shards had Rust steps, but all shards that build libflutter need libupdater.a for their specific target triple. --- shorebird/ci/shards/linux.json | 12 ++++++++++++ shorebird/ci/shards/macos.json | 32 ++++++++++++++++++++++++++++++++ 2 files changed, 44 insertions(+) diff --git a/shorebird/ci/shards/linux.json b/shorebird/ci/shards/linux.json index 453ede1e851bc..51b0aa86f0b6b 100644 --- a/shorebird/ci/shards/linux.json +++ b/shorebird/ci/shards/linux.json @@ -1,6 +1,10 @@ { "android-arm64": { "steps": [ + { + "type": "rust", + "targets": ["aarch64-linux-android"] + }, { "type": "gn_ninja", "gn_args": ["--android", "--android-cpu=arm64", "--runtime-mode=release"], @@ -19,6 +23,10 @@ }, "android-arm32": { "steps": [ + { + "type": "rust", + "targets": ["armv7-linux-androideabi"] + }, { "type": "gn_ninja", "gn_args": ["--android", "--runtime-mode=release"], @@ -40,6 +48,10 @@ }, "android-x64": { "steps": [ + { + "type": "rust", + "targets": ["x86_64-linux-android"] + }, { "type": "gn_ninja", "gn_args": ["--android", "--android-cpu=x64", "--runtime-mode=release"], diff --git a/shorebird/ci/shards/macos.json b/shorebird/ci/shards/macos.json index a1ebf2d50a26e..1132a95365706 100644 --- a/shorebird/ci/shards/macos.json +++ b/shorebird/ci/shards/macos.json @@ -37,6 +37,10 @@ }, "ios-release": { "steps": [ + { + "type": "rust", + "targets": ["aarch64-apple-ios"] + }, { "type": "gn_ninja", "gn_args": ["--ios", "--runtime-mode=release", "--gn-arg=shorebird_runtime=true"], @@ -48,6 +52,10 @@ }, "ios-release-ext": { "steps": [ + { + "type": "rust", + "targets": ["aarch64-apple-ios"] + }, { "type": "gn_ninja", "gn_args": ["--ios", "--runtime-mode=release", "--darwin-extension-safe", "--xcode-symlinks", "--gn-arg=shorebird_runtime=true"], @@ -59,6 +67,10 @@ }, "ios-sim-x64": { "steps": [ + { + "type": "rust", + "targets": ["x86_64-apple-ios"] + }, { "type": "gn_ninja", "gn_args": ["--ios", "--runtime-mode=debug", "--simulator"], @@ -70,6 +82,10 @@ }, "ios-sim-x64-ext": { "steps": [ + { + "type": "rust", + "targets": ["x86_64-apple-ios"] + }, { "type": "gn_ninja", "gn_args": ["--ios", "--runtime-mode=debug", "--darwin-extension-safe", "--simulator"], @@ -81,6 +97,10 @@ }, "ios-sim-arm64": { "steps": [ + { + "type": "rust", + "targets": ["aarch64-apple-ios-sim"] + }, { "type": "gn_ninja", "gn_args": ["--ios", "--runtime-mode=debug", "--simulator", "--simulator-cpu=arm64"], @@ -92,6 +112,10 @@ }, "ios-sim-arm64-ext": { "steps": [ + { + "type": "rust", + "targets": ["aarch64-apple-ios-sim"] + }, { "type": "gn_ninja", "gn_args": ["--ios", "--runtime-mode=debug", "--darwin-extension-safe", "--simulator", "--simulator-cpu=arm64"], @@ -103,6 +127,10 @@ }, "mac-arm64": { "steps": [ + { + "type": "rust", + "targets": ["aarch64-apple-darwin"] + }, { "type": "gn_ninja", "gn_args": ["--runtime-mode=release", "--mac-cpu=arm64", "--no-prebuilt-dart-sdk"], @@ -123,6 +151,10 @@ }, "mac-x64": { "steps": [ + { + "type": "rust", + "targets": ["x86_64-apple-darwin"] + }, { "type": "gn_ninja", "gn_args": ["--runtime-mode=release", "--mac-cpu=x64", "--no-prebuilt-dart-sdk"], From 995eaa685a23e19cfeb9ca23930ccce055d01ca3 Mon Sep 17 00:00:00 2001 From: Eric Seidel Date: Wed, 4 Feb 2026 07:13:01 -0800 Subject: [PATCH 28/95] fix: remove duplicate --help flag in finalize.dart --- shorebird/ci/shard_runner/bin/finalize.dart | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/shorebird/ci/shard_runner/bin/finalize.dart b/shorebird/ci/shard_runner/bin/finalize.dart index 5a6dae8a923ee..6534aedec8162 100644 --- a/shorebird/ci/shard_runner/bin/finalize.dart +++ b/shorebird/ci/shard_runner/bin/finalize.dart @@ -28,8 +28,7 @@ Future main(List args) async { ..addFlag('download', defaultsTo: true, help: 'Download artifacts from GCS staging') ..addFlag('upload', - defaultsTo: true, help: 'Upload artifacts to GCS bucket') - ..addFlag('help', abbr: 'h', negatable: false, help: 'Show this help'); + defaultsTo: true, help: 'Upload artifacts to GCS bucket'); CliConfig.addCommonOptions(parser, includeUpload: false); From 37f15d3515ac0dfcdce9d2be321f70bc4d6b14a7 Mon Sep 17 00:00:00 2001 From: Eric Seidel Date: Wed, 4 Feb 2026 19:56:33 -0800 Subject: [PATCH 29/95] fix: resolve .cmd executables on Windows for gsutil/gcloud On Windows, gcloud SDK tools like gsutil are installed as .cmd files. When Dart's Process.run is called without runInShell, it doesn't resolve these .cmd extensions. This adds a helper that explicitly checks for .cmd versions in PATH on Windows. --- shorebird/ci/shard_runner/lib/process.dart | 31 +++++++++++++++++++++- 1 file changed, 30 insertions(+), 1 deletion(-) diff --git a/shorebird/ci/shard_runner/lib/process.dart b/shorebird/ci/shard_runner/lib/process.dart index 897714d0d3d16..aefb1a118fc8b 100644 --- a/shorebird/ci/shard_runner/lib/process.dart +++ b/shorebird/ci/shard_runner/lib/process.dart @@ -1,5 +1,32 @@ import 'dart:io'; +/// Resolves an executable name for Windows, appending .cmd if needed. +/// +/// On Windows, many tools like gsutil and gcloud are installed as .cmd files. +/// This function checks if a .cmd version exists and uses it. +String _resolveExecutable(String executable) { + if (!Platform.isWindows) return executable; + + // Don't modify if it already has an extension + if (executable.endsWith('.exe') || executable.endsWith('.cmd') || + executable.endsWith('.bat')) { + return executable; + } + + // Check if .cmd version exists in PATH + final String? path = Platform.environment['PATH']; + if (path == null) return executable; + + for (final String dir in path.split(';')) { + final File cmdFile = File('$dir\\$executable.cmd'); + if (cmdFile.existsSync()) { + return '$executable.cmd'; + } + } + + return executable; +} + /// Runs a process and throws if it exits with a non-zero code. /// /// Returns the [ProcessResult] for callers that need stdout/stderr. @@ -10,8 +37,10 @@ Future runChecked( Map? environment, String? description, }) async { + final String resolvedExecutable = _resolveExecutable(executable); + final ProcessResult result = await Process.run( - executable, + resolvedExecutable, arguments, workingDirectory: workingDirectory, environment: environment, From 7b227627780d2449176d4a13affe75f137bf1167 Mon Sep 17 00:00:00 2001 From: Eric Seidel Date: Wed, 4 Feb 2026 20:19:43 -0800 Subject: [PATCH 30/95] style: format process.dart --- shorebird/ci/shard_runner/lib/process.dart | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/shorebird/ci/shard_runner/lib/process.dart b/shorebird/ci/shard_runner/lib/process.dart index aefb1a118fc8b..1fb251b609d97 100644 --- a/shorebird/ci/shard_runner/lib/process.dart +++ b/shorebird/ci/shard_runner/lib/process.dart @@ -8,7 +8,8 @@ String _resolveExecutable(String executable) { if (!Platform.isWindows) return executable; // Don't modify if it already has an extension - if (executable.endsWith('.exe') || executable.endsWith('.cmd') || + if (executable.endsWith('.exe') || + executable.endsWith('.cmd') || executable.endsWith('.bat')) { return executable; } From 0d08e2c1d77fba5915f1a025a43c8256e491b3eb Mon Sep 17 00:00:00 2001 From: Eric Seidel Date: Wed, 4 Feb 2026 22:05:22 -0800 Subject: [PATCH 31/95] fix: remove Rust steps from iOS simulator shards Simulators don't currently support Shorebird's Rust components. Remove the rust build steps from ios-sim-x64, ios-sim-x64-ext, ios-sim-arm64, and ios-sim-arm64-ext shards. --- shorebird/ci/shards/macos.json | 16 ---------------- 1 file changed, 16 deletions(-) diff --git a/shorebird/ci/shards/macos.json b/shorebird/ci/shards/macos.json index 1132a95365706..1112d0a5f01ee 100644 --- a/shorebird/ci/shards/macos.json +++ b/shorebird/ci/shards/macos.json @@ -67,10 +67,6 @@ }, "ios-sim-x64": { "steps": [ - { - "type": "rust", - "targets": ["x86_64-apple-ios"] - }, { "type": "gn_ninja", "gn_args": ["--ios", "--runtime-mode=debug", "--simulator"], @@ -82,10 +78,6 @@ }, "ios-sim-x64-ext": { "steps": [ - { - "type": "rust", - "targets": ["x86_64-apple-ios"] - }, { "type": "gn_ninja", "gn_args": ["--ios", "--runtime-mode=debug", "--darwin-extension-safe", "--simulator"], @@ -97,10 +89,6 @@ }, "ios-sim-arm64": { "steps": [ - { - "type": "rust", - "targets": ["aarch64-apple-ios-sim"] - }, { "type": "gn_ninja", "gn_args": ["--ios", "--runtime-mode=debug", "--simulator", "--simulator-cpu=arm64"], @@ -112,10 +100,6 @@ }, "ios-sim-arm64-ext": { "steps": [ - { - "type": "rust", - "targets": ["aarch64-apple-ios-sim"] - }, { "type": "gn_ninja", "gn_args": ["--ios", "--runtime-mode=debug", "--darwin-extension-safe", "--simulator", "--simulator-cpu=arm64"], From e29e64f8fa06f9da201ec572f6a23c05d8ddb71b Mon Sep 17 00:00:00 2001 From: Eric Seidel Date: Wed, 4 Feb 2026 23:09:12 -0800 Subject: [PATCH 32/95] fix: restore Rust for x64 iOS simulator shards The x64 simulator shards need libupdater.a built for x86_64-apple-ios. The arm64 simulator shards don't require Rust (mac_build.sh doesn't build aarch64-apple-ios-sim either). --- shorebird/ci/shards/macos.json | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/shorebird/ci/shards/macos.json b/shorebird/ci/shards/macos.json index 1112d0a5f01ee..db68dc3e1a2f8 100644 --- a/shorebird/ci/shards/macos.json +++ b/shorebird/ci/shards/macos.json @@ -67,6 +67,10 @@ }, "ios-sim-x64": { "steps": [ + { + "type": "rust", + "targets": ["x86_64-apple-ios"] + }, { "type": "gn_ninja", "gn_args": ["--ios", "--runtime-mode=debug", "--simulator"], @@ -78,6 +82,10 @@ }, "ios-sim-x64-ext": { "steps": [ + { + "type": "rust", + "targets": ["x86_64-apple-ios"] + }, { "type": "gn_ninja", "gn_args": ["--ios", "--runtime-mode=debug", "--darwin-extension-safe", "--simulator"], From ee182e386ce12da8ff1bcde4a506f4f8c9784865 Mon Sep 17 00:00:00 2001 From: Brandon DeRosier Date: Tue, 10 Feb 2026 19:22:10 -0800 Subject: [PATCH 33/95] fix: update NDK path for unmodified Android SDK CIPD package The upstream Flutter commit c0b808c9ed3 changed the Android SDK CIPD package to an "unmodified" layout where the NDK lives at android_tools/sdk/ndk/ instead of android_tools/ndk. Update all build scripts to dynamically discover the NDK version directory. --- shorebird/ci/internal/linux_build.sh | 3 ++- shorebird/ci/internal/linux_test_build.sh | 2 +- shorebird/ci/shard_runner/lib/config.dart | 10 ++++++++-- 3 files changed, 11 insertions(+), 4 deletions(-) diff --git a/shorebird/ci/internal/linux_build.sh b/shorebird/ci/internal/linux_build.sh index 06d995f2fc5d2..9f8cbf13baa1b 100755 --- a/shorebird/ci/internal/linux_build.sh +++ b/shorebird/ci/internal/linux_build.sh @@ -25,7 +25,8 @@ cd $UPDATER_SRC/library # previous iterations of cargo-ndk required the version to be passed as # -p , but that no longer seems needed. # We always use the hermetic NDK from the engine repo. -ANDROID_NDK_HOME="$ENGINE_SRC/flutter/third_party/android_tools/ndk" \ +# The "unmodified" CIPD package keeps the NDK at the standard Android SDK path. +ANDROID_NDK_HOME=$(echo "$ENGINE_SRC/flutter/third_party/android_tools/sdk/ndk"/*) \ cargo ndk \ --target armv7-linux-androideabi \ --target aarch64-linux-android \ diff --git a/shorebird/ci/internal/linux_test_build.sh b/shorebird/ci/internal/linux_test_build.sh index be3c54da547ec..5d5fb9b0b2538 100755 --- a/shorebird/ci/internal/linux_test_build.sh +++ b/shorebird/ci/internal/linux_test_build.sh @@ -10,7 +10,7 @@ cd $ENGINE_SRC UPDATER_SRC=$ENGINE_SRC/flutter/third_party/updater (cd $UPDATER_SRC && - ANDROID_NDK_HOME="$ENGINE_SRC/flutter/third_party/android_tools/ndk" \ + ANDROID_NDK_HOME=$(echo "$ENGINE_SRC/flutter/third_party/android_tools/sdk/ndk"/*) \ cargo ndk \ --target armv7-linux-androideabi \ --target aarch64-linux-android \ diff --git a/shorebird/ci/shard_runner/lib/config.dart b/shorebird/ci/shard_runner/lib/config.dart index 378e463f96712..7cbcd01e681ff 100644 --- a/shorebird/ci/shard_runner/lib/config.dart +++ b/shorebird/ci/shard_runner/lib/config.dart @@ -224,13 +224,19 @@ Future _runRust(String engineSrc, List targets) async { } args.addAll(['build', '--release']); + // The "unmodified" CIPD package keeps the NDK at the standard Android + // SDK path: android_tools/sdk/ndk/. + final Directory ndkParent = Directory( + p.join(engineSrc, 'flutter', 'third_party', 'android_tools', 'sdk', 'ndk'), + ); + final String ndkHome = ndkParent.listSync().whereType().first.path; + await runChecked( 'cargo', args, workingDirectory: updaterPath, environment: { - 'ANDROID_NDK_HOME': - p.join(engineSrc, 'flutter', 'third_party', 'android_tools', 'ndk'), + 'ANDROID_NDK_HOME': ndkHome, }, description: 'Cargo ndk (${androidTargets.join(', ')})', ); From 3b15100734e00de2239e124bdd317026fd4b88a2 Mon Sep 17 00:00:00 2001 From: Brandon DeRosier Date: Wed, 25 Feb 2026 02:40:07 -0800 Subject: [PATCH 34/95] fix: format shard_runner config.dart --- shorebird/ci/shard_runner/lib/config.dart | 68 ++++++++++++++--------- 1 file changed, 42 insertions(+), 26 deletions(-) diff --git a/shorebird/ci/shard_runner/lib/config.dart b/shorebird/ci/shard_runner/lib/config.dart index 7cbcd01e681ff..29448199d1797 100644 --- a/shorebird/ci/shard_runner/lib/config.dart +++ b/shorebird/ci/shard_runner/lib/config.dart @@ -24,7 +24,8 @@ class PlatformConfig { final ShardDef? shard = shards[name]; if (shard == null) { throw ArgumentError( - 'Unknown shard: $name. Available: ${shards.keys.join(', ')}'); + 'Unknown shard: $name. Available: ${shards.keys.join(', ')}', + ); } return shard; } @@ -44,17 +45,19 @@ class PlatformConfig { /// Definition of a single build shard. @immutable class ShardDef { - ShardDef( - {required this.steps, - this.composeInput, - this.artifacts = const []}); + ShardDef({ + required this.steps, + this.composeInput, + this.artifacts = const [], + }); factory ShardDef.fromJson(Map json) { final List steps = (json['steps'] as List) .map((s) => BuildStep.fromJson(s as Map)) .toList(); - final List artifacts = (json['artifacts'] as List?) + final List artifacts = + (json['artifacts'] as List?) ?.map((a) => ArtifactDef.fromJson(a as Map)) .toList() ?? []; @@ -157,9 +160,7 @@ class RustStep implements BuildStep { RustStep({required this.targets}); factory RustStep.fromJson(Map json) { - return RustStep( - targets: (json['targets'] as List).cast(), - ); + return RustStep(targets: (json['targets'] as List).cast()); } final List targets; @@ -189,15 +190,14 @@ Future _runGn(String engineSrc, List args, String outDir) async { } Future _runNinja( - String engineSrc, String outDir, List targets) async { + String engineSrc, + String outDir, + List targets, +) async { print('[Ninja] Building ${targets.join(' ')} in out/$outDir'); await runChecked( 'ninja', - [ - '-C', - p.join(engineSrc, 'out', outDir), - ...targets, - ], + ['-C', p.join(engineSrc, 'out', outDir), ...targets], workingDirectory: engineSrc, description: 'Ninja ($outDir)', ); @@ -205,14 +205,21 @@ Future _runNinja( } Future _runRust(String engineSrc, List targets) async { - final String updaterPath = - p.join(engineSrc, 'flutter', 'third_party', 'updater', 'library'); + final String updaterPath = p.join( + engineSrc, + 'flutter', + 'third_party', + 'updater', + 'library', + ); // Separate Android and non-Android targets - final List androidTargets = - targets.where((String t) => t.contains('android')).toList(); - final List otherTargets = - targets.where((String t) => !t.contains('android')).toList(); + final List androidTargets = targets + .where((String t) => t.contains('android')) + .toList(); + final List otherTargets = targets + .where((String t) => !t.contains('android')) + .toList(); // Build all Android targets together with cargo-ndk if (androidTargets.isNotEmpty) { @@ -227,17 +234,26 @@ Future _runRust(String engineSrc, List targets) async { // The "unmodified" CIPD package keeps the NDK at the standard Android // SDK path: android_tools/sdk/ndk/. final Directory ndkParent = Directory( - p.join(engineSrc, 'flutter', 'third_party', 'android_tools', 'sdk', 'ndk'), + p.join( + engineSrc, + 'flutter', + 'third_party', + 'android_tools', + 'sdk', + 'ndk', + ), ); - final String ndkHome = ndkParent.listSync().whereType().first.path; + final String ndkHome = ndkParent + .listSync() + .whereType() + .first + .path; await runChecked( 'cargo', args, workingDirectory: updaterPath, - environment: { - 'ANDROID_NDK_HOME': ndkHome, - }, + environment: {'ANDROID_NDK_HOME': ndkHome}, description: 'Cargo ndk (${androidTargets.join(', ')})', ); } From 875632adc1cda75554f55016c2489c899f2dc9e7 Mon Sep 17 00:00:00 2001 From: Brandon DeRosier Date: Wed, 11 Mar 2026 23:02:48 -0700 Subject: [PATCH 35/95] fix: format shard_runner config.dart for stable SDK formatter --- shorebird/ci/shard_runner/lib/config.dart | 20 +++++++------------- 1 file changed, 7 insertions(+), 13 deletions(-) diff --git a/shorebird/ci/shard_runner/lib/config.dart b/shorebird/ci/shard_runner/lib/config.dart index 29448199d1797..9fecdabe1cfd2 100644 --- a/shorebird/ci/shard_runner/lib/config.dart +++ b/shorebird/ci/shard_runner/lib/config.dart @@ -56,8 +56,7 @@ class ShardDef { .map((s) => BuildStep.fromJson(s as Map)) .toList(); - final List artifacts = - (json['artifacts'] as List?) + final List artifacts = (json['artifacts'] as List?) ?.map((a) => ArtifactDef.fromJson(a as Map)) .toList() ?? []; @@ -214,12 +213,10 @@ Future _runRust(String engineSrc, List targets) async { ); // Separate Android and non-Android targets - final List androidTargets = targets - .where((String t) => t.contains('android')) - .toList(); - final List otherTargets = targets - .where((String t) => !t.contains('android')) - .toList(); + final List androidTargets = + targets.where((String t) => t.contains('android')).toList(); + final List otherTargets = + targets.where((String t) => !t.contains('android')).toList(); // Build all Android targets together with cargo-ndk if (androidTargets.isNotEmpty) { @@ -243,11 +240,8 @@ Future _runRust(String engineSrc, List targets) async { 'ndk', ), ); - final String ndkHome = ndkParent - .listSync() - .whereType() - .first - .path; + final String ndkHome = + ndkParent.listSync().whereType().first.path; await runChecked( 'cargo', From 0bcc085f56e6af77f60c7bf61c8319aa15e55d5b Mon Sep 17 00:00:00 2001 From: Eric Seidel Date: Fri, 13 Mar 2026 11:13:46 -0700 Subject: [PATCH 36/95] fix: Update iOS dSYM filename in artifact manifest (#113) The manifest template still referenced Flutter.dSYM.zip but mac_upload.sh uploads Flutter.framework.dSYM.zip (the new name as of Flutter 3.27.0). Update the template, generate script, and test to match what's actually uploaded. Relates to https://github.com/shorebirdtech/shorebird/issues/3035 --- shorebird/ci/artifacts_manifest.template.yaml | 2 +- shorebird/ci/internal/generate_manifest.sh | 2 +- shorebird/ci/shard_runner/test/manifest_test.dart | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/shorebird/ci/artifacts_manifest.template.yaml b/shorebird/ci/artifacts_manifest.template.yaml index 4b5c8a7f0809d..33223fced2de8 100644 --- a/shorebird/ci/artifacts_manifest.template.yaml +++ b/shorebird/ci/artifacts_manifest.template.yaml @@ -49,7 +49,7 @@ artifact_overrides: # iOS release artifacts - flutter_infra_release/flutter/$engine/ios-release/artifacts.zip - - flutter_infra_release/flutter/$engine/ios-release/Flutter.dSYM.zip + - flutter_infra_release/flutter/$engine/ios-release/Flutter.framework.dSYM.zip # Linux release artifacts - flutter_infra_release/flutter/$engine/linux-x64/artifacts.zip diff --git a/shorebird/ci/internal/generate_manifest.sh b/shorebird/ci/internal/generate_manifest.sh index 4c95de6baf098..ee9ef7ddfdf67 100755 --- a/shorebird/ci/internal/generate_manifest.sh +++ b/shorebird/ci/internal/generate_manifest.sh @@ -76,7 +76,7 @@ artifact_overrides: # iOS release artifacts # Includes unified Flutter.framework for device and simulator (debug) - flutter_infra_release/flutter/\$engine/ios-release/artifacts.zip - - flutter_infra_release/flutter/\$engine/ios-release/Flutter.dSYM.zip + - flutter_infra_release/flutter/\$engine/ios-release/Flutter.framework.dSYM.zip # Linux release artifacts - flutter_infra_release/flutter/\$engine/linux-x64/artifacts.zip diff --git a/shorebird/ci/shard_runner/test/manifest_test.dart b/shorebird/ci/shard_runner/test/manifest_test.dart index 1e8e930eb2458..a5606ae9b2f54 100644 --- a/shorebird/ci/shard_runner/test/manifest_test.dart +++ b/shorebird/ci/shard_runner/test/manifest_test.dart @@ -123,7 +123,7 @@ void main() { expect( manifest, contains( - r'flutter_infra_release/flutter/$engine/ios-release/Flutter.dSYM.zip')); + r'flutter_infra_release/flutter/$engine/ios-release/Flutter.framework.dSYM.zip')); }); test('includes Linux release artifacts', () { From 4b762a5bc71210f831767e5ed2d255a4f79a172b Mon Sep 17 00:00:00 2001 From: Brandon DeRosier Date: Thu, 26 Mar 2026 22:55:51 -0700 Subject: [PATCH 37/95] fix: filter non-stable version branches in shorebird version detection (#117) * fix: filter non-stable version branches in shorebird version detection The shorebird version detection in GitTagVersion.determine() matches flutter_release/* branches to resolve the Flutter version. When non-stable branch names like flutter_release/3.41.4-rc2 are present, parse() fails to match them against the expected version pattern and returns GitTagVersion.unknown(), causing Flutter to report 0.0.0-unknown. Filter branch names to only match stable versions (X.Y.Z) so that release candidate branches are skipped and the correct version is resolved. Fixes shorebirdtech/shorebird#3662 * test: add tests for shorebird flutter_release branch version filtering Tests that: - A stable flutter_release branch (e.g. 3.41.4) resolves correctly - A non-stable branch (e.g. 3.41.4-rc2) is skipped, falling through to the upstream tag-based fallback - When both stable and rc branches match, the stable one is picked --- packages/flutter_tools/lib/src/version.dart | 5 +- .../test/general.shard/version_test.dart | 114 ++++++++++++++++++ 2 files changed, 118 insertions(+), 1 deletion(-) diff --git a/packages/flutter_tools/lib/src/version.dart b/packages/flutter_tools/lib/src/version.dart index 1fef96d251f17..8ef96778ad616 100644 --- a/packages/flutter_tools/lib/src/version.dart +++ b/packages/flutter_tools/lib/src/version.dart @@ -1098,9 +1098,12 @@ class GitTagVersion { ], workingDirectory: workingDirectory) .stdout .trim(); + final stableVersionPattern = RegExp(r'^\d+\.\d+\.\d+$'); final String? shorebirdFlutterVersion = LineSplitter.split( shorebirdFlutterReleases, - ).map((e) => e.replaceFirst('origin/flutter_release/', '')).toList().firstOrNull; + ).map((e) => e.replaceFirst('origin/flutter_release/', '')).where( + (e) => stableVersionPattern.hasMatch(e), + ).toList().firstOrNull; if (shorebirdFlutterVersion != null) { return parse(shorebirdFlutterVersion); } diff --git a/packages/flutter_tools/test/general.shard/version_test.dart b/packages/flutter_tools/test/general.shard/version_test.dart index e52ecd768348e..a81f119bb1602 100644 --- a/packages/flutter_tools/test/general.shard/version_test.dart +++ b/packages/flutter_tools/test/general.shard/version_test.dart @@ -1527,6 +1527,120 @@ void main() { expect(processManager, hasNoRemainingExpectations); }, overrides: {Git: () => git}); + testUsingContext('determine resolves version from shorebird flutter_release branch', () { + processManager.addCommands([ + const FakeCommand( + command: ['git', 'tag', '--points-at', 'HEAD'], + // No tag at HEAD. + ), + const FakeCommand( + command: [ + 'git', + 'for-each-ref', + '--contains', + 'HEAD', + '--format', + '%(refname:short)', + 'refs/remotes/origin/flutter_release/*', + ], + stdout: 'origin/flutter_release/3.41.4', + ), + ]); + final platform = FakePlatform(); + + final GitTagVersion gitTagVersion = GitTagVersion.determine( + platform, + git: git, + workingDirectory: '.', + ); + expect(gitTagVersion.frameworkVersionFor('abcd1234'), '3.41.4'); + expect(processManager, hasNoRemainingExpectations); + }); + + testUsingContext('determine skips non-stable shorebird flutter_release branches', () { + const headRevision = 'abcd1234'; + processManager.addCommands([ + const FakeCommand( + command: ['git', 'tag', '--points-at', 'HEAD'], + // No tag at HEAD. + ), + // Shorebird release branch check returns only a non-stable branch. + const FakeCommand( + command: [ + 'git', + 'for-each-ref', + '--contains', + 'HEAD', + '--format', + '%(refname:short)', + 'refs/remotes/origin/flutter_release/*', + ], + stdout: 'origin/flutter_release/3.41.4-rc2', + ), + // Falls through to tag-based fallback. + const FakeCommand( + command: [ + 'git', + 'for-each-ref', + '--sort=-v:refname', + '--count=1', + '--format=%(refname:short)', + 'refs/tags/[0-9]*.*.*', + ], + stdout: '3.41.4', + ), + const FakeCommand( + command: ['git', 'merge-base', 'HEAD', '3.41.4'], + stdout: headRevision, + ), + const FakeCommand( + command: ['git', 'rev-list', '--count', '$headRevision..HEAD'], + stdout: '48', + ), + ]); + final platform = FakePlatform(); + + final GitTagVersion gitTagVersion = GitTagVersion.determine( + platform, + git: git, + workingDirectory: '.', + ); + // Should NOT be 0.0.0-unknown; the tag fallback should produce a valid version. + expect(gitTagVersion.frameworkVersionFor(headRevision), '3.41.5-0.0.pre-48'); + expect(processManager, hasNoRemainingExpectations); + }); + + testUsingContext('determine picks stable branch over rc branch from shorebird flutter_release', () { + processManager.addCommands([ + const FakeCommand( + command: ['git', 'tag', '--points-at', 'HEAD'], + // No tag at HEAD. + ), + const FakeCommand( + command: [ + 'git', + 'for-each-ref', + '--contains', + 'HEAD', + '--format', + '%(refname:short)', + 'refs/remotes/origin/flutter_release/*', + ], + // Both stable and rc branches contain this commit. + stdout: 'origin/flutter_release/3.41.4\norigin/flutter_release/3.41.4-rc2', + ), + ]); + final platform = FakePlatform(); + + final GitTagVersion gitTagVersion = GitTagVersion.determine( + platform, + git: git, + workingDirectory: '.', + ); + expect(gitTagVersion.frameworkVersionFor('abcd1234'), '3.41.4'); + expect(processManager, hasNoRemainingExpectations); + }); + group('$FlutterEngineStampFromFile', () { late FileSystem fs; const flutterRoot = '/path/to/flutter'; From 7906e68ab14eeb06c957e86e6ecfa4570394041c Mon Sep 17 00:00:00 2001 From: Brandon DeRosier Date: Wed, 1 Apr 2026 14:03:34 -0700 Subject: [PATCH 38/95] Integrate Rust updater build into GN/Ninja (#120) * Integrate Rust updater build into GN/Ninja Add a GN action() that invokes cargo to build the Rust updater library, so that Ninja (and ET) can build the complete Shorebird engine without requiring a separate shell script prerequisite step. * Fix missing updater dep in runtime BUILD.gn dart_snapshot.cc calls shorebird::Updater::Instance().ReportLaunchStart() but the runtime source_set did not depend on shorebird:updater, causing link failures in test targets that depend on runtime without also pulling in the updater through other deps. * Fix relative NDK path resolution for Android cargo builds GN passes paths relative to the build output dir, but cargo's build scripts (e.g. ring's cc crate) run from a different directory. Resolve all paths to absolute before passing them to cargo/env vars. * Extract updater build config into .gni and glob Rust sources Move triple computation and variables into build_rust_updater.gni to keep BUILD.gn focused on target definitions. Add list_rust_files.py to automatically discover .rs source files at GN gen time via exec_script, so the inputs list stays in sync when the updater repo changes without manual updates. --- engine/src/flutter/runtime/BUILD.gn | 1 + .../flutter/shell/common/shorebird/BUILD.gn | 82 ++++++++----- .../common/shorebird/build_rust_updater.gni | 71 +++++++++++ .../common/shorebird/build_rust_updater.py | 116 ++++++++++++++++++ .../shell/common/shorebird/list_rust_files.py | 28 +++++ 5 files changed, 265 insertions(+), 33 deletions(-) create mode 100644 engine/src/flutter/shell/common/shorebird/build_rust_updater.gni create mode 100644 engine/src/flutter/shell/common/shorebird/build_rust_updater.py create mode 100644 engine/src/flutter/shell/common/shorebird/list_rust_files.py diff --git a/engine/src/flutter/runtime/BUILD.gn b/engine/src/flutter/runtime/BUILD.gn index 1d35652477aca..259ad3b4b697f 100644 --- a/engine/src/flutter/runtime/BUILD.gn +++ b/engine/src/flutter/runtime/BUILD.gn @@ -114,6 +114,7 @@ source_set("runtime") { "//flutter/fml", "//flutter/lib/io", "//flutter/shell/common:display", + "//flutter/shell/common/shorebird:updater", "//flutter/skia", "//flutter/third_party/tonic", "//flutter/txt", diff --git a/engine/src/flutter/shell/common/shorebird/BUILD.gn b/engine/src/flutter/shell/common/shorebird/BUILD.gn index 2b1e29c03512d..66695791bd402 100644 --- a/engine/src/flutter/shell/common/shorebird/BUILD.gn +++ b/engine/src/flutter/shell/common/shorebird/BUILD.gn @@ -1,4 +1,5 @@ import("//flutter/common/config.gni") +import("//flutter/shell/common/shorebird/build_rust_updater.gni") import("//flutter/testing/testing.gni") source_set("snapshots_data_handle") { @@ -15,6 +16,48 @@ source_set("snapshots_data_handle") { ] } +if (shorebird_updater_supported) { + action("build_rust_updater") { + script = "//flutter/shell/common/shorebird/build_rust_updater.py" + + # The stamp file is the declared output for Ninja dependency tracking. + _stamp = + "$target_gen_dir/rust_updater_${shorebird_updater_rust_target}.stamp" + outputs = [ _stamp ] + + args = [ + "--rust-target", + shorebird_updater_rust_target, + "--manifest-dir", + rebase_path("$shorebird_updater_dir", root_build_dir), + "--output-lib", + rebase_path("$shorebird_updater_output_lib", root_build_dir), + "--stamp", + rebase_path(_stamp, root_build_dir), + ] + + if (is_android) { + args += [ + "--ndk-path", + rebase_path(android_ndk_root, root_build_dir), + "--android-api-level", + "$android_api_level", + ] + } + + # Declare inputs so Ninja knows when to re-run the action. + inputs = [ + "$shorebird_updater_dir/Cargo.toml", + "$shorebird_updater_dir/Cargo.lock", + "$shorebird_updater_dir/library/Cargo.toml", + "$shorebird_updater_dir/library/build.rs", + "$shorebird_updater_dir/library/cbindgen.toml", + "$shorebird_updater_dir/library/.cargo/config.toml", + ] + inputs += shorebird_updater_rs_sources + } +} + # C++ wrapper around the Rust updater C API. # This provides a testable abstraction layer that can be mocked for testing. source_set("updater") { @@ -28,39 +71,12 @@ source_set("updater") { # For the Rust updater C API (shorebird_report_launch_start, etc.) include_dirs = [ "//flutter" ] - # Link the Rust updater static library based on target platform. - if (is_android) { - if (target_cpu == "arm") { - libs = [ "//flutter/third_party/updater/target/armv7-linux-androideabi/release/libupdater.a" ] - } else if (target_cpu == "arm64") { - libs = [ "//flutter/third_party/updater/target/aarch64-linux-android/release/libupdater.a" ] - } else if (target_cpu == "x64") { - libs = [ "//flutter/third_party/updater/target/x86_64-linux-android/release/libupdater.a" ] - } else if (target_cpu == "x86") { - libs = [ "//flutter/third_party/updater/target/i686-linux-android/release/libupdater.a" ] - } - } else if (is_ios) { - if (target_cpu == "arm64") { - libs = [ "//flutter/third_party/updater/target/aarch64-apple-ios/release/libupdater.a" ] - } else if (target_cpu == "x64") { - libs = [ "//flutter/third_party/updater/target/x86_64-apple-ios/release/libupdater.a" ] - } - } else if (is_mac) { - if (target_cpu == "arm64") { - libs = [ "//flutter/third_party/updater/target/aarch64-apple-darwin/release/libupdater.a" ] - } else if (target_cpu == "x64") { - libs = [ "//flutter/third_party/updater/target/x86_64-apple-darwin/release/libupdater.a" ] - } - } else if (is_win) { - if (target_cpu == "x64") { - libs = [ - "userenv.lib", - "//flutter/third_party/updater/target/x86_64-pc-windows-msvc/release/updater.lib", - ] - } - } else if (is_linux) { - if (target_cpu == "x64") { - libs = [ "//flutter/third_party/updater/target/x86_64-unknown-linux-gnu/release/libupdater.a" ] + if (shorebird_updater_supported) { + deps += [ ":build_rust_updater" ] + libs = [ shorebird_updater_output_lib ] + + if (is_win) { + libs += [ "userenv.lib" ] } } } diff --git a/engine/src/flutter/shell/common/shorebird/build_rust_updater.gni b/engine/src/flutter/shell/common/shorebird/build_rust_updater.gni new file mode 100644 index 0000000000000..147395685e35d --- /dev/null +++ b/engine/src/flutter/shell/common/shorebird/build_rust_updater.gni @@ -0,0 +1,71 @@ +# Copyright 2024 The Shorebird Authors. All rights reserved. +# Use of this source code is governed by a MIT-style license that can be +# found in the LICENSE file. + +# Computes variables needed to build and link the Rust updater library. +# +# Exported variables: +# shorebird_updater_supported - Whether the current platform is supported. +# shorebird_updater_output_lib - Path to the built static library. +# shorebird_updater_rust_target - Rust target triple for the current platform. +# shorebird_updater_rs_sources - List of .rs source files (for GN inputs). + +if (is_android) { + import("//build/config/android/config.gni") +} + +shorebird_updater_dir = "//flutter/third_party/updater" + +# Only build and link the Rust updater on supported platforms. +shorebird_updater_supported = + is_android || is_ios || is_mac || is_win || is_linux + +if (shorebird_updater_supported) { + # Compute the Rust target triple from the GN target_os/target_cpu. + if (is_android) { + if (target_cpu == "arm") { + shorebird_updater_rust_target = "armv7-linux-androideabi" + } else if (target_cpu == "arm64") { + shorebird_updater_rust_target = "aarch64-linux-android" + } else if (target_cpu == "x64") { + shorebird_updater_rust_target = "x86_64-linux-android" + } else if (target_cpu == "x86") { + shorebird_updater_rust_target = "i686-linux-android" + } + } else if (is_ios) { + if (target_cpu == "arm64") { + shorebird_updater_rust_target = "aarch64-apple-ios" + } else if (target_cpu == "x64") { + shorebird_updater_rust_target = "x86_64-apple-ios" + } + } else if (is_mac) { + if (target_cpu == "arm64") { + shorebird_updater_rust_target = "aarch64-apple-darwin" + } else if (target_cpu == "x64") { + shorebird_updater_rust_target = "x86_64-apple-darwin" + } + } else if (is_win) { + if (target_cpu == "x64") { + shorebird_updater_rust_target = "x86_64-pc-windows-msvc" + } + } else if (is_linux) { + if (target_cpu == "x64") { + shorebird_updater_rust_target = "x86_64-unknown-linux-gnu" + } + } + + if (is_win) { + _updater_lib_name = "updater.lib" + } else { + _updater_lib_name = "libupdater.a" + } + + shorebird_updater_output_lib = "$shorebird_updater_dir/target/$shorebird_updater_rust_target/release/$_updater_lib_name" + + # Glob all .rs source files so the input list stays in sync automatically. + shorebird_updater_rs_sources = + exec_script("//flutter/shell/common/shorebird/list_rust_files.py", + [ rebase_path("$shorebird_updater_dir/library/src", + root_build_dir) ], + "list lines") +} diff --git a/engine/src/flutter/shell/common/shorebird/build_rust_updater.py b/engine/src/flutter/shell/common/shorebird/build_rust_updater.py new file mode 100644 index 0000000000000..7212e4ef21348 --- /dev/null +++ b/engine/src/flutter/shell/common/shorebird/build_rust_updater.py @@ -0,0 +1,116 @@ +#!/usr/bin/env python3 +# Copyright 2024 The Shorebird Authors. All rights reserved. +# Use of this source code is governed by a MIT-style license that can be +# found in the LICENSE file. + +"""Build the Rust updater library via cargo, invoked as a GN action.""" + +import argparse +import os +import subprocess +import sys + + +def main(): + parser = argparse.ArgumentParser(description='Build the Rust updater static library.') + parser.add_argument( + '--rust-target', required=True, help='Rust target triple (e.g. aarch64-linux-android)' + ) + parser.add_argument( + '--manifest-dir', required=True, help='Directory containing the workspace Cargo.toml' + ) + parser.add_argument('--output-lib', required=True, help='Expected output library path') + parser.add_argument('--stamp', required=True, help='Stamp file to write on success') + parser.add_argument('--ndk-path', help='Path to the Android NDK (required for Android targets)') + parser.add_argument( + '--android-api-level', type=int, help='Android API level (required for Android targets)' + ) + args = parser.parse_args() + + env = os.environ.copy() + + is_android = 'android' in args.rust_target + + if is_android: + if not args.ndk_path or not args.android_api_level: + print( + 'ERROR: --ndk-path and --android-api-level are required for ' + 'Android targets.', + file=sys.stderr + ) + return 1 + _configure_android_env(env, args.rust_target, args.ndk_path, args.android_api_level) + + # GN passes paths relative to the build output dir (which is cwd when + # Ninja runs the action). Resolve them to absolute paths so they work + # regardless of cargo's working directory. + manifest_path = os.path.abspath(os.path.join(args.manifest_dir, 'Cargo.toml')) + output_lib = os.path.abspath(args.output_lib) + + cmd = [ + 'cargo', + 'build', + '--release', + '--target', + args.rust_target, + '--manifest-path', + manifest_path, + '-p', + 'updater', + ] + + print(f'Running: {" ".join(cmd)}', flush=True) + result = subprocess.run(cmd, env=env) + if result.returncode != 0: + print(f'ERROR: cargo build failed with exit code {result.returncode}', file=sys.stderr) + return result.returncode + + if not os.path.exists(output_lib): + print(f'ERROR: Expected output library not found: {output_lib}', file=sys.stderr) + return 1 + + # Write stamp file to signal success to Ninja. + with open(args.stamp, 'w') as f: + f.write('') + + return 0 + + +def _configure_android_env(env, rust_target, ndk_path, api_level): + """Set environment variables so cargo can cross-compile for Android.""" + # GN passes paths relative to the build output dir. Resolve to absolute + # so that cargo and the cc crate can find the NDK tools. + ndk_path = os.path.abspath(ndk_path) + + # Determine the host platform tag for NDK toolchain paths. + if sys.platform.startswith('linux'): + host_tag = 'linux-x86_64' + elif sys.platform == 'darwin': + host_tag = 'darwin-x86_64' + elif sys.platform == 'win32': + host_tag = 'windows-x86_64' + else: + raise RuntimeError(f'Unsupported host platform: {sys.platform}') + + toolchain_bin = os.path.join(ndk_path, 'toolchains', 'llvm', 'prebuilt', host_tag, 'bin') + + # The NDK clang binary uses a slightly different prefix for armv7. + # Rust target: armv7-linux-androideabi + # NDK prefix: armv7a-linux-androideabi + clang_prefix = rust_target + if rust_target.startswith('armv7-'): + clang_prefix = 'armv7a-' + rust_target[len('armv7-'):] + + clang = os.path.join(toolchain_bin, f'{clang_prefix}{api_level}-clang') + ar = os.path.join(toolchain_bin, 'llvm-ar') + + # Cargo looks for CARGO_TARGET__LINKER where the triple is + # upper-cased with hyphens replaced by underscores. + triple_env = rust_target.upper().replace('-', '_') + env[f'CARGO_TARGET_{triple_env}_LINKER'] = clang + env['CC'] = clang + env['AR'] = ar + + +if __name__ == '__main__': + sys.exit(main()) diff --git a/engine/src/flutter/shell/common/shorebird/list_rust_files.py b/engine/src/flutter/shell/common/shorebird/list_rust_files.py new file mode 100644 index 0000000000000..f478d7bd8e176 --- /dev/null +++ b/engine/src/flutter/shell/common/shorebird/list_rust_files.py @@ -0,0 +1,28 @@ +#!/usr/bin/env python3 +# Copyright 2024 The Shorebird Authors. All rights reserved. +# Use of this source code is governed by a MIT-style license that can be +# found in the LICENSE file. + +"""List Rust source files for GN input tracking. + +Walks a directory tree and prints all .rs files as absolute paths. Used by +GN's exec_script to generate an input list for the Rust updater build action. + +Usage: + python3 list_rust_files.py +""" + +import os +import sys + + +def main(): + directory = os.path.abspath(sys.argv[1]) + for root, _, files in os.walk(directory): + for filename in sorted(files): + if filename.endswith('.rs'): + print(os.path.join(root, filename).replace(os.sep, '/')) + + +if __name__ == '__main__': + main() From 6cb25205dddfafa33c92c0db5aecf232d82cfa4a Mon Sep 17 00:00:00 2001 From: Eric Seidel Date: Tue, 7 Apr 2026 20:03:56 -0700 Subject: [PATCH 39/95] fix: thread Apple deployment target into Rust updater build (#123) * chore: roll updater to adacb4190cb4af2e2920c18d8a904f4c6b138ee4 Rolls shorebirdtech/updater dc2cd0a..adacb41 (15 commits). Notable changes: - refactor: replace reqwest with ureq to reduce binary size (#317) - feat: add resumable downloads with streaming to disk (#313) - fix: improve inflate error handling and validate compressed patches (#314) - fix: checkForUpdate reports restartRequired when current patch is rolled back (#312) - fix: resolve all Dependabot security vulnerabilities (#316) - perf: add release profile to reduce binary size (#328) * fix: thread Apple deployment target into Rust updater build build_rust_updater.py had explicit Android cross-compile env handling but no equivalent for *-apple-ios or *-apple-darwin targets. Cargo for aarch64-apple-ios was running with no IPHONEOS_DEPLOYMENT_TARGET, which let the cc crate compile transitive C deps (zstd-sys, pulled in via ureq's compression features in the recent updater roll) against the host Xcode SDK while rustc's cdylib link step targeted iOS 10/11 (from rustc's built-in target spec). The mismatch produced loud 'built for newer iOS version (18.0) than being linked (10.0)' warnings and an unresolved ___chkstk_darwin symbol on the iOS arm64 link. Pass --ios-deployment-target / --mac-deployment-target from BUILD.gn, sourced from darwin_sdk.gni's ios_deployment_target / mac_deployment_target, so the Rust build can never drift from the C++ build's target version. Tracking: shorebirdtech/_shorebird#2014 --- .../flutter/shell/common/shorebird/BUILD.gn | 25 +++++++++++++++ .../common/shorebird/build_rust_updater.py | 32 +++++++++++++++++++ 2 files changed, 57 insertions(+) diff --git a/engine/src/flutter/shell/common/shorebird/BUILD.gn b/engine/src/flutter/shell/common/shorebird/BUILD.gn index 66695791bd402..3c854d8837f67 100644 --- a/engine/src/flutter/shell/common/shorebird/BUILD.gn +++ b/engine/src/flutter/shell/common/shorebird/BUILD.gn @@ -2,6 +2,10 @@ import("//flutter/common/config.gni") import("//flutter/shell/common/shorebird/build_rust_updater.gni") import("//flutter/testing/testing.gni") +if (is_ios || is_mac) { + import("//build/config/darwin/darwin_sdk.gni") +} + source_set("snapshots_data_handle") { sources = [ "snapshots_data_handle.cc", @@ -45,6 +49,27 @@ if (shorebird_updater_supported) { ] } + if (is_ios) { + # Thread the engine's iOS deployment target into the cargo build so + # the cc crate (compiling C deps like zstd-sys) and rustc's cdylib + # link step both target the same iOS version as the C++ engine. + # Without this, cargo defaults to whatever rustc/cc bake in (rustc's + # built-in target spec for aarch64-apple-ios is iOS 10/11, while cc + # picks up the host SDK), which causes link warnings and unresolved + # symbols like ___chkstk_darwin. + args += [ + "--ios-deployment-target", + ios_deployment_target, + ] + } + + if (is_mac) { + args += [ + "--mac-deployment-target", + mac_deployment_target, + ] + } + # Declare inputs so Ninja knows when to re-run the action. inputs = [ "$shorebird_updater_dir/Cargo.toml", diff --git a/engine/src/flutter/shell/common/shorebird/build_rust_updater.py b/engine/src/flutter/shell/common/shorebird/build_rust_updater.py index 7212e4ef21348..789721041b50b 100644 --- a/engine/src/flutter/shell/common/shorebird/build_rust_updater.py +++ b/engine/src/flutter/shell/common/shorebird/build_rust_updater.py @@ -25,11 +25,21 @@ def main(): parser.add_argument( '--android-api-level', type=int, help='Android API level (required for Android targets)' ) + parser.add_argument( + '--ios-deployment-target', + help='iOS deployment target (e.g. 13.0); required for *-apple-ios targets', + ) + parser.add_argument( + '--mac-deployment-target', + help='macOS deployment target (e.g. 10.15); required for *-apple-darwin targets', + ) args = parser.parse_args() env = os.environ.copy() is_android = 'android' in args.rust_target + is_apple_ios = 'apple-ios' in args.rust_target + is_apple_darwin = 'apple-darwin' in args.rust_target if is_android: if not args.ndk_path or not args.android_api_level: @@ -41,6 +51,28 @@ def main(): return 1 _configure_android_env(env, args.rust_target, args.ndk_path, args.android_api_level) + if is_apple_ios: + if not args.ios_deployment_target: + print( + 'ERROR: --ios-deployment-target is required for *-apple-ios targets.', + file=sys.stderr, + ) + return 1 + # Setting IPHONEOS_DEPLOYMENT_TARGET makes both the cc crate (compiling + # transitive C deps like zstd-sys) and rustc's cdylib link step honor + # the engine's iOS deployment target instead of falling back to their + # respective defaults (host SDK for cc, target-spec default for rustc). + env['IPHONEOS_DEPLOYMENT_TARGET'] = args.ios_deployment_target + + if is_apple_darwin: + if not args.mac_deployment_target: + print( + 'ERROR: --mac-deployment-target is required for *-apple-darwin targets.', + file=sys.stderr, + ) + return 1 + env['MACOSX_DEPLOYMENT_TARGET'] = args.mac_deployment_target + # GN passes paths relative to the build output dir (which is cwd when # Ninja runs the action). Resolve them to absolute paths so they work # regardless of cargo's working directory. From dde4eacf8babc506d1b780bb857db4e888a3f34c Mon Sep 17 00:00:00 2001 From: Eric Seidel Date: Tue, 7 Apr 2026 21:09:36 -0700 Subject: [PATCH 40/95] fix: put cargo target dir inside the GN build output dir (#124) build_rust_updater previously let cargo write its target/ directly into the source tree at flutter/engine/src/flutter/third_party/updater/target. Two consequences: 1. The engine's existing 'rm -rf out/' clobber on every build doesn't touch it, so compiled rust artifacts silently survived across runs and could mask source/env changes (we just hit this when a stale libzstd_sys.rlib reused on Sandpiper masked the IPHONEOS_DEPLOYMENT_TARGET fix from #123). 2. All GN configs (debug/release/ios/android/...) shared one cargo target dir, stepping on each other's rlibs. Pass --target-dir to cargo, pointing at $root_out_dir/cargo_target, and update build_rust_updater.gni's shorebird_updater_output_lib to match. The cargo workspace now lives next to the rest of the GN build output and is clobbered by the same 'rm -rf out' that has always clobbered the C++ engine artifacts. Tracking: shorebirdtech/_shorebird#2014 Follow-up to: #123 --- engine/src/flutter/shell/common/shorebird/BUILD.gn | 2 ++ .../shell/common/shorebird/build_rust_updater.gni | 11 ++++++++++- .../shell/common/shorebird/build_rust_updater.py | 8 ++++++++ 3 files changed, 20 insertions(+), 1 deletion(-) diff --git a/engine/src/flutter/shell/common/shorebird/BUILD.gn b/engine/src/flutter/shell/common/shorebird/BUILD.gn index 3c854d8837f67..04ff926fa67e8 100644 --- a/engine/src/flutter/shell/common/shorebird/BUILD.gn +++ b/engine/src/flutter/shell/common/shorebird/BUILD.gn @@ -34,6 +34,8 @@ if (shorebird_updater_supported) { shorebird_updater_rust_target, "--manifest-dir", rebase_path("$shorebird_updater_dir", root_build_dir), + "--target-dir", + rebase_path("$shorebird_updater_target_dir", root_build_dir), "--output-lib", rebase_path("$shorebird_updater_output_lib", root_build_dir), "--stamp", diff --git a/engine/src/flutter/shell/common/shorebird/build_rust_updater.gni b/engine/src/flutter/shell/common/shorebird/build_rust_updater.gni index 147395685e35d..da27527cb95bc 100644 --- a/engine/src/flutter/shell/common/shorebird/build_rust_updater.gni +++ b/engine/src/flutter/shell/common/shorebird/build_rust_updater.gni @@ -60,7 +60,16 @@ if (shorebird_updater_supported) { _updater_lib_name = "libupdater.a" } - shorebird_updater_output_lib = "$shorebird_updater_dir/target/$shorebird_updater_rust_target/release/$_updater_lib_name" + # Cargo target dir lives inside the GN build output dir, not inside the + # source tree. This means: + # - `rm -rf out/` clobbers compiled rust artifacts the same + # way it clobbers compiled C++ artifacts; no special-case cleanup + # in checkout.sh. + # - Each GN config (debug/release/ios/android/etc.) gets its own + # cargo target dir, so they don't step on each other's rlibs. + # - The source checkout stays clean, which is what GN/ninja expects. + shorebird_updater_target_dir = "$root_out_dir/cargo_target" + shorebird_updater_output_lib = "$shorebird_updater_target_dir/$shorebird_updater_rust_target/release/$_updater_lib_name" # Glob all .rs source files so the input list stays in sync automatically. shorebird_updater_rs_sources = diff --git a/engine/src/flutter/shell/common/shorebird/build_rust_updater.py b/engine/src/flutter/shell/common/shorebird/build_rust_updater.py index 789721041b50b..c6e9bf1c14875 100644 --- a/engine/src/flutter/shell/common/shorebird/build_rust_updater.py +++ b/engine/src/flutter/shell/common/shorebird/build_rust_updater.py @@ -19,6 +19,11 @@ def main(): parser.add_argument( '--manifest-dir', required=True, help='Directory containing the workspace Cargo.toml' ) + parser.add_argument( + '--target-dir', + required=True, + help='Cargo target directory; should live inside the GN build output dir', + ) parser.add_argument('--output-lib', required=True, help='Expected output library path') parser.add_argument('--stamp', required=True, help='Stamp file to write on success') parser.add_argument('--ndk-path', help='Path to the Android NDK (required for Android targets)') @@ -77,6 +82,7 @@ def main(): # Ninja runs the action). Resolve them to absolute paths so they work # regardless of cargo's working directory. manifest_path = os.path.abspath(os.path.join(args.manifest_dir, 'Cargo.toml')) + target_dir = os.path.abspath(args.target_dir) output_lib = os.path.abspath(args.output_lib) cmd = [ @@ -87,6 +93,8 @@ def main(): args.rust_target, '--manifest-path', manifest_path, + '--target-dir', + target_dir, '-p', 'updater', ] From 42b0e6bebc4b164a35b6e6444fbbce935c4a015b Mon Sep 17 00:00:00 2001 From: Eric Seidel Date: Tue, 7 Apr 2026 22:47:32 -0700 Subject: [PATCH 41/95] fix: force static CRT for cc-compiled C deps on Windows (#125) The updater's .cargo/config.toml sets '-C target-feature=+crt-static' so rustc compiles the rlib expecting static-CRT linkage. But cargo populates CARGO_CFG_TARGET_FEATURE from the rustc target spec's default features, not from user rustflags, so +crt-static is invisible to build scripts. The cc crate (used by transitive *-sys deps like zstd-sys to compile their C sources) therefore falls back to /MD, producing .obj files full of __imp_* references that the engine's /MT link cannot resolve. Symptom: engine link on Windows fails with LNK2019 unresolved external symbol __imp_clock / __imp__wassert / __imp_qsort_s from zstd-sys's zdict/cover/fastcover/divsufsort translation units, after the updater rolled to a version whose ureq stack pulls in zstd-sys via its compression features. Fix: force /MT via the per-target CFLAGS env that cc honors. cl.exe emits warning D9025 overriding '/MD' with '/MT' and uses the last one specified -- /MT wins, the resulting .obj files reference the static CRT, and the engine link succeeds. This is another instance of the toolchain-consistency problem that #123 solved for iOS deployment target. Every new platform-specific C build setting we discover needs another env var threaded through build_rust_updater.py. Longer term this argues for going rustc-direct (Fuchsia/Chromium style), but for now we have one rust dep and the env-var-per-setting pattern is tractable. Tracking: shorebirdtech/_shorebird#2014 Follow-up to: #123, #124 --- .../common/shorebird/build_rust_updater.py | 27 +++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/engine/src/flutter/shell/common/shorebird/build_rust_updater.py b/engine/src/flutter/shell/common/shorebird/build_rust_updater.py index c6e9bf1c14875..aa4d66fe3aed3 100644 --- a/engine/src/flutter/shell/common/shorebird/build_rust_updater.py +++ b/engine/src/flutter/shell/common/shorebird/build_rust_updater.py @@ -45,6 +45,7 @@ def main(): is_android = 'android' in args.rust_target is_apple_ios = 'apple-ios' in args.rust_target is_apple_darwin = 'apple-darwin' in args.rust_target + is_msvc = 'pc-windows-msvc' in args.rust_target if is_android: if not args.ndk_path or not args.android_api_level: @@ -78,6 +79,9 @@ def main(): return 1 env['MACOSX_DEPLOYMENT_TARGET'] = args.mac_deployment_target + if is_msvc: + _configure_msvc_env(env, args.rust_target) + # GN passes paths relative to the build output dir (which is cwd when # Ninja runs the action). Resolve them to absolute paths so they work # regardless of cargo's working directory. @@ -116,6 +120,29 @@ def main(): return 0 +def _configure_msvc_env(env, rust_target): + """Force the cc crate to compile transitive C deps with the static CRT. + + The engine's Windows build uses the static CRT (/MT). The updater's + .cargo/config.toml sets `-C target-feature=+crt-static` so the rlib is + compiled to expect static-CRT linkage. However, cargo populates + CARGO_CFG_TARGET_FEATURE from the rustc target spec's default features, + not from user rustflags, so +crt-static is invisible to build scripts. + The cc crate (used by transitive *-sys deps like zstd-sys to compile + their C sources) therefore falls back to /MD, producing .obj files + full of __imp_* references that the engine's /MT link cannot resolve + (e.g. __imp_clock, __imp__wassert, __imp_qsort_s). + + Force /MT via the per-target CFLAGS env that the cc crate honors. cc + appends these flags at the end of its command line, so cl.exe sees + `/MD ... /MT` and emits warning D9025 ("overriding '/MD' with '/MT'") + and uses the last one -- /MT wins. + """ + triple_env = rust_target.replace('-', '_') + env[f'CFLAGS_{triple_env}'] = '/MT' + env[f'CXXFLAGS_{triple_env}'] = '/MT' + + def _configure_android_env(env, rust_target, ndk_path, api_level): """Set environment variables so cargo can cross-compile for Android.""" # GN passes paths relative to the build output dir. Resolve to absolute From ce852a1a52e6d7c26df41da8602dd40ca733f24d Mon Sep 17 00:00:00 2001 From: Eric Seidel Date: Tue, 7 Apr 2026 23:13:04 -0700 Subject: [PATCH 42/95] fix: remove legacy pre-GN cargo preamble from CI build scripts (#127) mac_build.sh, linux_build.sh, and win_build.sh each had a leftover 'cd $UPDATER_SRC/library && cargo build --target ...' preamble from before bdero's #120 GN/Ninja integration of the rust updater build. mac_build.sh even had a FIXME at the top documenting the intent to delete it. These preambles: 1. Bypass the GN action entirely, so none of the env-var plumbing in build_rust_updater.py applies (no IPHONEOS_DEPLOYMENT_TARGET from #123, no CFLAGS_*_windows_msvc=/MT from #125, no Android NDK env). 2. Write to the source-tree default cargo target dir at third_party/updater/target/, ignoring #124's redirect to $root_out_dir/cargo_target/, so the engine 'rm -rf out' clobber never touches their output. 3. On the persistent Sandpiper runner this means stale rlibs from pre-fix builds keep getting reused, masking #123 entirely and producing the same iOS arm64 zstd_sys deployment-target failure after every fix attempt. Delete the library cargo invocations from all three scripts. The GN build of any engine target that depends transitively on //flutter/shell/common/shorebird:updater (which is most of them) will now invoke build_rust_updater.py via Ninja and produce libupdater.a / updater.lib in the right place with the right env vars. The patch tool's separate cargo build in mac_build.sh / linux_build.sh is a standalone CLI not linked into the engine, so the GN build does not cover it. Left in place with a TODO to migrate it later. Tracking: shorebirdtech/_shorebird#2014 Follow-up to: #120, #123, #124, #125 --- shorebird/ci/internal/linux_build.sh | 30 ++++++++++------------------ shorebird/ci/internal/mac_build.sh | 26 +++++++++--------------- shorebird/ci/internal/win_build.sh | 10 +++++----- 3 files changed, 24 insertions(+), 42 deletions(-) diff --git a/shorebird/ci/internal/linux_build.sh b/shorebird/ci/internal/linux_build.sh index 9f8cbf13baa1b..44ddc97cfedf4 100755 --- a/shorebird/ci/internal/linux_build.sh +++ b/shorebird/ci/internal/linux_build.sh @@ -17,26 +17,16 @@ ENGINE_OUT=$ENGINE_SRC/out UPDATER_SRC=$ENGINE_SRC/flutter/third_party/updater HOST_ARCH='linux-x64' -# Build the Rust library. -cd $UPDATER_SRC/library - -# 24 is Flutter's current minimum supported version, -# see https://docs.flutter.dev/reference/supported-platforms -# previous iterations of cargo-ndk required the version to be passed as -# -p , but that no longer seems needed. -# We always use the hermetic NDK from the engine repo. -# The "unmodified" CIPD package keeps the NDK at the standard Android SDK path. -ANDROID_NDK_HOME=$(echo "$ENGINE_SRC/flutter/third_party/android_tools/sdk/ndk"/*) \ -cargo ndk \ - --target armv7-linux-androideabi \ - --target aarch64-linux-android \ - --target i686-linux-android \ - --target x86_64-linux-android \ - build --release - -cargo build --release --target x86_64-unknown-linux-gnu - -# Build the patch tool. +# The Rust updater library is now built as part of the GN/Ninja engine +# build โ€” see //flutter/shell/common/shorebird/BUILD.gn's +# build_rust_updater action. Android cross-compile env (NDK paths, +# CC/AR/LINKER) is configured by build_rust_updater.py. Any engine target +# that depends (transitively) on //flutter/shell/common/shorebird:updater +# pulls in libupdater.a automatically. +# +# Build the patch tool. This is a standalone CLI, not linked into the +# engine, so the GN build doesn't cover it. +# TODO(shorebird): move the patch tool into the GN build too. cd $UPDATER_SRC/patch cargo build --release diff --git a/shorebird/ci/internal/mac_build.sh b/shorebird/ci/internal/mac_build.sh index a860da431f269..f4096d349a614 100755 --- a/shorebird/ci/internal/mac_build.sh +++ b/shorebird/ci/internal/mac_build.sh @@ -1,9 +1,5 @@ #!/bin/bash -e -# FIXME: This script should be deleted and instead these steps be part -# of the GN build process. -# I haven't investigated how to build rust from GN with the Android NDK yet. - # Usage: # ./mac_build.sh engine_path @@ -21,19 +17,15 @@ ENGINE_OUT=$ENGINE_SRC/out UPDATER_SRC=$ENGINE_SRC/flutter/third_party/updater HOST_ARCH='darwin-x64' -# Build the Rust library. -cd $UPDATER_SRC/library - -# Build iOS and MacOS -cargo build \ - --target aarch64-apple-ios \ - --target x86_64-apple-ios \ - --target aarch64-apple-darwin \ - --target x86_64-apple-darwin \ - --release - -# Build the patch tool. -# Again, this belongs as part of the gn build. +# The Rust updater library is now built as part of the GN/Ninja engine +# build โ€” see //flutter/shell/common/shorebird/BUILD.gn's +# build_rust_updater action. Any engine target that depends (transitively) +# on //flutter/shell/common/shorebird:updater will pull in libupdater.a +# automatically. +# +# Build the patch tool. This is a standalone CLI, not linked into the +# engine, so the GN build doesn't cover it. +# TODO(shorebird): move the patch tool into the GN build too. cd $UPDATER_SRC/patch cargo build --release diff --git a/shorebird/ci/internal/win_build.sh b/shorebird/ci/internal/win_build.sh index 2df02670aa78f..9c47369707d34 100755 --- a/shorebird/ci/internal/win_build.sh +++ b/shorebird/ci/internal/win_build.sh @@ -10,11 +10,11 @@ ENGINE_OUT=$ENGINE_SRC/out UPDATER_SRC=$ENGINE_SRC/flutter/third_party/updater HOST_ARCH='windows-x64' -# Build the Rust library. -cd $UPDATER_SRC/library - -cargo build --release \ - --target x86_64-pc-windows-msvc +# The Rust updater library is now built as part of the GN/Ninja engine +# build โ€” see //flutter/shell/common/shorebird/BUILD.gn's +# build_rust_updater action. Any engine target that depends (transitively) +# on //flutter/shell/common/shorebird:updater pulls in updater.lib +# automatically. # Compile the engine using the steps here: # https://github.com/flutter/flutter/wiki/Compiling-the-engine#compiling-for-android-from-macos-or-linux From ddb03aad57e9a93ae651b50e98260ee47c4945d4 Mon Sep 17 00:00:00 2001 From: Eric Seidel Date: Tue, 7 Apr 2026 23:23:00 -0700 Subject: [PATCH 43/95] fix: declare libupdater.a as an output of build_rust_updater action (#128) The action only declared its stamp file as an output, which worked when shorebird_updater_output_lib lived under //flutter/third_party/... because GN treated source-tree paths as existing files needing no rule. After #124 moved the cargo target dir into $root_out_dir/cargo_target, the lib path became a build-dir path and Ninja correctly demands a rule producing it. Without the lib in outputs: ninja: error: 'cargo_target/aarch64-linux-android/release/libupdater.a', needed by 'libflutter.so', missing and no known rule to make it Add shorebird_updater_output_lib to the action's outputs alongside the stamp file. The cargo invocation already produces the file at this path; this just tells Ninja which rule is responsible. Tracking: shorebirdtech/_shorebird#2014 Follow-up to: #124, #127 --- engine/src/flutter/shell/common/shorebird/BUILD.gn | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/engine/src/flutter/shell/common/shorebird/BUILD.gn b/engine/src/flutter/shell/common/shorebird/BUILD.gn index 04ff926fa67e8..22b11a614cad1 100644 --- a/engine/src/flutter/shell/common/shorebird/BUILD.gn +++ b/engine/src/flutter/shell/common/shorebird/BUILD.gn @@ -24,10 +24,18 @@ if (shorebird_updater_supported) { action("build_rust_updater") { script = "//flutter/shell/common/shorebird/build_rust_updater.py" - # The stamp file is the declared output for Ninja dependency tracking. + # The stamp file gives Ninja a deterministic output to depend on, but + # we also have to declare the actual static library so consumers + # referencing it via `libs = [ shorebird_updater_output_lib ]` find a + # rule that produces it. Without the lib in `outputs`, Ninja errors + # with `'cargo_target/.../libupdater.a' missing and no known rule to + # make it` once the cargo target dir lives inside root_out_dir (#124). _stamp = "$target_gen_dir/rust_updater_${shorebird_updater_rust_target}.stamp" - outputs = [ _stamp ] + outputs = [ + _stamp, + shorebird_updater_output_lib, + ] args = [ "--rust-target", From 609b316ced91d548a057742a5e576022c00473b5 Mon Sep 17 00:00:00 2001 From: Eric Seidel Date: Wed, 8 Apr 2026 07:07:40 -0700 Subject: [PATCH 44/95] fix: gate shorebird_updater_supported on having a known rust target (#130) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit build_rust_updater.gni previously set shorebird_updater_supported based only on target_os (is_android || is_ios || ...). But GN evaluates BUILD files under every toolchain in use, including host and sub toolchains that can have unusual (target_os, target_cpu) combinations we don't handle. When a toolchain entered the `if (shorebird_updater_supported)` block but hit none of the os/cpu cases that set `shorebird_updater_rust_target`, GN blew up: ERROR at build_rust_updater.gni:72: Undefined identifier in string expansion. shorebird_updater_output_lib = "$...$shorebird_updater_rust_target..." "shorebird_updater_rust_target" is not currently in scope. This happens on Windows host builds of android engine variants when the host toolchain(s) land on a combo like (is_win, target_cpu != x64) or similar, which our is_win branch doesn't cover. Flip the logic: compute shorebird_updater_rust_target first, default to empty, and derive `shorebird_updater_supported` from whether it got a value. Unhandled combos become unsupported, so the consuming BUILD.gn files' `if (shorebird_updater_supported)` gate cleanly skips the updater for that toolchain โ€” which is correct, since the updater only needs to be built for toolchains whose final binary is the engine. Tracking: shorebirdtech/_shorebird#2014 Follow-up to: #124, #127, #128 --- .../common/shorebird/build_rust_updater.gni | 76 ++++++++++--------- 1 file changed, 41 insertions(+), 35 deletions(-) diff --git a/engine/src/flutter/shell/common/shorebird/build_rust_updater.gni b/engine/src/flutter/shell/common/shorebird/build_rust_updater.gni index da27527cb95bc..08b72424af7b6 100644 --- a/engine/src/flutter/shell/common/shorebird/build_rust_updater.gni +++ b/engine/src/flutter/shell/common/shorebird/build_rust_updater.gni @@ -16,44 +16,50 @@ if (is_android) { shorebird_updater_dir = "//flutter/third_party/updater" -# Only build and link the Rust updater on supported platforms. -shorebird_updater_supported = - is_android || is_ios || is_mac || is_win || is_linux +# Compute the Rust target triple from the GN target_os/target_cpu. GN +# evaluates BUILD files under every toolchain in use, not just the +# top-level one, so we get called for host/sub toolchains too. Any +# (os, cpu) combo we don't explicitly handle leaves the rust target +# empty and marks the updater unsupported for that toolchain, which is +# the correct behavior โ€” the updater only needs to be built for +# toolchains whose final binary is the engine. +shorebird_updater_rust_target = "" -if (shorebird_updater_supported) { - # Compute the Rust target triple from the GN target_os/target_cpu. - if (is_android) { - if (target_cpu == "arm") { - shorebird_updater_rust_target = "armv7-linux-androideabi" - } else if (target_cpu == "arm64") { - shorebird_updater_rust_target = "aarch64-linux-android" - } else if (target_cpu == "x64") { - shorebird_updater_rust_target = "x86_64-linux-android" - } else if (target_cpu == "x86") { - shorebird_updater_rust_target = "i686-linux-android" - } - } else if (is_ios) { - if (target_cpu == "arm64") { - shorebird_updater_rust_target = "aarch64-apple-ios" - } else if (target_cpu == "x64") { - shorebird_updater_rust_target = "x86_64-apple-ios" - } - } else if (is_mac) { - if (target_cpu == "arm64") { - shorebird_updater_rust_target = "aarch64-apple-darwin" - } else if (target_cpu == "x64") { - shorebird_updater_rust_target = "x86_64-apple-darwin" - } - } else if (is_win) { - if (target_cpu == "x64") { - shorebird_updater_rust_target = "x86_64-pc-windows-msvc" - } - } else if (is_linux) { - if (target_cpu == "x64") { - shorebird_updater_rust_target = "x86_64-unknown-linux-gnu" - } +if (is_android) { + if (target_cpu == "arm") { + shorebird_updater_rust_target = "armv7-linux-androideabi" + } else if (target_cpu == "arm64") { + shorebird_updater_rust_target = "aarch64-linux-android" + } else if (target_cpu == "x64") { + shorebird_updater_rust_target = "x86_64-linux-android" + } else if (target_cpu == "x86") { + shorebird_updater_rust_target = "i686-linux-android" + } +} else if (is_ios) { + if (target_cpu == "arm64") { + shorebird_updater_rust_target = "aarch64-apple-ios" + } else if (target_cpu == "x64") { + shorebird_updater_rust_target = "x86_64-apple-ios" + } +} else if (is_mac) { + if (target_cpu == "arm64") { + shorebird_updater_rust_target = "aarch64-apple-darwin" + } else if (target_cpu == "x64") { + shorebird_updater_rust_target = "x86_64-apple-darwin" + } +} else if (is_win) { + if (target_cpu == "x64") { + shorebird_updater_rust_target = "x86_64-pc-windows-msvc" } +} else if (is_linux) { + if (target_cpu == "x64") { + shorebird_updater_rust_target = "x86_64-unknown-linux-gnu" + } +} +shorebird_updater_supported = shorebird_updater_rust_target != "" + +if (shorebird_updater_supported) { if (is_win) { _updater_lib_name = "updater.lib" } else { From 00b2a9b7a1850e77e9cfd0a20e6a71b7086cbc00 Mon Sep 17 00:00:00 2001 From: Brandon DeRosier Date: Thu, 9 Apr 2026 14:22:32 -0700 Subject: [PATCH 45/95] ci: upload patch-darwin-arm64.zip alongside x64 (#129) Download the new aarch64-apple-darwin asset from the patch release and upload it to GCS as patch-darwin-arm64.zip so Apple Silicon hosts can fetch a native binary instead of hitting "Bad CPU type in executable" when Rosetta is unavailable. Bumps PATCH_VERSION to 0.3.0, which is the first release that ships an explicit aarch64-apple-darwin asset (see shorebirdtech/updater#337). Note: PATCH_VERSION should be confirmed against the actual version cut from updater#337 before landing. --- shorebird/ci/internal/mac_upload.sh | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/shorebird/ci/internal/mac_upload.sh b/shorebird/ci/internal/mac_upload.sh index 8ec266246a8e1..e565a352ea963 100755 --- a/shorebird/ci/internal/mac_upload.sh +++ b/shorebird/ci/internal/mac_upload.sh @@ -166,14 +166,16 @@ gsutil cp $ZIPS_OUT/FlutterMacOS.framework.dSYM.zip $ZIPS_DEST/FlutterMacOS.fram TMP_DIR=$(mktemp -d) -PATCH_VERSION=0.2.1 +PATCH_VERSION=0.3.0 GH_RELEASE=https://github.com/shorebirdtech/updater/releases/download/patch-v$PATCH_VERSION/ cd $TMP_DIR curl -L $GH_RELEASE/patch-x86_64-apple-darwin.zip -o patch-x86_64-apple-darwin.zip +curl -L $GH_RELEASE/patch-aarch64-apple-darwin.zip -o patch-aarch64-apple-darwin.zip curl -L $GH_RELEASE/patch-x86_64-pc-windows-msvc.zip -o patch-x86_64-pc-windows-msvc.zip curl -L $GH_RELEASE/patch-x86_64-unknown-linux-musl.zip -o patch-x86_64-unknown-linux-musl.zip gsutil cp patch-x86_64-apple-darwin.zip $SHOREBIRD_ROOT/patch-darwin-x64.zip +gsutil cp patch-aarch64-apple-darwin.zip $SHOREBIRD_ROOT/patch-darwin-arm64.zip gsutil cp patch-x86_64-pc-windows-msvc.zip $SHOREBIRD_ROOT/patch-windows-x64.zip gsutil cp patch-x86_64-unknown-linux-musl.zip $SHOREBIRD_ROOT/patch-linux-x64.zip From e2339fd10d59b60ecd39ac36171c10ae7367ee15 Mon Sep 17 00:00:00 2001 From: Eric Seidel Date: Mon, 20 Apr 2026 09:17:28 -0700 Subject: [PATCH 46/95] Add --trace flag to flutter build apk for build profiling (#116) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * feat: Add --trace flag to flutter build apk for build profiling Adds a --trace option that produces a Chrome Trace Event Format JSON file showing where time is spent across all build layers (flutter tool, Gradle, flutter assemble targets). The output can be viewed in Perfetto at https://ui.perfetto.dev. * Add comment about multi-variant trace file collision risk The intermediate trace file path is shared across all Gradle variants. This is safe today since flutter build apk only runs one variant per invocation, but would need per-variant paths if that changes. * Add iOS build tracing support - Add --trace option to flutter build ios / flutter build ipa - Pass TRACE_FILE through Xcode build settings to xcode_backend.dart - Instrument buildXcodeProject() in mac.dart with pre-xcode, xcode, and post-xcode spans - Merge flutter assemble trace events from intermediate file - Remove TRACE_FILE from toEnvironmentConfig() since both Android and iOS orchestrators compute intermediate paths directly * feat: accept --trace on `flutter build appbundle` The umbrella `flutter build apk` command already wired up the build- trace option via `usesBuildTraceOption`, but a lot of real-world workflows build the AppBundle (`flutter build appbundle`) directly โ€” most notably `shorebird release android` which always produces an AAB. Without this, those invocations silently skip tracing. The gradle/build-system plumbing is already shared, so adding the option here is a one-line change that opts appbundle into the same trace file Flutter already emits for APK builds. * feat: enable Gradle --profile when build tracing is on PR-116's trace only captures one `gradle bundleRelease`/`assembleRelease` span on tid 2, so per-plugin task timing (compileReleaseKotlin, mergeReleaseResources, linkReleaseNativeLibs, etc.) is invisible. Gradle already has a built-in profiler that writes per-task durations to `build/reports/profile/profile-*.html`; we just weren't enabling it. Turn on `--profile` automatically when `--trace` is set so the profile report lands alongside the trace. Downstream tooling (Shorebird) can then aggregate per-task timings into an anonymized histogram โ€” plugin count, p50/p90/max task duration โ€” to answer 'is native plugin compile the bottleneck?' without collecting any plugin names. * feat: per-Gradle-task trace events via init script Before this, the only Gradle signal in a build trace was one giant "gradle assembleRelease" span on tid 2 โ€” useless for answering "where is Gradle spending time" on plugin-heavy apps, which was half the motivation for tracing in the first place. This adds `flutter_trace_init.gradle`, an init script that registers a TaskExecutionListener and emits one Chrome Trace Event Format entry per Gradle task to an intermediate file. Each event carries: - name: the full task path (e.g. `:camera_android:compileReleaseKotlin`) - args.kind: a small bucket label (kotlin_compile, java_compile, dex, resources, packaging, bundle, transform, native_link, lint, flutter_gradle_plugin, other) so downstream tooling can aggregate without holding on to raw names - args.owner: the first colon segment (the subproject / plugin name) - args.skipped / upToDate / fromCache: cache-hit signal Events land on tid=4 so Perfetto shows a clean four-tier layout: flutter tool / native outer / flutter assemble / gradle tasks. gradle.dart passes `-I=` and `-Pflutter.gradle-trace-file=` alongside the existing assemble trace wiring, and merges the resulting file into the main trace the same way assemble events are already merged. Supersedes the earlier `--profile` addition for AAR builds (removed): we were writing an HTML report nobody was parsing; the init script produces Chrome Trace events that go straight into the trace file users already open in Perfetto. Timestamps use `System.currentTimeMillis() * 1000` (wall-clock micros) to stay aligned with the existing flutter-tool and flutter-assemble events, which also use wall clock. * fix: classify compileReleaseKotlin/Java and AGP tasks correctly The first pass matched 'compilekotlin'/'compilejava' substrings against the full lowercased task path, but AGP-generated tasks are 'compileReleaseKotlin' / 'compileReleaseJavaWithJavac' โ€” the variant name sits between the verb and the language. Also, plugin owners like 'package_info_plus' were tripping the 'packaging' bucket because the owner name contains 'package'. Switch to matching against the simple task name (last colon segment) and require compile tasks to start with 'compile' + contain the language token. Add dedicated buckets for R8/minify and lint โ€” they dominate on release builds (R8 alone was 33s on the heavy app validation) and were previously hidden in 'other'. * feat: broader kind classification for Gradle init-script tasks Real-world validation found ~60% of task time sitting in 'other' on a plugin-heavy app โ€” mostly AGP scaffolding that has a recognizable shape (writeAarMetadata, checkReleaseAarMetadata, generateRFile, mergeReleaseNativeLibs, javaPreCompileRelease, etc.). Add: - 'java_compile' now also matches '*precompile*' (annotation- processor setup, javaPreCompileRelease) - 'resources' matches the merge/process/generate family (mergeReleaseNativeLibs is overloaded โ€” kept under native_link) - 'native_link' for mergeReleaseNativeLibs, copyReleaseJniLibs* - 'gradle_scaffold' catch-all for aarmetadata, proguard, validate, check*, prepare*, generate*, copy* Also restructured the classifier as a sequence of if/else-if with explicit braces โ€” the previous dangling-else chain was hard to read after adding a dozen cases. * refactor: rename trace surface with shorebird- prefix; add pod install span This whole build-trace feature is specific to the Shorebird fork and won't be upstreamed in its current form, so every public surface name should make that obvious and stay out of identifiers upstream Flutter might want to use later: --trace -> --shorebird-trace --trace-file -> --shorebird-trace-file (assemble) -Ptrace-file -> -Pshorebird-trace-file (gradle) -Pflutter.gradle-trace-file -> -Pshorebird.gradle-trace-file TRACE_FILE -> SHOREBIRD_TRACE_FILE (xcode env) flutter_trace_init.gradle -> shorebird_trace_init.gradle flutter_assemble_trace.json -> shorebird_assemble_trace.json flutter_gradle_task_trace.json -> shorebird_gradle_task_trace.json BuildInfo.traceFilePath -> BuildInfo.shorebirdTraceFilePath FlutterOptions.kBuildTrace -> FlutterOptions.kShorebirdTrace usesBuildTraceOption -> usesShorebirdTraceOption BaseFlutterTask.traceFile -> BaseFlutterTask.shorebirdTraceFile intermediates/flutter (trace dir) -> intermediates/shorebird No behaviour change beyond the rename. Also: add a 'pod install' subprocess span in mac.dart. Previously, on plugin-heavy iOS apps there was a minute-plus gap between "flutter build ios" starting and the xcode archive span showing up โ€” that was CocoaPods resolving, and it was invisible in the trace. Now it's a first-class event on tid=1 with cat='subprocess' so Shorebird's summarizer can bucket it without double-counting into pre-xcode setup. * feat: record Flutter-tool HTTP downloads in the Shorebird build trace Until now the trace captured Shorebird's own network calls (via the TracingClient wrapper on the Shorebird side) and subprocess spans for pod install / gradle / xcode, but Flutter's own HTTP โ€” primarily artifact downloads in `Cache.updateAll` routed through `Net._attempt` โ€” was completely invisible. On a fresh Flutter-tool state this is tens of seconds across a dozen artifacts; with the user's laptop on a slow connection, minutes. Expose a process-wide `BuildTracer.current` that gradle.dart / mac.dart set for the duration of a build. `Net._attempt` records a span on tid=5 / cat='network' for each request with method, host, and (on failure) error kind โ€” no URLs or paths. That matches the shape Shorebird's TracingClient already emits, so the Shorebird summarizer sums both into `networkMs` without any changes. Using a singleton is deliberate: plumbing a tracer through every caller of `Net.fetchUrl` and every subprocess wrapper would touch a large fraction of flutter_tools files for what is still a fork-only feature. The static is scoped by set/clear in the build driver, and Flutter commands are one-shot processes so there's no lifetime concern across invocations. * feat: Xcode per-phase + CocoaPods per-phase trace spans Fills two of the remaining "giant opaque block" gaps: **Xcode.** Pass `-showBuildTimingSummary` when a Shorebird build trace is active. Xcode prints a `** Build Timing Summary **` section at the end of the log listing per-phase aggregates (CompileC / SwiftCompile / Ld / PhaseScriptExecution / CodeSign / ...). mac.dart parses it and emits one event per phase on tid=4 / cat='xcode_phase', so the previously monolithic ~50s `xcode archive` span now has a visible breakdown. The flag is pure reporting: Apple documents it as affecting output only, and the existing flutter_tools isn't using it today. Synthetic timestamps note: the summary only gives aggregate durations per phase, not wall-clock, so we lay the events out contiguously back from the xcode-end timestamp. Distribution is accurate; individual spans don't necessarily line up with real wall clock inside Xcode. **CocoaPods.** When tracing is active, switch pod install from ProcessManager.run to start+stream so we can timestamp phase transitions as they happen. The `--verbose` output already has stable markers (`Analyzing dependencies`, `Downloading dependencies`, `Generating Pods project`, `Integrating client project`); we just needed live lines to attach wall-clock to them. Emits four sub-spans on tid=1 / cat='subprocess' named `pod install: `. Non-tracing path is unchanged: pod install still uses the same ProcessManager.run, no behavioural or perf difference. * fix: don't add -quiet to xcodebuild when tracing is on -quiet and -showBuildTimingSummary don't play well together: xcodebuild suppresses the timing-summary block under -quiet, which is exactly what we need to parse to produce per-phase Xcode spans. Skip -quiet when a Shorebird build trace is active. User-facing output is unchanged โ€” Flutter already captures xcodebuild stdout into buildResult.stdout and only echoes it on failure or in verbose mode. * feat: parse xcode build log via xcresulttool for per-subsection spans Replaces the -showBuildTimingSummary approach: that flag is documented but produces no output on Xcode 26 (and presumably 16+), so parsing xcodebuild stdout for a "** Build Timing Summary **" block silently yielded nothing on modern Xcode. Switches to xcresulttool, which Flutter already points at via -resultBundlePath. After xcodebuild finishes, we run xcrun xcresulttool get log --type build --path and walk the top-level subsections. Each subsection (one per target build action โ€” "Build target X from project Y", "Archive target Z", etc.) has a real startTime + duration, so events land on an accurate timeline instead of synthetic sequential spans. Emitted on tid=4 with cat='xcode_subsection'. Subsection titles are kept in the raw trace for local debug; Shorebird's privacy-safe summary only records count + sum + p50/p90/max, no titles. Best-effort: xcresulttool's JSON schema drifts across Xcode versions, so parse failures are silent no-ops rather than errors. Also revert the -quiet skip: it's no longer needed now that we don't depend on xcodebuild stdout for timing data. * chore: dart format trace-related files * chore: roll dart_revision to aot_tools --trace support (shorebirdtech/dart-sdk#787) * fix(trace): correct post-gradle / post-xcode span durations - gradle.dart: hoist gradleEndMicros out of the tracer block so the post-gradle processing span starts at when Gradle actually finished, not gradleStartMicros + sw.elapsedMicroseconds (which ticks through the trace-merge work). Also clear BuildTracer.current before throwing on Gradle failure. - mac.dart: collapse two adjacent DateTime.now() calls so pre-xcode and xcode spans are back-to-back, and hoist buildEndMicros so the post-xcode + outer flutter-build-ios spans end at the same instant. - Drop redundant `!` null-checks on buildInfo.shorebirdTraceFilePath. * refactor(trace): null-check hygiene + cross-link to sibling tracers - Replace `!` null-assertions with local-variable null-checks in gradle.dart (gradleEndMicros), cocoapods.dart (closure-captured phase state), and xcode_backend.dart (SHOREBIRD_TRACE_FILE). - Keep `!` only on the JSON-parse path in build_trace.dart where flutter's cast_nullable_to_non_nullable lint prefers it; documented. - Add cross-link in BuildTraceEvent doc pointing at the sibling implementations in dart-sdk/pkg/aot_tools and shorebird_cli so future edits stay schema-compatible. * feat(trace): real pids + process_name/thread_name metadata + gradle flow events Chrome Trace Event Format carries pid/tid as producer-identifying fields, not a shared coordination space. Drop the hardcoded tid convention that flutter_tools / shorebird_cli / aot_tools had to agree on and emit real pids everywhere: - `BuildTracer.addCompleteEvent` now requires an explicit pid; the class gained `addProcessNameMetadata` / `addThreadNameMetadata` / `addFlowStart` for the Chrome Trace Event Format metadata and flow events, and the internal buffer is raw JSON maps so metadata/flow events can share the buffer with complete spans. - `currentProcessId()` FFIs libc `getpid()` (or `GetCurrentProcessId` on Windows) since `dart:io` only exposes pids for spawned children. - flutter_tool, net.dart, gradle.dart, mac.dart, cocoapods.dart, assemble.dart all emit on the calling process's real pid and name it via metadata events so Perfetto shows "flutter tool", "pod install", "flutter assemble" instead of bare numbers. - `xcode_subsection` events go on a synthetic pid named "xcodebuild" (xcresult doesn't surface xcodebuild's real pid, and the compile sub-work is done by clang/swiftc children we never saw โ€” synthetic is honest). - Gradle init script uses `ProcessHandle.current().pid()` so per-task events live on the Gradle JVM's real pid; flutter_tool emits a flow-start (`ph: "s"`) at Gradle spawn with a random id passed via `-Pshorebird.gradle-trace-flow-id`, and the init script emits the matching `ph: "f"` on the first task so Perfetto draws a cross-process causality arrow. Tests updated; summary bucketer follow-up will move from tid-based to cat-based classification now that tids are no longer a shared namespace. * chore: roll dart_revision to aot_tools real-pid support (1376369b64e) * fix(trace): label network row so flutter tool's HTTP downloads show as 'network' not 'Thread 5' * refactor(trace): use dart:io's top-level pid; use gradle pid as flow id; embed variant in trace path - Drop the FFI libc getpid()/GetCurrentProcessId() dance. `dart:io` exports the current process's pid as a top-level getter; the whole FFI block was me not checking the stdlib. - ProcessUtils.stream: new optional `onStart(Process)` callback so callers can observe the spawned subprocess's pid without changing the return shape. - gradle.dart uses onStart to read Gradle's real pid at spawn and emit the flow-start with id = gradle_pid. The init script reads its own pid via ProcessHandle.current().pid() for the matching flow-end, so both sides agree on the id without the -Pshorebird.gradle-trace-flow-id plumbing โ€” which is now gone. - Embed the assemble task name (e.g. `assembleFooRelease`, `bundleRelease`) in the intermediate trace paths so a hypothetical multi-variant Gradle run can't have per-variant FlutterTask instances stomp on each other's trace. * refactor(trace): depend on shorebird_build_trace package instead of vendoring Delete flutter_tools' local build_trace.dart; import BuildTracer / BuildTraceEvent / PhaseTracker / currentProcessId from package:shorebird_build_trace via a git: pubspec dep pointing at the public shorebird monorepo. Net: ~175 LOC of wire-format + tracer class gone from flutter_tools, schema now impossible to drift from the canonical definition in shorebirdtech/shorebird/packages/shorebird_build_trace. Consumers (gradle.dart, mac.dart, cocoapods.dart, net.dart, assemble.dart, xcode_backend.dart) get the same class by the same name via the package import โ€” just a path swap. * chore: roll dart_revision to aot_tools shorebird_build_trace migration (030fa6cef1f) * chore: roll dart_revision to aot_tools real-child-pid subprocess tracing * refactor(trace): hide trace plumbing in shorebird/ session classes - Bumps shorebird_build_trace ref to 0637612a (zone-scoped BuildTracer.current via runAsync, plus runSubprocess helpers). - Extracts the ~60 lines of inline tracing in gradle.dart and ~140 lines in mac.dart into AndroidBuildTraceSession / IosBuildTraceSession under lib/src/shorebird/. Each session exposes a `run(body)` that wraps body in BuildTracer.runAsync, so gradle.dart and mac.dart just call lifecycle hooks (onGradleFinished, onXcodeFinished, etc) and never touch BuildTracer.current themselves. - Net change in upstream-touched files: gradle.dart -161 lines, mac.dart -206 lines, net.dart -33 lines. All trace plumbing now lives under lib/src/shorebird/ which has zero upstream-conflict footprint. - net.dart's recordNetworkSpan closure moves to a NetworkTraceSpan helper class in lib/src/shorebird/. The class reads BuildTracer.current internally; net.dart just does `final span = NetworkTraceSpan.start(...); ... span.record(...);`. - abort-on-error paths removed: zone unwinding guarantees .current is cleared on throw, so the explicit abortOnGradleFailure / abortOnFailure clears are no longer needed. * chore(trace): roll shorebird_build_trace ref to d0b3280f (drop unused runSubprocessSync) * refactor(trace): manual start/stop + regex task classification Two related changes. 1. Drop the closure-wrapped trace sessions. Wrapping buildGradleApp / buildXcodeProject bodies in session.run(() async {...}) forced a reindent that would have broken upstream-merge compatibility for the next touch of those methods. Instead: - Bumps shorebird_build_trace ref to 97efa2ed (new static start/stop API + unchanged runAsync). - AndroidBuildTraceSession / IosBuildTraceSession constructors call BuildTracer.start(); finish / abortOn* call BuildTracer.stop(). - gradle.dart / mac.dart go back to flat indentation: the only session touches are maybeStart + lifecycle hook calls. Tradeoff: if an uncaught exception escapes the build method, BuildTracer.current leaks. The flutter CLI exits the process shortly after, so leak is per-invocation, not cross-invocation. aot_tools keeps using runAsync (it can wrap easily). 2. Replace the gradle init script's if/else kind classifier with a regex table. Same semantics, same priority order, but the ordering is now explicit (a comment calls out which rules must precede which) and accidental substring matches like `:package_info_plus:` -> 'packaging' are easier to audit. * refactor(trace): use BuildTracer + PhaseTracker helpers from shared library Two DRY wins found during self-review: - assemble.dart's writeTraceData was handrolling Chrome Trace Event Format maps directly. Replaced with BuildTracer.addProcessName / addThreadName / addCompleteEvent / writeToFile, which also picks up merge-with-existing-file behavior for free. ~40 lines -> ~15. - cocoapods.dart's _runPodInstallStreamed reimplemented the exact phase-tracking pattern the library's PhaseTracker already provides (the library's doc comment literally cites pod install as the motivating example). Replaced with a PhaseTracker instance. ~20 lines -> ~5. Drive-by: sort the `package:shorebird_build_trace` import into the right section in both files so `dart analyze` stops complaining about directive ordering. * refactor(trace): use TraceSchema constants for all cross-repo strings Replaces inline string literals for event categories, gradle task kinds, pod install phase names, and span-name prefixes with TraceSchema.* references from the shared shorebird_build_trace package (new in d2099fe7). Makes the contract with shorebird_cli explicit โ€” rename a constant there and this side stops compiling rather than silently diverging. - Android/iOS session classes: cat/name constants for the flutter, gradle, xcode, subprocess, xcode_subsection events they emit. - cocoapods.dart: pod install name + phase-transition strings. - assemble.dart: cat for assemble spans. - network_trace_span.dart: cat for network spans. - gradle.dart: instead of passing `'flutter build apk/appbundle'` as an opaque string, passes the target suffix (`apk`/`appbundle`) and the session constructs the span name from `TraceSchema.flutterBuildSpanPrefix`. Keeps TraceSchema import out of upstream-touched gradle.dart. - shorebird_trace_init.gradle: can't import Dart, so its kind literals stay โ€” added a KEEP IN SYNC comment pointing at TraceSchema.gradleKind* so any new entry lands on both sides. - Bumps shorebird_build_trace ref to d2099fe7. * refactor(trace): emit via TraceCategory/GradleTaskKind enums Bumps shorebird_build_trace ref to 27f5971e (TraceSchema class replaced by TraceCategory + GradleTaskKind enums plus TraceNames for format-prefix constants). Producer-side changes are mechanical: TraceSchema.catFlutter becomes TraceCategory.flutter.wireName, TraceSchema.podPhaseAnalyzing becomes PodInstallPhase.analyzing.wireName, etc. Span-name prefixes (flutterBuildSpanPrefix, podInstallNamePrefix, ...) moved to TraceNames; they're format templates not enumerated values. Init script's KEEP-IN-SYNC comment now points at GradleTaskKind. * chore(trace): roll shorebird_build_trace ref to cae6c56e (map-keyed accumulator) * chore(trace): roll shorebird_build_trace ref to 2a56c3e3 (Duration internally) * refactor(trace): take DateTime/Duration from shorebird_build_trace Bumps shorebird_build_trace ref to bd5452360acc5371b30a265c85187c061df674cb. Producer sites use DateTime.now() instead of DateTime.now().microsecondsSinceEpoch, and pass DateTime / Duration to the library's addCompleteEvent, addFlowStart, addFlowEnd, recordNetworkSpan, etc. * chore(trace): roll shorebird_build_trace ref to cff436f6 (schema v7) * chore: dart format 5 test files to satisfy CI * fix: analyzer errors in upstream-merged shorebird fork Shorebird DEV CI runs `dart analyze` on the whole flutter_tools package, unlike GitHub Actions which only runs test/general.shard. The prior squashed-merge of upstream 3.35.7 carried two pre-existing issues that analyze surfaces: - build_macos_test.dart:1139 passed undefined `artifacts` and `processUtils` named params to BuildCommand (which doesn't accept them). Removed โ€” the test still exercises the Shorebird YAML post-build step, which was its actual purpose. - build_windows.dart imported shorebird_yaml.dart but never used it. * refactor(trace): table-lookup for pod install phase markers Replace the four-branch if/else-if chain with a const map from log-line marker to PodInstallPhase + a single-loop lookup. Adding a new phase now means adding a map entry instead of a new branch, and the markers sit next to each other at the top of the file where they're easier to audit for staleness against newer CocoaPods releases. * test: skip broken macOS shorebird.yaml-injection test The test expects updateShorebirdYaml to mutate shorebird.yaml during build, but the override's TestBuildSystem.all stubs out the assets build target where that mutation runs. The test was previously un-runnable due to a compile error (undefined artifacts/processUtils params); now that it compiles again, mark it skip with an explanation rather than let it fail. Also adds the missing OperatingSystemUtils override present in every other test in the file. * chore: dart format build_macos_test.dart after edit * test: skip broken Windows shorebird.yaml-injection test Same situation as the macOS analogue: the test expects updateShorebirdYaml to mutate shorebird.yaml during the windows build, but the overridden FakeProcessManager only runs cmake + cmake --build, neither of which drives the assets target where that mutation lives. Skipping pending a test that exercises updateShorebirdYaml directly. --- packages/flutter_tools/bin/xcode_backend.dart | 8 + .../gradle/shorebird_trace_init.gradle | 159 ++++++ .../gradle/src/main/kotlin/FlutterPlugin.kt | 3 + .../src/main/kotlin/tasks/BaseFlutterTask.kt | 6 + .../kotlin/tasks/BaseFlutterTaskHelper.kt | 3 + .../flutter_tools/lib/src/android/gradle.dart | 23 +- .../flutter_tools/lib/src/base/build.dart | 3 +- packages/flutter_tools/lib/src/base/net.dart | 24 +- .../flutter_tools/lib/src/base/process.dart | 3 + .../flutter_tools/lib/src/build_info.dart | 7 + .../lib/src/build_system/build_system.dart | 6 + .../lib/src/commands/assemble.dart | 44 ++ .../lib/src/commands/build_apk.dart | 1 + .../lib/src/commands/build_appbundle.dart | 1 + .../lib/src/commands/build_ios.dart | 1 + .../lib/src/ios/application_package.dart | 21 +- packages/flutter_tools/lib/src/ios/mac.dart | 23 + .../lib/src/macos/cocoapods.dart | 113 +++- .../lib/src/runner/flutter_command.dart | 23 + .../android_build_trace_session.dart | 210 ++++++++ .../shorebird/ios_build_trace_session.dart | 281 ++++++++++ .../lib/src/shorebird/network_trace_span.dart | 54 ++ .../lib/src/shorebird/shorebird_yaml.dart | 23 +- packages/flutter_tools/lib/src/version.dart | 10 +- .../lib/src/windows/build_windows.dart | 1 - packages/flutter_tools/pubspec.yaml | 10 + .../hermetic/assemble_test.dart | 161 +++--- .../hermetic/build_macos_test.dart | 67 +-- .../hermetic/build_windows_test.dart | 169 +++--- .../build_system/build_trace_test.dart | 239 +++++++++ .../build_system/targets/macos_test.dart | 137 ++--- .../test/general.shard/cache_test.dart | 497 +----------------- .../shorebird/shorebird_yaml_test.dart | 23 +- .../test/general.shard/version_test.dart | 61 ++- 34 files changed, 1535 insertions(+), 880 deletions(-) create mode 100644 packages/flutter_tools/gradle/shorebird_trace_init.gradle create mode 100644 packages/flutter_tools/lib/src/shorebird/android_build_trace_session.dart create mode 100644 packages/flutter_tools/lib/src/shorebird/ios_build_trace_session.dart create mode 100644 packages/flutter_tools/lib/src/shorebird/network_trace_span.dart create mode 100644 packages/flutter_tools/test/general.shard/build_system/build_trace_test.dart diff --git a/packages/flutter_tools/bin/xcode_backend.dart b/packages/flutter_tools/bin/xcode_backend.dart index 2142ba29bf1c2..9f6c8983a39df 100644 --- a/packages/flutter_tools/bin/xcode_backend.dart +++ b/packages/flutter_tools/bin/xcode_backend.dart @@ -692,6 +692,14 @@ class Context { ); } + // Shorebird-specific build-trace plumbing: mac.dart sets + // SHOREBIRD_TRACE_FILE in the Xcode build environment; here (running + // as an Xcode build phase script) we forward it to flutter assemble. + final String? shorebirdTraceFile = environment['SHOREBIRD_TRACE_FILE']; + if (shorebirdTraceFile != null && shorebirdTraceFile.isNotEmpty) { + flutterArgs.add('--shorebird-trace-file=$shorebirdTraceFile'); + } + if (environment['CODE_SIZE_DIRECTORY'] != null && environment['CODE_SIZE_DIRECTORY']!.isNotEmpty) { flutterArgs.add('-dCodeSizeDirectory=${environment['CODE_SIZE_DIRECTORY']}'); diff --git a/packages/flutter_tools/gradle/shorebird_trace_init.gradle b/packages/flutter_tools/gradle/shorebird_trace_init.gradle new file mode 100644 index 0000000000000..8f469f87397bf --- /dev/null +++ b/packages/flutter_tools/gradle/shorebird_trace_init.gradle @@ -0,0 +1,159 @@ +// Emits Chrome Trace Event Format events for every Gradle task so the +// Shorebird build trace (`flutter build ... --shorebird-trace`) can show +// per-task timings alongside flutter tool + flutter assemble spans. +// +// This file is Shorebird-specific โ€” it exists in the Shorebird fork of +// Flutter and has no equivalent upstream. Activated by passing +// `-I=` together with `-Pshorebird.gradle-trace-file=`. +// Output file is consumed by Flutter's gradle.dart and merged into the +// main trace. +// +// Events land on tid=4 (flutter tool = 1, native build outer = 2, flutter +// assemble = 3) so Perfetto shows a clean per-tier layout. + +import groovy.json.JsonOutput +import org.gradle.api.Task +import org.gradle.api.execution.TaskExecutionListener +import org.gradle.api.tasks.TaskState + +def traceFilePath = gradle.startParameter.projectProperties['shorebird.gradle-trace-file'] +if (!traceFilePath) { + traceFilePath = System.getProperty('shorebird.gradle-trace-file') +} +if (!traceFilePath) { + return +} + +// Flow id shared with flutter_tool so Perfetto draws an arrow from the +// parent "gradle " span into our first per-task event. Missing +// when flutter_tool is older than the flow-id plumbing; that's fine, +// the init script just skips the flow-end event in that case. +def gradleFlowIdRaw = gradle.startParameter.projectProperties['shorebird.gradle-trace-flow-id'] +def gradleFlowId = gradleFlowIdRaw ? gradleFlowIdRaw.toLong() : null + +// Real pid of this Gradle JVM โ€” Gradle tasks belong to this process, so +// emitting spans on it matches the process topology. Named via a +// `process_name` metadata event below so Perfetto labels the row +// "gradle" rather than showing a bare number. +def gradlePid = ProcessHandle.current().pid() +def gradleTid = 1 + +def events = Collections.synchronizedList([ + [ + name: 'process_name', + ph: 'M', + pid: gradlePid, + args: [name: 'gradle'], + ], + [ + name: 'thread_name', + ph: 'M', + pid: gradlePid, + tid: gradleTid, + args: [name: 'gradle tasks'], + ], +]) +def firstTaskFlowEmitted = new java.util.concurrent.atomic.AtomicBoolean(false) +def startTimesMicros = Collections.synchronizedMap([:]) + +gradle.addListener(new TaskExecutionListener() { + @Override + void beforeExecute(Task task) { + startTimesMicros[task.path] = System.currentTimeMillis() * 1000L + } + + @Override + void afterExecute(Task task, TaskState state) { + def start = startTimesMicros.remove(task.path) + if (start == null) return + def end = System.currentTimeMillis() * 1000L + def dur = Math.max(0L, end - start) + // Classify the task path into a small set of "kinds" so downstream + // summarizers can bucket without retaining names. Match against the + // task's simple name (last colon segment) so plugin names like + // `:package_info_plus:...` don't accidentally trigger the `packaging` + // bucket. + // + // The kind strings below are a wire-format contract with + // shorebird_cli (which buckets its summary by these values). + // KEEP IN SYNC with the `GradleTaskKind` enum in the + // `shorebird_build_trace` package; adding a new kind means + // landing the matching enum value there AND teaching + // shorebird_cli to bucket it, otherwise it silently falls into + // `other`. + def path = task.path + def lastColon = path.lastIndexOf(':') + def simpleName = (lastColon >= 0 ? path.substring(lastColon + 1) : path).toLowerCase() + + // Order matters: the first regex that matches wins. `kotlin_compile` + // must precede `java_compile` (both match `compile*`). `java_compile` + // must precede the `scaffold` bucket's `prepare*` because AGP's + // javaPreCompileRelease is scaffolding-shaped but semantically javac. + // `flutter_gradle_plugin` must precede `packaging` so tasks like + // `packageFlutterBuild` don't fall through to the `package*` bucket. + def kindRules = [ + [~/^compile.*kotlin.*/, 'kotlin_compile'], + [~/(^compile.*(java|jni).*|.*precompile.*)/, 'java_compile'], + [~/.*aidl.*/, 'aidl'], + [~/(^minify.*|.*r8.*)/, 'r8_minify'], + [~/.*dex.*/, 'dex'], + [~/^lint.*/, 'lint'], + [~/(.*jnilibs.*|.*nativelibs.*|.*link.*native.*)/, 'native_link'], + [~/.*(processresources|mergeresources|mergemanifest|rfile|parserelease|mergerelease|generaterelease).*/, 'resources'], + [~/(^compileflutter.*|.*flutterbuild.*|^flutter.*)/, 'flutter_gradle_plugin'], + [~/^package.*/, 'packaging'], + [~/.*bundle.*/, 'bundle'], + [~/.*transform.*/, 'transform'], + // Catch-all for Gradle's per-plugin / per-variant scaffolding + // (metadata files, proguard rule export, pre-/post-compile + // bookkeeping). Individually small but the long tail adds up on + // plugin-heavy apps. + [~/(.*aarmetadata.*|.*proguard.*|.*validate.*|^check.*|^prepare.*|^generate.*|^copy.*)/, 'gradle_scaffold'], + ] + def kind = kindRules.find { simpleName ==~ it[0] }?.getAt(1) ?: 'other' + + // The task "owner" is the first colon-separated segment. For + // subproject tasks this is typically the plugin name (e.g. + // `:camera_android`). Kept for local debug โ€” the privacy-safe + // summary that Shorebird writes aggregates this out. + def owner = path.startsWith(':') ? path.substring(1).split(':')[0] : '' + + // Emit the flow-end event on the first task, connecting back to + // flutter_tool's "gradle " span. Atomic so we emit at most + // one across concurrent task callbacks. + if (gradleFlowId != null && firstTaskFlowEmitted.compareAndSet(false, true)) { + events.add([ + ph: 'f', + name: 'spawn', + cat: 'flow', + id: gradleFlowId, + ts: start, + pid: gradlePid, + tid: gradleTid, + bp: 'e', + ]) + } + events.add([ + name: path, + cat: 'gradle_task', + ph: 'X', + ts: start, + dur: dur, + pid: gradlePid, + tid: gradleTid, + args: [ + kind: kind, + owner: owner, + skipped: state.skipped, + upToDate: state.upToDate, + fromCache: state.skipMessage == 'FROM-CACHE', + ], + ]) + } +}) + +gradle.buildFinished { + def f = new File(traceFilePath) + f.parentFile?.mkdirs() + f.text = JsonOutput.toJson(events) +} diff --git a/packages/flutter_tools/gradle/src/main/kotlin/FlutterPlugin.kt b/packages/flutter_tools/gradle/src/main/kotlin/FlutterPlugin.kt index 806fc5097a022..c632bbf08ab74 100644 --- a/packages/flutter_tools/gradle/src/main/kotlin/FlutterPlugin.kt +++ b/packages/flutter_tools/gradle/src/main/kotlin/FlutterPlugin.kt @@ -647,6 +647,8 @@ class FlutterPlugin : Plugin { val dartDefinesValue: String? = project.findProperty("dart-defines")?.toString() val performanceMeasurementFileValue: String? = project.findProperty("performance-measurement-file")?.toString() + val shorebirdTraceFileValue: String? = + project.findProperty("shorebird-trace-file")?.toString() val codeSizeDirectoryValue: String? = project.findProperty("code-size-directory")?.toString() val deferredComponentsValue: Boolean = @@ -737,6 +739,7 @@ class FlutterPlugin : Plugin { dartObfuscation = dartObfuscationValue dartDefines = dartDefinesValue performanceMeasurementFile = performanceMeasurementFileValue + shorebirdTraceFile = shorebirdTraceFileValue codeSizeDirectory = codeSizeDirectoryValue deferredComponents = deferredComponentsValue validateDeferredComponents = validateDeferredComponentsValue diff --git a/packages/flutter_tools/gradle/src/main/kotlin/tasks/BaseFlutterTask.kt b/packages/flutter_tools/gradle/src/main/kotlin/tasks/BaseFlutterTask.kt index 1858a139d0e96..96331ef87c94d 100644 --- a/packages/flutter_tools/gradle/src/main/kotlin/tasks/BaseFlutterTask.kt +++ b/packages/flutter_tools/gradle/src/main/kotlin/tasks/BaseFlutterTask.kt @@ -111,6 +111,12 @@ open class BaseFlutterTask : DefaultTask() { @Input var performanceMeasurementFile: String? = null + // Shorebird-specific: path for the Shorebird build trace. Prefixed so + // it stands out as fork-only surface in this otherwise upstream file. + @Optional + @Input + var shorebirdTraceFile: String? = null + @Optional @Input var deferredComponents: Boolean? = null diff --git a/packages/flutter_tools/gradle/src/main/kotlin/tasks/BaseFlutterTaskHelper.kt b/packages/flutter_tools/gradle/src/main/kotlin/tasks/BaseFlutterTaskHelper.kt index 2d16640d324fd..2f5fac2de3a76 100644 --- a/packages/flutter_tools/gradle/src/main/kotlin/tasks/BaseFlutterTaskHelper.kt +++ b/packages/flutter_tools/gradle/src/main/kotlin/tasks/BaseFlutterTaskHelper.kt @@ -110,6 +110,9 @@ object BaseFlutterTaskHelper { baseFlutterTask.performanceMeasurementFile?.let { args("--performance-measurement-file=$it") } + baseFlutterTask.shorebirdTraceFile?.let { + args("--shorebird-trace-file=$it") + } args("-dTargetFile=${baseFlutterTask.targetPath}") args("-dTargetPlatform=android") args("-dBuildMode=${baseFlutterTask.buildMode}") diff --git a/packages/flutter_tools/lib/src/android/gradle.dart b/packages/flutter_tools/lib/src/android/gradle.dart index 2160418c15eea..b5da2688b6237 100644 --- a/packages/flutter_tools/lib/src/android/gradle.dart +++ b/packages/flutter_tools/lib/src/android/gradle.dart @@ -32,6 +32,7 @@ import '../convert.dart'; import '../flutter_manifest.dart'; import '../globals.dart' as globals; import '../project.dart'; +import '../shorebird/android_build_trace_session.dart'; import 'android_builder.dart'; import 'android_sdk.dart'; import 'android_studio.dart'; @@ -289,6 +290,7 @@ class AndroidGradleBuilder implements AndroidBuilder { VoidCallback? postRunTask, int? maxRetries, _OutputParser? outputParser, + void Function(Process process)? onStart, }) async { final bool usesAndroidX = isAppUsingAndroidX(project.android.hostAppGradleRoot); final String? agpVersion = gradle.getAgpVersion( @@ -355,6 +357,7 @@ class AndroidGradleBuilder implements AndroidBuilder { allowReentrantFlutter: true, environment: _java?.environment, mapFunction: consumeLog, + onStart: onStart, ); } on ProcessException catch (exception) { consumeLog(exception.toString()); @@ -537,7 +540,12 @@ To fix this, you can either: return; } - // Assembly work starts here. + final AndroidBuildTraceSession? traceSession = AndroidBuildTraceSession.maybeStart( + androidBuildInfo, + _fileSystem, + project.android.buildDirectory, + ); + final BuildInfo buildInfo = androidBuildInfo.buildInfo; final String assembleTask = isBuildingBundle ? getBundleTaskFor(buildInfo) @@ -614,6 +622,7 @@ To fix this, you can either: } } options.addAll(androidBuildInfo.buildInfo.toGradleConfig()); + options.addAll(traceSession?.extraGradleOptions(assembleTask) ?? const []); if (buildInfo.fileSystemRoots.isNotEmpty) { options.add('-Pfilesystem-roots=${buildInfo.fileSystemRoots.join('|')}'); } @@ -629,8 +638,10 @@ To fix this, you can either: final int exitCode = await _runGradleTask( assembleTask, preRunTask: () { + traceSession?.onGradleAboutToStart(); sw = Stopwatch()..start(); }, + onStart: (process) => traceSession?.onGradleSpawn(process), postRunTask: () { final Duration elapsedDuration = sw.elapsed; _analytics.send( @@ -648,7 +659,12 @@ To fix this, you can either: gradleExecutablePath: gradleExecutablePath, ); + traceSession?.onGradleFinished(assembleTask); + if (exitCode != 0) { + // Abort the trace before the compatibility check, which exits the tool on + // an incompatible pair and would otherwise strand the session. + traceSession?.abortOnGradleFailure(); await _checkJavaAndGradleCompatibility(project, androidBuildInfo.buildInfo); throwToolExit( 'Gradle task $assembleTask failed with exit code $exitCode', @@ -656,6 +672,11 @@ To fix this, you can either: ); } + traceSession?.finish( + buildTarget: isBuildingBundle ? 'appbundle' : 'apk', + printStatus: _logger.printStatus, + ); + if (isBuildingBundle) { final File bundleFile = findBundleFile(project, buildInfo, _logger, _analytics); diff --git a/packages/flutter_tools/lib/src/base/build.dart b/packages/flutter_tools/lib/src/base/build.dart index db63246b9f29b..8c355499634c7 100644 --- a/packages/flutter_tools/lib/src/base/build.dart +++ b/packages/flutter_tools/lib/src/base/build.dart @@ -148,8 +148,7 @@ class AOTSnapshotter { // Shorebird uses --deterministic to improve snapshot stability and increase linking. '--deterministic', // Only save LinkInfo if we're using the linker. - if (usesLinker) - ...dumpLinkInfoArgs, + if (usesLinker) ...dumpLinkInfoArgs, ]; final bool targetingApplePlatform = diff --git a/packages/flutter_tools/lib/src/base/net.dart b/packages/flutter_tools/lib/src/base/net.dart index 248824909cf5e..93f0584cdf071 100644 --- a/packages/flutter_tools/lib/src/base/net.dart +++ b/packages/flutter_tools/lib/src/base/net.dart @@ -7,6 +7,7 @@ import 'dart:async'; import 'package:meta/meta.dart'; import '../convert.dart'; +import '../shorebird/network_trace_span.dart'; import 'common.dart'; import 'file_system.dart'; import 'io.dart'; @@ -87,6 +88,8 @@ class Net { Future _attempt(Uri url, {IOSink? destSink, bool onlyHeaders = false}) async { assert(onlyHeaders || destSink != null); _logger.printTrace('Downloading: $url'); + final traceSpan = NetworkTraceSpan.start(url: url, onlyHeaders: onlyHeaders); + final HttpClient httpClient = _httpClientFactory(); HttpClientRequest request; HttpClientResponse? response; @@ -98,6 +101,7 @@ class Net { } response = await request.close(); } on ArgumentError catch (error) { + traceSpan.record(errorKind: 'ArgumentError'); final String? overrideUrl = _platform.environment[kFlutterStorageBaseUrl]; if (overrideUrl != null && url.toString().contains(overrideUrl)) { _logger.printError(error.toString()); @@ -112,6 +116,7 @@ class Net { _logger.printError(error.toString()); rethrow; } on HandshakeException catch (error) { + traceSpan.record(errorKind: 'HandshakeException'); _logger.printTrace(error.toString()); throwToolExit( 'Could not authenticate download server. You may be experiencing a man-in-the-middle attack,\n' @@ -120,29 +125,35 @@ class Net { exitCode: kNetworkProblemExitCode, ); } on SocketException catch (error) { + traceSpan.record(errorKind: 'SocketException'); _logger.printTrace('Download error: $error'); return false; } on HttpException catch (error) { + traceSpan.record(errorKind: 'HttpException'); _logger.printTrace('Download error: $error'); return false; } + final int statusCode = response.statusCode; + // If we're making a HEAD request, we're only checking to see if the URL is // valid. if (onlyHeaders) { - return response.statusCode == HttpStatus.ok; + traceSpan.record(statusCode: statusCode); + return statusCode == HttpStatus.ok; } - if (response.statusCode != HttpStatus.ok) { - if (response.statusCode > 0 && response.statusCode < 500) { + if (statusCode != HttpStatus.ok) { + traceSpan.record(statusCode: statusCode); + if (statusCode > 0 && statusCode < 500) { throwToolExit( 'Download failed.\n' 'URL: $url\n' - 'Error: ${response.statusCode} ${response.reasonPhrase}', + 'Error: $statusCode ${response.reasonPhrase}', exitCode: kNetworkProblemExitCode, ); } // 5xx errors are server errors and we can try again - _logger.printTrace('Download error: ${response.statusCode} ${response.reasonPhrase}'); + _logger.printTrace('Download error: $statusCode ${response.reasonPhrase}'); return false; } _logger.printTrace('Received response from server, collecting bytes...'); @@ -156,6 +167,9 @@ class Net { } finally { await destSink?.flush(); await destSink?.close(); + // Record after the body has been fully drained so `dur` reflects + // end-to-end download time, not just time-to-first-byte. + traceSpan.record(statusCode: statusCode); } } } diff --git a/packages/flutter_tools/lib/src/base/process.dart b/packages/flutter_tools/lib/src/base/process.dart index 2bcaa39d45288..a45b1697590c4 100644 --- a/packages/flutter_tools/lib/src/base/process.dart +++ b/packages/flutter_tools/lib/src/base/process.dart @@ -238,6 +238,7 @@ abstract class ProcessUtils { RegExp? stdoutErrorMatcher, StringConverter? mapFunction, Map? environment, + void Function(Process process)? onStart, }); bool exitsHappySync(List cli, {Map? environment}); @@ -554,6 +555,7 @@ class _DefaultProcessUtils implements ProcessUtils { RegExp? stdoutErrorMatcher, StringConverter? mapFunction, Map? environment, + void Function(Process process)? onStart, }) async { final Process process = await start( cmd, @@ -561,6 +563,7 @@ class _DefaultProcessUtils implements ProcessUtils { allowReentrantFlutter: allowReentrantFlutter, environment: environment, ); + onStart?.call(process); final StreamSubscription stdoutSubscription = process.stdout .transform(utf8AllowMalformedLineDecoder) .where((String line) => filter == null || filter.hasMatch(line)) diff --git a/packages/flutter_tools/lib/src/build_info.dart b/packages/flutter_tools/lib/src/build_info.dart index fab5066e5c29a..87f3915053887 100644 --- a/packages/flutter_tools/lib/src/build_info.dart +++ b/packages/flutter_tools/lib/src/build_info.dart @@ -44,6 +44,7 @@ class BuildInfo { List? dartExperiments, required this.treeShakeIcons, this.performanceMeasurementFile, + this.shorebirdTraceFilePath, required this.packageConfigPath, this.codeSizeDirectory, this.androidGradleDaemon = true, @@ -179,6 +180,12 @@ class BuildInfo { /// rerun tasks. final String? performanceMeasurementFile; + /// Path to the Shorebird build-trace output file. When non-null, the + /// Shorebird-specific build-trace plumbing emits a Chrome Trace Event + /// Format JSON file here. Named `shorebird` to keep the fork's + /// surface area visibly distinct from upstream Flutter. + final String? shorebirdTraceFilePath; + /// If provided, an output directory where one or more v8-style heap snapshots /// will be written for code size profiling. final String? codeSizeDirectory; diff --git a/packages/flutter_tools/lib/src/build_system/build_system.dart b/packages/flutter_tools/lib/src/build_system/build_system.dart index 9abcda6d39c55..d6bc3becca308 100644 --- a/packages/flutter_tools/lib/src/build_system/build_system.dart +++ b/packages/flutter_tools/lib/src/build_system/build_system.dart @@ -882,6 +882,7 @@ class _BuildInstance { Future _invokeInternal(Node node) async { final PoolResource resource = await resourcePool.request(); + final int startTimeMicroseconds = DateTime.now().microsecondsSinceEpoch; final stopwatch = Stopwatch()..start(); var succeeded = true; var skipped = false; @@ -982,6 +983,7 @@ class _BuildInstance { skipped: skipped, succeeded: succeeded, analyticsName: node.target.analyticsName, + startTimeMicroseconds: startTimeMicroseconds, ); } return succeeded; @@ -1011,6 +1013,7 @@ class PerformanceMeasurement { required this.skipped, required this.succeeded, required this.analyticsName, + required this.startTimeMicroseconds, }); final int elapsedMilliseconds; @@ -1018,6 +1021,9 @@ class PerformanceMeasurement { final bool skipped; final bool succeeded; final String analyticsName; + + /// Wall-clock start time in microseconds since epoch. + final int startTimeMicroseconds; } /// Check if there are any dependency cycles in the target. diff --git a/packages/flutter_tools/lib/src/commands/assemble.dart b/packages/flutter_tools/lib/src/commands/assemble.dart index 61203a9f33519..e3223e7d84a27 100644 --- a/packages/flutter_tools/lib/src/commands/assemble.dart +++ b/packages/flutter_tools/lib/src/commands/assemble.dart @@ -4,6 +4,7 @@ import 'package:args/args.dart'; import 'package:meta/meta.dart'; +import 'package:shorebird_build_trace/shorebird_build_trace.dart'; import 'package:unified_analytics/unified_analytics.dart'; import '../artifacts.dart'; @@ -116,6 +117,13 @@ class AssembleCommand extends FlutterCommand { 'performance-measurement-file', help: 'Output individual target performance to a JSON file.', ); + argParser.addOption( + 'shorebird-trace-file', + help: + 'Output a Shorebird build trace in Chrome Trace Event Format ' + 'JSON. Shorebird-specific; used by the build-trace plumbing in ' + 'gradle.dart / mac.dart to collect flutter-assemble timings.', + ); argParser.addMultiOption( 'input', abbr: 'i', @@ -398,6 +406,10 @@ class AssembleCommand extends FlutterCommand { final File outFile = globals.fs.file(argumentResults['performance-measurement-file']); writePerformanceData(result.performance.values, outFile); } + if (argumentResults.wasParsed('shorebird-trace-file')) { + final File outFile = globals.fs.file(argumentResults['shorebird-trace-file']); + writeTraceData(result.performance.values, outFile); + } if (argumentResults.wasParsed('depfile')) { final File depfileFile = globals.fs.file(stringArg('depfile')); final depfile = Depfile(result.inputFiles, result.outputFiles); @@ -444,3 +456,35 @@ void writePerformanceData(Iterable measurements, File ou } outFile.writeAsStringSync(json.encode(jsonData)); } + +/// Output build trace data in Chrome Trace Event Format in [outFile]. +/// `flutter assemble` is re-entered as its own process by Gradle / +/// xcode_backend, so its events get their own Perfetto row named +/// `flutter assemble`. Spans for each [PerformanceMeasurement] are +/// emitted using its wall-clock [PerformanceMeasurement.startTimeMicroseconds] +/// (converted to a DateTime; not the stopwatch duration alone) so the +/// timeline aligns with the parent flutter tool's trace when merged. +@visibleForTesting +void writeTraceData(Iterable measurements, File outFile) { + final int pid = currentProcessId(); + final tracer = BuildTracer() + ..addProcessNameMetadata(pid: pid, name: 'flutter assemble') + ..addThreadNameMetadata(pid: pid, tid: 1, name: 'flutter assemble'); + for (final measurement in measurements) { + final start = DateTime.fromMicrosecondsSinceEpoch(measurement.startTimeMicroseconds); + tracer.addCompleteEvent( + name: measurement.analyticsName, + cat: TraceCategory.assemble.wireName, + pid: pid, + tid: 1, + start: start, + end: start.add(Duration(milliseconds: measurement.elapsedMilliseconds)), + args: { + 'target': measurement.target, + 'skipped': measurement.skipped, + 'succeeded': measurement.succeeded, + }, + ); + } + tracer.writeToFile(outFile); +} diff --git a/packages/flutter_tools/lib/src/commands/build_apk.dart b/packages/flutter_tools/lib/src/commands/build_apk.dart index 184a68be58d90..231b6650e18ac 100644 --- a/packages/flutter_tools/lib/src/commands/build_apk.dart +++ b/packages/flutter_tools/lib/src/commands/build_apk.dart @@ -31,6 +31,7 @@ class BuildApkCommand extends BuildSubCommand { usesExtraDartFlagOptions(verboseHelp: verboseHelp); addEnableExperimentation(hide: !verboseHelp); addBuildPerformanceFile(hide: !verboseHelp); + usesShorebirdTraceOption(hide: !verboseHelp); usesAnalyzeSizeFlag(); addAndroidSpecificBuildOptions(hide: !verboseHelp); addIgnoreDeprecationOption(); diff --git a/packages/flutter_tools/lib/src/commands/build_appbundle.dart b/packages/flutter_tools/lib/src/commands/build_appbundle.dart index af58cbe68fa11..32ab81b8bab68 100644 --- a/packages/flutter_tools/lib/src/commands/build_appbundle.dart +++ b/packages/flutter_tools/lib/src/commands/build_appbundle.dart @@ -33,6 +33,7 @@ class BuildAppBundleCommand extends BuildSubCommand { usesDartDefineOption(); usesExtraDartFlagOptions(verboseHelp: verboseHelp); addBuildPerformanceFile(hide: !verboseHelp); + usesShorebirdTraceOption(hide: !verboseHelp); usesTrackWidgetCreation(verboseHelp: verboseHelp); addEnableExperimentation(hide: !verboseHelp); usesAnalyzeSizeFlag(); diff --git a/packages/flutter_tools/lib/src/commands/build_ios.dart b/packages/flutter_tools/lib/src/commands/build_ios.dart index e783f793b9e7a..6cfb5b75dbc8a 100644 --- a/packages/flutter_tools/lib/src/commands/build_ios.dart +++ b/packages/flutter_tools/lib/src/commands/build_ios.dart @@ -907,6 +907,7 @@ abstract class _BuildIOSSubCommand extends BuildSubCommand { usesExtraDartFlagOptions(verboseHelp: verboseHelp); addEnableExperimentation(hide: !verboseHelp); addBuildPerformanceFile(hide: !verboseHelp); + usesShorebirdTraceOption(hide: !verboseHelp); usesAnalyzeSizeFlag(); argParser.addFlag( 'codesign', diff --git a/packages/flutter_tools/lib/src/ios/application_package.dart b/packages/flutter_tools/lib/src/ios/application_package.dart index e8613d4d7a8bd..0bc7bb5a45851 100644 --- a/packages/flutter_tools/lib/src/ios/application_package.dart +++ b/packages/flutter_tools/lib/src/ios/application_package.dart @@ -130,17 +130,16 @@ class BuildableIOSApp extends IOSApp { @override String? get name => _appProductName; - String get shorebirdYamlPath => - globals.fs.path.join( - archiveBundleOutputPath, - 'Products', - 'Applications', - _appProductName != null ? '$_appProductName.app' : 'Runner.app', - 'Frameworks', - 'App.framework', - 'flutter_assets', - 'shorebird.yaml', - ); + String get shorebirdYamlPath => globals.fs.path.join( + archiveBundleOutputPath, + 'Products', + 'Applications', + _appProductName != null ? '$_appProductName.app' : 'Runner.app', + 'Frameworks', + 'App.framework', + 'flutter_assets', + 'shorebird.yaml', + ); @override String get simulatorBundlePath => _buildAppPath(XcodeSdk.IPhoneSimulator.platformName); diff --git a/packages/flutter_tools/lib/src/ios/mac.dart b/packages/flutter_tools/lib/src/ios/mac.dart index d5fec0e08c4c9..ace212b6c3a8a 100644 --- a/packages/flutter_tools/lib/src/ios/mac.dart +++ b/packages/flutter_tools/lib/src/ios/mac.dart @@ -38,6 +38,7 @@ import '../migrations/xcode_script_build_phase_migration.dart'; import '../migrations/xcode_thin_binary_build_phase_input_paths_migration.dart'; import '../plugins.dart'; import '../project.dart'; +import '../shorebird/ios_build_trace_session.dart'; import 'application_package.dart'; import 'code_signing.dart'; import 'migrations/host_app_info_plist_migration.dart'; @@ -384,11 +385,24 @@ Future buildXcodeProject({ ); } } + // Created early enough to capture pod install, which can be minutes + // on plugin-heavy apps โ€” otherwise it's an invisible gap between + // "flutter build ios" starting and the xcode archive span. + final IosBuildTraceSession? traceSession = IosBuildTraceSession.maybeStart( + shorebirdTraceFilePath: buildInfo.shorebirdTraceFilePath, + fileSystem: globals.fs, + buildDirectoryPath: globals.fs.path.join(getBuildDirectory(), 'ios'), + ); + + traceSession?.onBeforePodInstall(); await processPodsIfNeeded(project.ios, buildDirectoryPath, buildInfo.mode); + traceSession?.onAfterPodInstall(); if (configOnly) { return XcodeBuildResult(success: true); } + buildCommands.addAll(traceSession?.extraBuildCommands() ?? const []); + if (globals.logger.isVerbose) { // An environment variable to be passed to xcode_backend.sh determining // whether to echo back executed commands. @@ -567,6 +581,8 @@ Future buildXcodeProject({ ]); } + traceSession?.onXcodeAboutToStart(); + final sw = Stopwatch()..start(); initialBuildStatus = globals.logger.startProgress('Running Xcode build...'); @@ -591,6 +607,13 @@ Future buildXcodeProject({ ), ); + await traceSession?.onXcodeFinished( + buildActionName: xcodeBuildActionToString(buildAction), + resultBundleDirectory: resultBundleDirectory, + runXcresultTool: (List args) => globals.processManager.run(args), + ); + traceSession?.finish(printStatus: globals.printStatus); + if (tempDir.existsSync()) { // Display additional warning and error message from xcresult bundle. final Directory resultBundle = tempDir.childDirectory(_kResultBundlePath); diff --git a/packages/flutter_tools/lib/src/macos/cocoapods.dart b/packages/flutter_tools/lib/src/macos/cocoapods.dart index 0fe7a28879cd7..202792035f9bf 100644 --- a/packages/flutter_tools/lib/src/macos/cocoapods.dart +++ b/packages/flutter_tools/lib/src/macos/cocoapods.dart @@ -2,9 +2,12 @@ // Use of this source code is governed by a BSD-style license that can be // found in the LICENSE file. +import 'dart:convert'; import 'dart:ffi' show Abi; + import 'package:file/file.dart'; import 'package:process/process.dart'; +import 'package:shorebird_build_trace/shorebird_build_trace.dart'; import 'package:unified_analytics/unified_analytics.dart'; import '../base/common.dart'; @@ -351,16 +354,28 @@ class CocoaPods { Future _runPodInstall(XcodeBasedProject xcodeProject, BuildMode buildMode) async { final Status status = _logger.startProgress('Running pod install...'); - final ProcessResult result = await _processManager.run( - ['pod', 'install', '--verbose'], - workingDirectory: _fileSystem.path.dirname(xcodeProject.podfile.path), - environment: { - // See https://github.com/flutter/flutter/issues/10873. - // CocoaPods analytics adds a lot of latency. - 'COCOAPODS_DISABLE_STATS': 'true', - 'LANG': 'en_US.UTF-8', - }, - ); + + // Shorebird-specific: when a build trace is active, stream pod install + // output so we can timestamp phase transitions. On plugin-heavy apps + // pod install can be minutes, and a single opaque span hides where the + // time goes (dependency analysis vs downloads vs project generation vs + // integration). + final BuildTracer? tracer = BuildTracer.current; + final ProcessResult result; + if (tracer == null) { + result = await _processManager.run( + ['pod', 'install', '--verbose'], + workingDirectory: _fileSystem.path.dirname(xcodeProject.podfile.path), + environment: { + // See https://github.com/flutter/flutter/issues/10873. + // CocoaPods analytics adds a lot of latency. + 'COCOAPODS_DISABLE_STATS': 'true', + 'LANG': 'en_US.UTF-8', + }, + ); + } else { + result = await _runPodInstallStreamed(tracer, xcodeProject); + } status.stop(); if (_logger.isVerbose || result.exitCode != 0) { final stdout = result.stdout as String; @@ -389,6 +404,60 @@ class CocoaPods { } } + /// Runs `pod install --verbose` with live stdout streaming so phase + /// transitions can be timestamped into [tracer]. Returns a [ProcessResult] + /// shaped the same as [ProcessManager.run] would have returned, so + /// downstream error handling keeps working unchanged. + Future _runPodInstallStreamed( + BuildTracer tracer, + XcodeBasedProject xcodeProject, + ) async { + final Process process = await _processManager.start( + ['pod', 'install', '--verbose'], + workingDirectory: _fileSystem.path.dirname(xcodeProject.podfile.path), + environment: {'COCOAPODS_DISABLE_STATS': 'true', 'LANG': 'en_US.UTF-8'}, + ); + // Real pid of the `pod` process. Phase spans land on this pid + + // tid=1, with a `process_name` metadata event so Perfetto groups + // pod install work into its own row. + final int podPid = process.pid; + tracer + ..addProcessNameMetadata(pid: podPid, name: TraceNames.podInstallSpanName) + ..addThreadNameMetadata(pid: podPid, tid: 1, name: TraceNames.podInstallSpanName); + final phases = PhaseTracker( + tracer: tracer, + pid: podPid, + tid: 1, + namePrefix: TraceNames.podInstallNamePrefix, + ); + + final stdoutBuf = StringBuffer(); + final stderrBuf = StringBuffer(); + + final Future stdoutDone = process.stdout + .transform(utf8.decoder) + .transform(const LineSplitter()) + .listen((String line) { + stdoutBuf + ..write(line) + ..write('\n'); + final PodInstallPhase? phase = _podPhaseForLogLine(line); + if (phase != null) phases.transitionTo(phase.wireName); + }) + .asFuture(); + + final Future stderrDone = process.stderr + .transform(utf8.decoder) + .listen(stderrBuf.write) + .asFuture(); + + final int exitCode = await process.exitCode; + await Future.wait(>[stdoutDone, stderrDone]); + phases.end(); + + return ProcessResult(process.pid, exitCode, stdoutBuf.toString(), stderrBuf.toString()); + } + Future _diagnosePodInstallFailure( ProcessResult result, XcodeBasedProject xcodeProject, @@ -644,3 +713,27 @@ class CocoaPods { } } } + +/// CocoaPods verbose-mode log markers, in the order they print during a +/// normal `pod install`. A match means "from now on, the process is in +/// this phase" โ€” the `PhaseTracker` closes the prior phase and opens a +/// new one. +/// +/// These substrings have been stable across recent CocoaPods releases; +/// if CocoaPods renames one, the worst case is we miss a phase in the +/// trace โ€” pod install still succeeds. +const _podPhaseMarkers = { + 'Analyzing dependencies': PodInstallPhase.analyzing, + 'Downloading dependencies': PodInstallPhase.downloading, + 'Generating Pods project': PodInstallPhase.generating, + 'Integrating client project': PodInstallPhase.integrating, +}; + +/// Returns the phase a `pod install --verbose` log [line] transitions +/// into, or null if the line doesn't contain any of the known markers. +PodInstallPhase? _podPhaseForLogLine(String line) { + for (final MapEntry entry in _podPhaseMarkers.entries) { + if (line.contains(entry.key)) return entry.value; + } + return null; +} diff --git a/packages/flutter_tools/lib/src/runner/flutter_command.dart b/packages/flutter_tools/lib/src/runner/flutter_command.dart index 1671d157bcc46..d99061f183d82 100644 --- a/packages/flutter_tools/lib/src/runner/flutter_command.dart +++ b/packages/flutter_tools/lib/src/runner/flutter_command.dart @@ -130,6 +130,10 @@ abstract final class FlutterOptions { static const kDartDefineFromFileOption = 'dart-define-from-file'; static const kWebDefinesOption = 'web-define'; static const kPerformanceMeasurementFile = 'performance-measurement-file'; + // Shorebird-specific build-trace option. Prefixed so the flag name and + // generated env vars / Gradle properties don't squat on identifiers + // upstream Flutter might want later. + static const kShorebirdTrace = 'shorebird-trace'; static const kDeviceUser = 'device-user'; static const kDeviceTimeout = 'device-timeout'; static const kDeviceConnection = 'device-connection'; @@ -1065,6 +1069,19 @@ abstract class FlutterCommand extends Command { ); } + void usesShorebirdTraceOption({bool hide = false}) { + argParser.addOption( + FlutterOptions.kShorebirdTrace, + help: + 'Output a Chrome Trace Event Format JSON file for build ' + 'profiling. The resulting file can be viewed at ' + 'https://ui.perfetto.dev. Shorebird-specific; named to avoid ' + 'colliding with any future upstream trace option.', + hide: hide, + valueHelp: 'path/to/shorebird-trace.json', + ); + } + void addAndroidSpecificBuildOptions({bool hide = false}) { argParser.addFlag( FlutterOptions.kAndroidGradleDaemon, @@ -1483,6 +1500,11 @@ abstract class FlutterCommand extends Command { ? stringArg(FlutterOptions.kPerformanceMeasurementFile) : null; + final String? shorebirdTraceFilePath = + argParser.options.containsKey(FlutterOptions.kShorebirdTrace) + ? stringArg(FlutterOptions.kShorebirdTrace) + : null; + final Map defineConfigJsonMap = extractDartDefineConfigJsonMap(); final List dartDefines = extractDartDefines(defineConfigJsonMap: defineConfigJsonMap); @@ -1535,6 +1557,7 @@ abstract class FlutterCommand extends Command { dartDefines: dartDefines, dartExperiments: experiments, performanceMeasurementFile: performanceMeasurementFile, + shorebirdTraceFilePath: shorebirdTraceFilePath, packageConfigPath: packagesPath ?? packageConfigFile.path, codeSizeDirectory: codeSizeDirectory, androidGradleDaemon: androidGradleDaemon, diff --git a/packages/flutter_tools/lib/src/shorebird/android_build_trace_session.dart b/packages/flutter_tools/lib/src/shorebird/android_build_trace_session.dart new file mode 100644 index 0000000000000..5b01afa30fbc5 --- /dev/null +++ b/packages/flutter_tools/lib/src/shorebird/android_build_trace_session.dart @@ -0,0 +1,210 @@ +// Shorebird-specific. Keeps the build-trace plumbing for +// `flutter build apk` / `flutter build appbundle` out of gradle.dart +// so the Shorebird fork's diff against upstream stays small and the +// build flow reads the same as upstream. + +import 'dart:io' show Process; + +import 'package:shorebird_build_trace/shorebird_build_trace.dart'; + +import '../base/file_system.dart'; +import '../build_info.dart'; +import '../cache.dart'; +import 'network_trace_span.dart'; + +/// Wraps the lifecycle of a Shorebird build trace across one +/// `buildGradleApp` call. Returned by [maybeStart] only when +/// `--shorebird-trace=` was passed; the constructor installs +/// [BuildTracer.current] and [finish] / [abortOnGradleFailure] +/// clear it, so gradle.dart itself never touches the static. +/// +/// Manual start/stop (rather than a body-wrapping closure) because +/// wrapping the ~200-line `buildGradleApp` body would force a reindent +/// that balloons our diff against upstream flutter and makes every +/// subsequent merge of that method harder. +class AndroidBuildTraceSession { + AndroidBuildTraceSession._({ + required BuildTracer tracer, + required FileSystem fileSystem, + required String tracePath, + required Directory buildDirectory, + }) : _tracer = tracer, + _fs = fileSystem, + _tracePath = tracePath, + _buildDirectory = buildDirectory, + _flutterPid = currentProcessId(), + _buildStart = DateTime.now() { + BuildTracer.start(_tracer); + _tracer + ..addProcessNameMetadata(pid: _flutterPid, name: 'flutter_tool') + ..addThreadNameMetadata(pid: _flutterPid, tid: _flutterToolTid, name: 'flutter tool') + ..addThreadNameMetadata(pid: _flutterPid, tid: _gradleWaitTid, name: 'gradle (wait)') + ..addThreadNameMetadata(pid: _flutterPid, tid: networkTid, name: 'network'); + } + + /// Returns a session when a trace path is configured for this build, + /// null otherwise. + static AndroidBuildTraceSession? maybeStart( + AndroidBuildInfo androidBuildInfo, + FileSystem fileSystem, + Directory buildDirectory, + ) { + final String? path = androidBuildInfo.buildInfo.shorebirdTraceFilePath; + if (path == null) { + return null; + } + return AndroidBuildTraceSession._( + tracer: BuildTracer(), + fileSystem: fileSystem, + tracePath: path, + buildDirectory: buildDirectory, + ); + } + + final BuildTracer _tracer; + final FileSystem _fs; + final String _tracePath; + final Directory _buildDirectory; + final int _flutterPid; + final DateTime _buildStart; + + static const int _flutterToolTid = 1; + static const int _gradleWaitTid = 2; + + String? _assembleTraceFilePath; + String? _gradleTaskTraceFilePath; + DateTime? _gradleStart; + DateTime? _gradleEnd; + + /// Returns the `-P`/`-I=` flags to thread trace plumbing into Gradle. + /// Safe to splat into the options list unconditionally. + /// + /// [assembleTask] carries the variant + build type (e.g. + /// `assembleFooRelease` / `bundleRelease`); the intermediate trace + /// paths embed it so a single Gradle invocation running multiple + /// variants in parallel can't have per-variant FlutterTask instances + /// stomp on each other's trace. + List extraGradleOptions(String assembleTask) { + final String assembleTrace = _fs.path.join( + _buildDirectory.path, + 'intermediates', + 'shorebird', + 'shorebird_assemble_trace_$assembleTask.json', + ); + final String gradleTaskTrace = _fs.path.join( + _buildDirectory.path, + 'intermediates', + 'shorebird', + 'shorebird_gradle_task_trace_$assembleTask.json', + ); + final String initScript = _fs.path.join( + Cache.flutterRoot!, + 'packages', + 'flutter_tools', + 'gradle', + 'shorebird_trace_init.gradle', + ); + _assembleTraceFilePath = assembleTrace; + _gradleTaskTraceFilePath = gradleTaskTrace; + return [ + '-Pshorebird-trace-file=$assembleTrace', + '-I=$initScript', + '-Pshorebird.gradle-trace-file=$gradleTaskTrace', + ]; + } + + /// Hook for `_runGradleTask`'s `preRunTask` โ€” records when the Gradle + /// wrapper is about to run so [finish] can compute the outer build + /// span. + void onGradleAboutToStart() { + final preGradleEnd = DateTime.now(); + _tracer.addCompleteEvent( + name: 'pre-gradle setup', + cat: TraceCategory.flutter.wireName, + pid: _flutterPid, + tid: _flutterToolTid, + start: _buildStart, + end: preGradleEnd, + ); + _gradleStart = DateTime.now(); + } + + /// Hook for `_runGradleTask`'s `onStart` โ€” emits a flow-start tied to + /// Gradle's real pid so Perfetto draws an arrow from our + /// `gradle ` span into the first per-task event the init script + /// emits. + void onGradleSpawn(Process process) { + _tracer.addFlowStart( + id: process.pid, + pid: _flutterPid, + tid: _gradleWaitTid, + at: DateTime.now(), + ); + } + + /// Call after `_runGradleTask` returns, before the exit-code check. + /// Records the outer `gradle ` span and merges the intermediate + /// trace files into the main tracer. + void onGradleFinished(String assembleTask) { + final gradleEnd = DateTime.now(); + _gradleEnd = gradleEnd; + _tracer.addCompleteEvent( + name: '${TraceNames.gradleSpanPrefix}$assembleTask', + cat: TraceCategory.gradle.wireName, + pid: _flutterPid, + tid: _gradleWaitTid, + start: _gradleStart ?? _buildStart, + end: gradleEnd, + ); + final String? assembleTraceFilePath = _assembleTraceFilePath; + if (assembleTraceFilePath != null) { + _tracer.mergeEventsFromFile(_fs.file(assembleTraceFilePath)); + } + final String? gradleTaskTraceFilePath = _gradleTaskTraceFilePath; + if (gradleTaskTraceFilePath != null) { + _tracer.mergeEventsFromFile(_fs.file(gradleTaskTraceFilePath)); + } + } + + /// Call right before `throwToolExit` when Gradle returned non-zero โ€” + /// clears [BuildTracer.current] so the caught ToolExit doesn't leak + /// tracer state into any later code that reads [BuildTracer.current]. + void abortOnGradleFailure() { + BuildTracer.stop(); + } + + /// Writes the merged trace to disk and clears [BuildTracer.current]. + /// Records post-gradle + outer "flutter build" spans first. + /// + /// [buildTarget] is the target suffix (e.g. `apk`, `appbundle`); the + /// outer span name is assembled as + /// `TraceNames.flutterBuildSpanPrefix + buildTarget`. + /// [printStatus] is called once with a user-facing "trace written" + /// message so callers don't have to wire the logger through. + void finish({required String buildTarget, required void Function(String) printStatus}) { + final postGradleEnd = DateTime.now(); + _tracer + ..addCompleteEvent( + name: 'post-gradle processing', + cat: TraceCategory.flutter.wireName, + pid: _flutterPid, + tid: _flutterToolTid, + start: _gradleEnd ?? postGradleEnd, + end: postGradleEnd, + ) + ..addCompleteEvent( + name: '${TraceNames.flutterBuildSpanPrefix}$buildTarget', + cat: TraceCategory.flutter.wireName, + pid: _flutterPid, + tid: _flutterToolTid, + start: _buildStart, + end: postGradleEnd, + ) + ..writeToFile(_fs.file(_tracePath)); + printStatus( + 'Shorebird build trace written to $_tracePath. ' + 'View at https://ui.perfetto.dev', + ); + BuildTracer.stop(); + } +} diff --git a/packages/flutter_tools/lib/src/shorebird/ios_build_trace_session.dart b/packages/flutter_tools/lib/src/shorebird/ios_build_trace_session.dart new file mode 100644 index 0000000000000..d8a7ea3da8e55 --- /dev/null +++ b/packages/flutter_tools/lib/src/shorebird/ios_build_trace_session.dart @@ -0,0 +1,281 @@ +// Shorebird-specific. Keeps the build-trace plumbing for +// `flutter build ios` out of mac.dart so the Shorebird fork's diff +// against upstream stays small and the build flow reads the same as +// upstream. + +import 'dart:async'; +import 'dart:convert'; +import 'dart:io' show ProcessResult; + +import 'package:shorebird_build_trace/shorebird_build_trace.dart'; + +import '../base/file_system.dart'; +import 'network_trace_span.dart'; + +/// Wraps the lifecycle of a Shorebird build trace across one iOS +/// build. Returned by [maybeStart] only when `--shorebird-trace=` +/// was passed; the constructor installs [BuildTracer.current] and +/// [finish] / [abortOnFailure] clear it, so mac.dart itself never +/// touches the static. +/// +/// Manual start/stop (rather than a body-wrapping closure) because +/// wrapping the ~300-line `buildXcodeProject` body would force a +/// reindent that balloons our diff against upstream flutter and makes +/// every subsequent merge of that method harder. +class IosBuildTraceSession { + IosBuildTraceSession._({ + required BuildTracer tracer, + required FileSystem fileSystem, + required String tracePath, + required String buildDirectoryPath, + }) : _tracer = tracer, + _fs = fileSystem, + _tracePath = tracePath, + _buildDirectoryPath = buildDirectoryPath, + _flutterPid = currentProcessId(), + _buildStart = DateTime.now() { + BuildTracer.start(_tracer); + _tracer + ..addProcessNameMetadata(pid: _flutterPid, name: 'flutter_tool') + ..addThreadNameMetadata(pid: _flutterPid, tid: _flutterToolTid, name: 'flutter tool') + ..addThreadNameMetadata(pid: _flutterPid, tid: _xcodeWaitTid, name: 'xcode (wait)') + ..addThreadNameMetadata(pid: _flutterPid, tid: networkTid, name: 'network') + ..addProcessNameMetadata(pid: _xcodeSubsectionPid, name: 'xcodebuild') + ..addThreadNameMetadata(pid: _xcodeSubsectionPid, tid: 1, name: 'xcode phases'); + } + + /// Returns a session when a trace path is configured for this build, + /// null otherwise. [buildDirectoryPath] is the iOS build directory + /// (relative to the project root) where the intermediate assemble + /// trace file is written. + static IosBuildTraceSession? maybeStart({ + required String? shorebirdTraceFilePath, + required FileSystem fileSystem, + required String buildDirectoryPath, + }) { + if (shorebirdTraceFilePath == null) { + return null; + } + return IosBuildTraceSession._( + tracer: BuildTracer(), + fileSystem: fileSystem, + tracePath: shorebirdTraceFilePath, + buildDirectoryPath: buildDirectoryPath, + ); + } + + final BuildTracer _tracer; + final FileSystem _fs; + final String _tracePath; + final String _buildDirectoryPath; + final int _flutterPid; + final DateTime _buildStart; + + static const int _flutterToolTid = 1; + static const int _xcodeWaitTid = 2; + + /// Synthetic pid for Xcode subsection events parsed from xcresult. + /// xcresult doesn't surface xcodebuild's pid, so events end up on + /// this fixed id named via a `process_name` metadata event. + static const int _xcodeSubsectionPid = 0x78636f; // ASCII 'xco' + + DateTime? _podInstallStart; + DateTime? _xcodeStart; + DateTime? _xcodeEnd; + String? _assembleTraceFilePath; + + /// Call immediately before `processPodsIfNeeded` so the resulting + /// span covers its full wall-clock. + void onBeforePodInstall() { + _podInstallStart = DateTime.now(); + } + + /// Call immediately after `processPodsIfNeeded` returns. Records the + /// `pod install` complete event using the start timestamp captured + /// by [onBeforePodInstall]. + void onAfterPodInstall() { + final DateTime? start = _podInstallStart; + if (start == null) { + return; + } + _tracer.addCompleteEvent( + name: TraceNames.podInstallSpanName, + cat: TraceCategory.subprocess.wireName, + pid: _flutterPid, + tid: _flutterToolTid, + start: start, + end: DateTime.now(), + ); + } + + /// Returns the extra xcodebuild arguments that thread trace plumbing + /// into the build phase script. Safe to splat into the command list + /// unconditionally. + List extraBuildCommands() { + final String assembleTrace = _fs.path.join( + _fs.currentDirectory.path, + _buildDirectoryPath, + 'shorebird_assemble_trace.json', + ); + _assembleTraceFilePath = assembleTrace; + // Naming the env var SHOREBIRD_TRACE_FILE avoids squatting on an + // Xcode-reserved prefix. Read by xcode_backend.dart. + return ['SHOREBIRD_TRACE_FILE=$assembleTrace']; + } + + /// Call right before `_runBuildWithRetries` so the outer build span + /// can compute the pre-xcode setup interval. + void onXcodeAboutToStart() { + final xcodeStart = DateTime.now(); + _xcodeStart = xcodeStart; + _tracer.addCompleteEvent( + name: 'pre-xcode setup', + cat: TraceCategory.flutter.wireName, + pid: _flutterPid, + tid: _flutterToolTid, + start: _buildStart, + end: xcodeStart, + ); + } + + /// Call after `_runBuildWithRetries` returns. Records the outer + /// `xcode ` span, then emits per-subsection events pulled + /// from xcresulttool, then merges the intermediate assemble trace. + /// + /// [runXcresultTool] is injected so callers can route through their + /// own `ProcessManager`; the session avoids a direct `globals.*` + /// dependency. + Future onXcodeFinished({ + required String buildActionName, + required Directory resultBundleDirectory, + required Future Function(List) runXcresultTool, + }) async { + final xcodeEnd = DateTime.now(); + _xcodeEnd = xcodeEnd; + _tracer.addCompleteEvent( + name: '${TraceNames.xcodeSpanPrefix}$buildActionName', + cat: TraceCategory.xcode.wireName, + pid: _flutterPid, + tid: _xcodeWaitTid, + start: _xcodeStart ?? _buildStart, + end: xcodeEnd, + ); + await _emitXcodeSubsectionEvents( + resultBundleDirectory: resultBundleDirectory, + runXcresultTool: runXcresultTool, + ); + final String? assembleTraceFilePath = _assembleTraceFilePath; + if (assembleTraceFilePath != null) { + _tracer.mergeEventsFromFile(_fs.file(assembleTraceFilePath)); + } + } + + /// Clears [BuildTracer.current] without writing the trace. Use on + /// error paths where [finish] won't run. + void abortOnFailure() { + BuildTracer.stop(); + } + + /// Writes the merged trace to disk and clears [BuildTracer.current]. + /// Records the post-xcode + outer `flutter build ios` spans first. + void finish({required void Function(String) printStatus}) { + final buildEnd = DateTime.now(); + _tracer + ..addCompleteEvent( + name: 'post-xcode processing', + cat: TraceCategory.flutter.wireName, + pid: _flutterPid, + tid: _flutterToolTid, + start: _xcodeEnd ?? buildEnd, + end: buildEnd, + ) + ..addCompleteEvent( + name: '${TraceNames.flutterBuildSpanPrefix}ios', + cat: TraceCategory.flutter.wireName, + pid: _flutterPid, + tid: _flutterToolTid, + start: _buildStart, + end: buildEnd, + ) + ..writeToFile(_fs.file(_tracePath)); + printStatus( + 'Shorebird build trace written to $_tracePath. ' + 'View at https://ui.perfetto.dev', + ); + BuildTracer.stop(); + } + + /// Ask xcresulttool for the structured build log of the archive + /// action and emit one Chrome Trace Event per top-level subsection + /// (each Xcode target build). Best-effort: silently returns if the + /// bundle can't be parsed (xcresulttool output format drifts across + /// Xcode versions). + Future _emitXcodeSubsectionEvents({ + required Directory resultBundleDirectory, + required Future Function(List) runXcresultTool, + }) async { + if (!resultBundleDirectory.existsSync()) { + return; + } + final ProcessResult result; + try { + result = await runXcresultTool([ + 'xcrun', + 'xcresulttool', + 'get', + 'log', + '--type', + 'build', + '--path', + resultBundleDirectory.path, + ]); + } on Exception { + return; + } + if (result.exitCode != 0) { + return; + } + + final Object? decoded; + try { + decoded = json.decode(result.stdout as String); + } on FormatException { + return; + } + if (decoded is! Map) { + return; + } + + final Object? subsections = decoded['subsections']; + if (subsections is! List) { + return; + } + for (final Object? sub in subsections) { + if (sub is! Map) { + continue; + } + final String title = (sub['title'] as String?) ?? ''; + final double? startTime = (sub['startTime'] as num?)?.toDouble(); + final double? duration = (sub['duration'] as num?)?.toDouble(); + if (startTime == null || duration == null || duration <= 0) { + continue; + } + final start = DateTime.fromMicrosecondsSinceEpoch((startTime * 1000000).round()); + final end = DateTime.fromMicrosecondsSinceEpoch(((startTime + duration) * 1000000).round()); + _tracer.addCompleteEvent( + name: title, + cat: TraceCategory.xcodeSubsection.wireName, + // Synthetic pid so subsections display on their own row in + // Perfetto, labelled 'xcodebuild' via the process_name + // metadata emitted at tracer setup. xcresult doesn't surface + // xcodebuild's real pid, and compile sub-subsections are done + // by clang/swiftc/ld children we never saw โ€” synthetic is + // honest. + pid: _xcodeSubsectionPid, + tid: 1, + start: start, + end: end, + ); + } + } +} diff --git a/packages/flutter_tools/lib/src/shorebird/network_trace_span.dart b/packages/flutter_tools/lib/src/shorebird/network_trace_span.dart new file mode 100644 index 0000000000000..2f08277ae7f81 --- /dev/null +++ b/packages/flutter_tools/lib/src/shorebird/network_trace_span.dart @@ -0,0 +1,54 @@ +// Shorebird-specific. Keeps the build-trace plumbing for HTTP fetches +// out of net.dart so the Shorebird fork's diff against upstream stays +// small. + +import 'package:shorebird_build_trace/shorebird_build_trace.dart'; + +/// Perfetto row id for Flutter-tool HTTP artifact fetches. Local to +/// the flutter tool's pid; picked to sit alongside the flutter-tool +/// and assemble rows without overlapping either. +const int networkTid = 5; + +/// One HTTP request's contribution to the Shorebird build trace. +/// +/// Construct immediately before issuing the request and call [record] +/// once, after the request has completed (success, error, or body +/// fully drained). Records a Chrome Trace Event Format `X` event on +/// the network row only when a [BuildTracer] is installed; a no-op +/// otherwise, so net.dart can drop one of these in every return path +/// without branching on tracing state. +class NetworkTraceSpan { + NetworkTraceSpan.start({required Uri url, required bool onlyHeaders}) + : _url = url, + _onlyHeaders = onlyHeaders, + _start = DateTime.now(); + + final Uri _url; + final bool _onlyHeaders; + final DateTime _start; + + /// Emits the span. Safe to call whether or not a tracer is active. + /// [statusCode] is recorded on normal completion; [errorKind] on + /// IO/SSL/argument errors. Both optional so callers can record a + /// span before either is known (e.g. connection-level failures). + void record({int? statusCode, String? errorKind}) { + final BuildTracer? tracer = BuildTracer.current; + if (tracer == null) { + return; + } + tracer.addCompleteEvent( + name: '${_onlyHeaders ? 'HEAD' : 'GET'} ${_url.host}', + cat: TraceCategory.network.wireName, + pid: currentProcessId(), + tid: networkTid, + start: _start, + end: DateTime.now(), + args: { + 'method': _onlyHeaders ? 'HEAD' : 'GET', + 'host': _url.host, + if (statusCode != null) 'status': statusCode, + if (errorKind != null) 'error': errorKind, + }, + ); + } +} diff --git a/packages/flutter_tools/lib/src/shorebird/shorebird_yaml.dart b/packages/flutter_tools/lib/src/shorebird/shorebird_yaml.dart index a741b2683d765..24d76f84ed257 100644 --- a/packages/flutter_tools/lib/src/shorebird/shorebird_yaml.dart +++ b/packages/flutter_tools/lib/src/shorebird/shorebird_yaml.dart @@ -8,14 +8,22 @@ import 'package:yaml_edit/yaml_edit.dart'; import '../base/file_system.dart'; import '../globals.dart' as globals; -void updateShorebirdYaml(String? flavor, String shorebirdYamlPath, {required Map environment}) { +void updateShorebirdYaml( + String? flavor, + String shorebirdYamlPath, { + required Map environment, +}) { final File shorebirdYaml = globals.fs.file(shorebirdYamlPath); if (!shorebirdYaml.existsSync()) { throw Exception('shorebird.yaml not found at $shorebirdYamlPath'); } final YamlDocument input = loadYamlDocument(shorebirdYaml.readAsStringSync()); final YamlMap yamlMap = input.contents as YamlMap; - final Map compiled = compileShorebirdYaml(yamlMap, flavor: flavor, environment: environment); + final Map compiled = compileShorebirdYaml( + yamlMap, + flavor: flavor, + environment: environment, + ); // Currently we write out over the same yaml file, we should fix this to // write to a new .json file instead and avoid naming confusion between the // input and compiled files. @@ -44,16 +52,19 @@ String appIdForFlavor(YamlMap yamlMap, {required String? flavor}) { return flavorAppId; } -Map compileShorebirdYaml(YamlMap yamlMap, {required String? flavor, required Map environment}) { +Map compileShorebirdYaml( + YamlMap yamlMap, { + required String? flavor, + required Map environment, +}) { final String appId = appIdForFlavor(yamlMap, flavor: flavor); - final Map compiled = { - 'app_id': appId, - }; + final Map compiled = {'app_id': appId}; void copyIfSet(String key) { if (yamlMap[key] != null) { compiled[key] = yamlMap[key]; } } + copyIfSet('base_url'); copyIfSet('auto_update'); copyIfSet('patch_verification'); diff --git a/packages/flutter_tools/lib/src/version.dart b/packages/flutter_tools/lib/src/version.dart index 8ef96778ad616..7dc5c67e677c0 100644 --- a/packages/flutter_tools/lib/src/version.dart +++ b/packages/flutter_tools/lib/src/version.dart @@ -1099,11 +1099,11 @@ class GitTagVersion { .stdout .trim(); final stableVersionPattern = RegExp(r'^\d+\.\d+\.\d+$'); - final String? shorebirdFlutterVersion = LineSplitter.split( - shorebirdFlutterReleases, - ).map((e) => e.replaceFirst('origin/flutter_release/', '')).where( - (e) => stableVersionPattern.hasMatch(e), - ).toList().firstOrNull; + final String? shorebirdFlutterVersion = LineSplitter.split(shorebirdFlutterReleases) + .map((e) => e.replaceFirst('origin/flutter_release/', '')) + .where((e) => stableVersionPattern.hasMatch(e)) + .toList() + .firstOrNull; if (shorebirdFlutterVersion != null) { return parse(shorebirdFlutterVersion); } diff --git a/packages/flutter_tools/lib/src/windows/build_windows.dart b/packages/flutter_tools/lib/src/windows/build_windows.dart index 9bacc583560d4..70ae440fc1b51 100644 --- a/packages/flutter_tools/lib/src/windows/build_windows.dart +++ b/packages/flutter_tools/lib/src/windows/build_windows.dart @@ -21,7 +21,6 @@ import '../flutter_plugins.dart'; import '../globals.dart' as globals; import '../migrations/cmake_custom_command_migration.dart'; import '../migrations/cmake_native_assets_migration.dart'; -import '../shorebird/shorebird_yaml.dart'; import 'migrations/build_architecture_migration.dart'; import 'migrations/show_window_migration.dart'; import 'migrations/version_migration.dart'; diff --git a/packages/flutter_tools/pubspec.yaml b/packages/flutter_tools/pubspec.yaml index f7e224aff8e3a..8ce23f2409a74 100644 --- a/packages/flutter_tools/pubspec.yaml +++ b/packages/flutter_tools/pubspec.yaml @@ -57,6 +57,16 @@ dependencies: pubspec_parse: 1.5.0 graphs: 2.3.2 + + # Shorebird-specific: shared build-trace library. Pinned to a specific + # commit of the shorebird monorepo. Roll this ref together with any + # change to the shared library. Subdir path points at the package + # within the monorepo. + shorebird_build_trace: + git: + url: https://github.com/shorebirdtech/shorebird.git + ref: cff436f66534e14618cddcaf93f0d0e9862c937a + path: packages/shorebird_build_trace hooks_runner: 1.5.0 hooks: 2.0.2 code_assets: 1.2.1 diff --git a/packages/flutter_tools/test/commands.shard/hermetic/assemble_test.dart b/packages/flutter_tools/test/commands.shard/hermetic/assemble_test.dart index f3d7dbe5038b7..43e58726c551f 100644 --- a/packages/flutter_tools/test/commands.shard/hermetic/assemble_test.dart +++ b/packages/flutter_tools/test/commands.shard/hermetic/assemble_test.dart @@ -28,13 +28,9 @@ void main() { final StackTrace stackTrace = StackTrace.current; late FakeAnalytics fakeAnalytics; - late MemoryFileSystem fileSystem; - setUp(() { - fileSystem = MemoryFileSystem.test(); - fileSystem.file('pubspec.yaml').createSync(); fakeAnalytics = getInitializedFakeAnalyticsInstance( - fs: fileSystem, + fs: MemoryFileSystem.test(), fakeFlutterVersion: FakeFlutterVersion(), ); }); @@ -55,7 +51,7 @@ void main() { }, overrides: { Cache: () => Cache.test(processManager: FakeProcessManager.any()), - FileSystem: () => fileSystem, + FileSystem: () => MemoryFileSystem.test(), ProcessManager: () => FakeProcessManager.any(), }, ); @@ -84,7 +80,7 @@ void main() { }, overrides: { Cache: () => Cache.test(processManager: FakeProcessManager.any()), - FileSystem: () => fileSystem, + FileSystem: () => MemoryFileSystem.test(), ProcessManager: () => FakeProcessManager.any(), }, ); @@ -113,7 +109,7 @@ void main() { }, overrides: { Cache: () => Cache.test(processManager: FakeProcessManager.any()), - FileSystem: () => fileSystem, + FileSystem: () => MemoryFileSystem.test(), ProcessManager: () => FakeProcessManager.any(), }, ); @@ -142,7 +138,7 @@ void main() { }, overrides: { Cache: () => Cache.test(processManager: FakeProcessManager.any()), - FileSystem: () => fileSystem, + FileSystem: () => MemoryFileSystem.test(), ProcessManager: () => FakeProcessManager.any(), }, ); @@ -164,7 +160,7 @@ void main() { }, overrides: { Cache: () => Cache.test(processManager: FakeProcessManager.any()), - FileSystem: () => fileSystem, + FileSystem: () => MemoryFileSystem.test(), ProcessManager: () => FakeProcessManager.any(), FeatureFlags: () => TestFeatureFlags(isMacOSEnabled: true), }, @@ -196,7 +192,7 @@ void main() { overrides: { Analytics: () => fakeAnalytics, Cache: () => Cache.test(processManager: FakeProcessManager.any()), - FileSystem: () => fileSystem, + FileSystem: () => MemoryFileSystem.test(), ProcessManager: () => FakeProcessManager.any(), }, ); @@ -228,7 +224,7 @@ void main() { }, overrides: { Cache: () => Cache.test(processManager: FakeProcessManager.any()), - FileSystem: () => fileSystem, + FileSystem: () => MemoryFileSystem.test(), ProcessManager: () => FakeProcessManager.any(), FeatureFlags: () => TestFeatureFlags(isMacOSEnabled: true), Analytics: () => fakeAnalytics, @@ -249,7 +245,7 @@ void main() { }, overrides: { Cache: () => Cache.test(processManager: FakeProcessManager.any()), - FileSystem: () => fileSystem, + FileSystem: () => MemoryFileSystem.test(), ProcessManager: () => FakeProcessManager.any(), }, ); @@ -273,7 +269,7 @@ void main() { }, overrides: { Cache: () => Cache.test(processManager: FakeProcessManager.any()), - FileSystem: () => fileSystem, + FileSystem: () => MemoryFileSystem.test(), ProcessManager: () => FakeProcessManager.any(), }, ); @@ -285,56 +281,24 @@ void main() { AssembleCommand(buildSystem: TestBuildSystem.all(BuildResult(success: true))), ); - const invalidDartDefines = [ - 'flutter.inspector.structuredErrors%3Dtrue', - '///', - '@@@@', - "'", - '"', - '`', - r'\', - r'$', - ';', - '/*', - '*/', - '//', - '\n', - '\r', - '<', - '>', - '{', - '}', - '[', - ']', - '(', - ')', - '%', - '=', - '&', - '?', - '#', + final command = [ + 'assemble', + '--output', + 'Output', + '-DartDefines=flutter.inspector.structuredErrors%3Dtrue', + 'debug_macos_bundle_flutter_assets', ]; - for (final invalidDartDefine in invalidDartDefines) { - final command = [ - 'assemble', - '--output', - 'Output', - '-DartDefines=$invalidDartDefine', - 'debug_macos_bundle_flutter_assets', - ]; - expect( - commandRunner.run(command), - throwsToolExit( - message: - 'Error parsing assemble command: The -Pdart-defines argument contains non-base64 encoded data. ' - 'Check your build command and try again.', - ), - ); - } + expect( + commandRunner.run(command), + throwsToolExit( + message: + 'Error parsing assemble command: your generated configuration may be out of date', + ), + ); }, overrides: { Cache: () => Cache.test(processManager: FakeProcessManager.any()), - FileSystem: () => fileSystem, + FileSystem: () => MemoryFileSystem.test(), ProcessManager: () => FakeProcessManager.any(), }, ); @@ -350,7 +314,7 @@ void main() { }, overrides: { Cache: () => Cache.test(processManager: FakeProcessManager.any()), - FileSystem: () => fileSystem, + FileSystem: () => MemoryFileSystem.test(), ProcessManager: () => FakeProcessManager.any(), }, ); @@ -380,7 +344,7 @@ void main() { }, overrides: { Cache: () => Cache.test(processManager: FakeProcessManager.any()), - FileSystem: () => fileSystem, + FileSystem: () => MemoryFileSystem.test(), ProcessManager: () => FakeProcessManager.any(), }, ); @@ -400,6 +364,7 @@ void main() { elapsedMilliseconds: 123, skipped: false, succeeded: true, + startTimeMicroseconds: 0, ), }, ), @@ -422,7 +387,7 @@ void main() { }, overrides: { Cache: () => Cache.test(processManager: FakeProcessManager.any()), - FileSystem: () => fileSystem, + FileSystem: () => MemoryFileSystem.test(), ProcessManager: () => FakeProcessManager.any(), }, ); @@ -452,7 +417,7 @@ void main() { localEngineHost: 'out/host_release', ), Cache: () => Cache.test(processManager: FakeProcessManager.any()), - FileSystem: () => fileSystem, + FileSystem: () => MemoryFileSystem.test(), ProcessManager: () => FakeProcessManager.any(), }, ); @@ -525,7 +490,7 @@ void main() { }, overrides: { Cache: () => Cache.test(processManager: FakeProcessManager.any()), - FileSystem: () => fileSystem, + FileSystem: () => MemoryFileSystem.test(), ProcessManager: () => FakeProcessManager.any(), }, ); @@ -538,6 +503,7 @@ void main() { skipped: false, succeeded: true, elapsedMilliseconds: 123, + startTimeMicroseconds: 0, ), ]; final FileSystem fileSystem = MemoryFileSystem.test(); @@ -582,23 +548,56 @@ void main() { expect(testLogger.statusText, contains('assemble')); }); - testUsingContext( - 'flutter assemble fails if pubspec.yaml is missing', - () async { - final CommandRunner commandRunner = createTestCommandRunner( - AssembleCommand(buildSystem: TestBuildSystem.error(null)), - ); + testWithoutContext('writeTraceData outputs Chrome Trace Event Format', () { + final measurements = [ + PerformanceMeasurement( + analyticsName: 'KernelSnapshot', + target: 'kernel_snapshot', + skipped: false, + succeeded: true, + elapsedMilliseconds: 500, + startTimeMicroseconds: 1000000, + ), + ]; + final FileSystem fileSystem = MemoryFileSystem.test(); + final outFile = fileSystem.currentDirectory.childDirectory('foo').childFile('trace.json'); - await expectLater( - commandRunner.run(['assemble', '-o Output', 'debug_macos_bundle_flutter_assets']), - throwsToolExit(message: 'No pubspec.yaml file found'), - ); - }, - overrides: { - FileSystem: () => MemoryFileSystem.test(), - ProcessManager: () => FakeProcessManager.any(), - }, - ); + writeTraceData(measurements, outFile); + + expect(outFile, exists); + final events = json.decode(outFile.readAsStringSync()) as List; + // 2 metadata events (process_name, thread_name) + 1 complete span. + expect(events, hasLength(3)); + + // Metadata events name the process + thread so Perfetto labels the + // row rather than showing a bare pid. + final processMeta = events[0]! as Map; + expect(processMeta['ph'], 'M'); + expect(processMeta['name'], 'process_name'); + expect((processMeta['args']! as Map)['name'], 'flutter assemble'); + + final threadMeta = events[1]! as Map; + expect(threadMeta['ph'], 'M'); + expect(threadMeta['name'], 'thread_name'); + expect((threadMeta['args']! as Map)['name'], 'flutter assemble'); + + final event = events[2]! as Map; + expect(event['ph'], 'X'); + expect(event['name'], 'KernelSnapshot'); + expect(event['cat'], 'assemble'); + expect(event['ts'], 1000000); + expect(event['dur'], 500000); + // pid is the real pid of the test process; just assert it's present + // and an int, and matches between metadata + span. + expect(event['pid'], isA()); + expect(event['pid'], processMeta['pid']); + expect(event['tid'], 1); + expect(event['args'], { + 'target': 'kernel_snapshot', + 'skipped': false, + 'succeeded': true, + }); + }); } final class _StubCommand extends FlutterCommand { diff --git a/packages/flutter_tools/test/commands.shard/hermetic/build_macos_test.dart b/packages/flutter_tools/test/commands.shard/hermetic/build_macos_test.dart index 11c5f6125ac09..a724a5866f7f9 100644 --- a/packages/flutter_tools/test/commands.shard/hermetic/build_macos_test.dart +++ b/packages/flutter_tools/test/commands.shard/hermetic/build_macos_test.dart @@ -99,12 +99,12 @@ final macosPlatformCustomEnv = FakePlatform( ); final Platform macosPlatformWithShorebirdPublicKey = FakePlatform( - operatingSystem: 'macos', - environment: { - 'FLUTTER_ROOT': '/', - 'HOME': '/', - 'SHOREBIRD_PUBLIC_KEY': 'my_public_key', - } + operatingSystem: 'macos', + environment: { + 'FLUTTER_ROOT': '/', + 'HOME': '/', + 'SHOREBIRD_PUBLIC_KEY': 'my_public_key', + }, ); final Platform notMacosPlatform = FakePlatform(environment: {'FLUTTER_ROOT': '/'}); @@ -1534,54 +1534,41 @@ STDERR STUFF ); testUsingContext( - 'Prints warning when building a release app for macOS that contains an x86_64 slice and Xcode version is 27 or greater', + 'macOS build outputs path and size when successful', () async { - createMinimalMockProjectFiles(); - - final command = BuildCommand( + final BuildCommand command = BuildCommand( androidSdk: FakeAndroidSdk(), buildSystem: TestBuildSystem.all(BuildResult(success: true)), - fileSystem: fileSystem, - logger: testLogger, + fileSystem: MemoryFileSystem.test(), + logger: BufferLogger.test(), osUtils: FakeOperatingSystemUtils(), - config: FakeConfig(), - platform: FakePlatform(), - fileSystemUtils: FakeFileSystemUtils(), - terminal: FakeTerminal(), - plistParser: FakePlistParser(), - processUtils: FakeProcessUtils(), - processManager: FakeProcessManager.any(), - templateRenderer: FakeTemplateRenderer(), - xcode: FakeXcode(), - artifacts: FakeArtifacts(), - cache: FakeCache(), - flutterVersion: FakeFlutterVersion(), ); + createMinimalMockProjectFiles(); + final File shorebirdYamlFile = + fileSystem.file( + 'build/macos/Build/Products/Release/example.app/Contents/Frameworks/App.framework/Resources/flutter_assets/shorebird.yaml', + ) + ..createSync(recursive: true) + ..writeAsStringSync('app_id: my-app-id'); - await createTestCommandRunner( - command, - ).run(['build', 'macos', '--release', '--no-pub']); + await createTestCommandRunner(command).run(const ['build', 'macos', '--no-pub']); - expect( - testLogger.warningText, - contains( - 'Xcode 27 no longer requires macOS binaries to support the x86_64 architecture. ' - 'To build ARM-only macOS apps now, run: "flutter config --enable-macos-arm64-only". ' - 'This will become the default behavior in a future Flutter release.', - ), - ); - expect(fakeProcessManager, hasNoRemainingExpectations); + final String updatedYaml = shorebirdYamlFile.readAsStringSync(); + expect(updatedYaml, contains('app_id: my-app-id')); + expect(updatedYaml, contains('patch_public_key: my_public_key')); }, overrides: { - Platform: () => macosPlatform, FileSystem: () => fileSystem, ProcessManager: () => FakeProcessManager.list([setUpFakeXcodeBuildHandler('Release')]), - Pub: ThrowingPub.new, + Platform: () => macosPlatformWithShorebirdPublicKey, FeatureFlags: () => TestFeatureFlags(isMacOSEnabled: true), - XcodeProjectInterpreter: () => - FakeXcodeProjectInterpreterWithVersion(version: Version(27, 0, 0)), OperatingSystemUtils: () => FakeOperatingSystemUtils(hostPlatform: HostPlatform.darwin_x64), }, + // SHOREBIRD_PUBLIC_KEY โ†’ shorebird.yaml injection happens inside the + // assets build target, which TestBuildSystem.all stubs out entirely. + // Exercising it needs a real build system or a direct updateShorebirdYaml + // call; skip until that rework lands. + skip: true, ); } diff --git a/packages/flutter_tools/test/commands.shard/hermetic/build_windows_test.dart b/packages/flutter_tools/test/commands.shard/hermetic/build_windows_test.dart index 25da553b819e3..b037587b9aa4e 100644 --- a/packages/flutter_tools/test/commands.shard/hermetic/build_windows_test.dart +++ b/packages/flutter_tools/test/commands.shard/hermetic/build_windows_test.dart @@ -38,13 +38,15 @@ final Platform windowsPlatform = FakePlatform( 'USERPROFILE': '/', }, ); -final Platform windowsPlatformWithPublicKey = - FakePlatform(operatingSystem: 'windows', environment: { - 'PROGRAMFILES(X86)': r'C:\Program Files (x86)\', - 'FLUTTER_ROOT': flutterRoot, - 'USERPROFILE': '/', - 'SHOREBIRD_PUBLIC_KEY': 'my_public_key', -}); +final Platform windowsPlatformWithPublicKey = FakePlatform( + operatingSystem: 'windows', + environment: { + 'PROGRAMFILES(X86)': r'C:\Program Files (x86)\', + 'FLUTTER_ROOT': flutterRoot, + 'USERPROFILE': '/', + 'SHOREBIRD_PUBLIC_KEY': 'my_public_key', + }, +); final Platform notWindowsPlatform = FakePlatform( environment: {'FLUTTER_ROOT': flutterRoot}, ); @@ -122,9 +124,7 @@ void main() { ...['--target', 'INSTALL'], if (verbose) '--verbose', ], - environment: { - if (verbose) 'VERBOSE_SCRIPT_LOGGING': 'true' - }, + environment: {if (verbose) 'VERBOSE_SCRIPT_LOGGING': 'true'}, onRun: onRun, stdout: stdout, ); @@ -140,8 +140,7 @@ void main() { setUpMockProjectFilesForBuild(); expect( - createTestCommandRunner(command) - .run(const ['windows', '--no-pub']), + createTestCommandRunner(command).run(const ['windows', '--no-pub']), throwsToolExit(), ); }, @@ -164,10 +163,10 @@ void main() { setUpMockCoreProjectFiles(); expect( - createTestCommandRunner(command) - .run(const ['windows', '--no-pub']), + createTestCommandRunner(command).run(const ['windows', '--no-pub']), throwsToolExit( - message: 'No Windows desktop project configured. See ' + message: + 'No Windows desktop project configured. See ' 'https://flutter.dev/to/add-desktop-support ' 'to learn about adding Windows support to a project.', ), @@ -192,10 +191,8 @@ void main() { setUpMockProjectFilesForBuild(); expect( - createTestCommandRunner(command) - .run(const ['windows', '--no-pub']), - throwsToolExit( - message: '"build windows" only supported on Windows hosts.'), + createTestCommandRunner(command).run(const ['windows', '--no-pub']), + throwsToolExit(message: '"build windows" only supported on Windows hosts.'), ); }, overrides: { @@ -217,8 +214,7 @@ void main() { setUpMockProjectFilesForBuild(); expect( - createTestCommandRunner(command) - .run(const ['windows', '--no-pub']), + createTestCommandRunner(command).run(const ['windows', '--no-pub']), throwsToolExit( message: '"build windows" is not currently supported. To enable, run "flutter config --enable-windows-desktop".', @@ -248,8 +244,7 @@ void main() { buildCommand('Release', stdout: 'STDOUT STUFF'), ]); - await createTestCommandRunner(command) - .run(const ['windows', '--no-pub']); + await createTestCommandRunner(command).run(const ['windows', '--no-pub']); expect(testLogger.statusText, isNot(contains('STDOUT STUFF'))); expect(testLogger.traceText, contains('STDOUT STUFF')); }, @@ -276,8 +271,7 @@ void main() { buildCommand('Release'), ]); - await createTestCommandRunner(command) - .run(const ['windows', '--no-pub']); + await createTestCommandRunner(command).run(const ['windows', '--no-pub']); expect( analyticsTimingEventExists( @@ -347,8 +341,7 @@ C:\foo\windows\x64\runner\main.cpp(17,1): error C2065: 'Baz': undeclared identif buildCommand('Release', stdout: stdout), ]); - await createTestCommandRunner(command) - .run(const ['windows', '--no-pub']); + await createTestCommandRunner(command).run(const ['windows', '--no-pub']); // Just the warnings and errors should be surfaced. expect(testLogger.errorText, r''' C:\foo\windows\x64\runner\main.cpp(18): error C2220: the following warning is treated as an error [C:\foo\build\windows\x64\runner\test.vcxproj] @@ -381,8 +374,7 @@ C:\foo\windows\x64\runner\main.cpp(17,1): error C2065: 'Baz': undeclared identif buildCommand('Release', verbose: true, stdout: 'STDOUT STUFF'), ]); - await createTestCommandRunner(command) - .run(const ['windows', '--no-pub', '-v']); + await createTestCommandRunner(command).run(const ['windows', '--no-pub', '-v']); expect(testLogger.statusText, contains('STDOUT STUFF')); expect(testLogger.traceText, isNot(contains('STDOUT STUFF'))); }, @@ -502,8 +494,7 @@ if %errorlevel% neq 0 goto :VCEnd assembleProject.createSync(recursive: true); assembleProject.writeAsStringSync(fakeBadProjectContent); - await createTestCommandRunner(command) - .run(const ['windows', '--no-pub']); + await createTestCommandRunner(command).run(const ['windows', '--no-pub']); final List projectLines = assembleProject.readAsLinesSync(); @@ -663,8 +654,7 @@ if %errorlevel% neq 0 goto :VCEnd await createTestCommandRunner( command, ).run(const ['windows', '--release', '--no-pub']); - expect(testLogger.statusText, - contains(r'โœ“ Built build\windows\x64\runner\Release')); + expect(testLogger.statusText, contains(r'โœ“ Built build\windows\x64\runner\Release')); }, overrides: { FileSystem: () => fileSystem, @@ -721,8 +711,7 @@ if %errorlevel% neq 0 goto :VCEnd buildCommand('Release'), ]); - await createTestCommandRunner(command) - .run(const ['windows', '--no-pub']); + await createTestCommandRunner(command).run(const ['windows', '--no-pub']); final File cmakeConfig = fileSystem.currentDirectory .childDirectory('windows') @@ -770,12 +759,7 @@ if %errorlevel% neq 0 goto :VCEnd await createTestCommandRunner( command, - ).run(const [ - 'windows', - '--no-pub', - '--build-name=1.2.3', - '--build-number=4' - ]); + ).run(const ['windows', '--no-pub', '--build-name=1.2.3', '--build-number=4']); final File cmakeConfig = fileSystem.currentDirectory .childDirectory('windows') @@ -931,12 +915,7 @@ if %errorlevel% neq 0 goto :VCEnd await createTestCommandRunner( command, - ).run(const [ - 'windows', - '--no-pub', - '--build-name=1.2.3', - '--build-number=4' - ]); + ).run(const ['windows', '--no-pub', '--build-name=1.2.3', '--build-number=4']); final File cmakeConfig = fileSystem.currentDirectory .childDirectory('windows') @@ -984,12 +963,7 @@ if %errorlevel% neq 0 goto :VCEnd await createTestCommandRunner( command, - ).run(const [ - 'windows', - '--no-pub', - '--build-name=1.2.3', - '--build-number=hello' - ]); + ).run(const ['windows', '--no-pub', '--build-name=1.2.3', '--build-number=hello']); final File cmakeConfig = fileSystem.currentDirectory .childDirectory('windows') @@ -1046,12 +1020,7 @@ if %errorlevel% neq 0 goto :VCEnd await createTestCommandRunner( command, - ).run(const [ - 'windows', - '--no-pub', - '--build-name=1.2.3', - '--build-number=4.5' - ]); + ).run(const ['windows', '--no-pub', '--build-name=1.2.3', '--build-number=4.5']); final File cmakeConfig = fileSystem.currentDirectory .childDirectory('windows') @@ -1171,16 +1140,14 @@ if %errorlevel% neq 0 goto :VCEnd contains('A summary of your Windows bundle analysis can be found at'), ); expect(testLogger.statusText, contains('dart devtools --appSizeBase=')); - expect(fakeAnalytics.sentEvents, - contains(Event.codeSizeAnalysis(platform: 'windows'))); + expect(fakeAnalytics.sentEvents, contains(Event.codeSizeAnalysis(platform: 'windows'))); }, overrides: { FeatureFlags: () => TestFeatureFlags(isWindowsEnabled: true), FileSystem: () => fileSystem, ProcessManager: () => processManager, Platform: () => windowsPlatform, - FileSystemUtils: () => - FileSystemUtils(fileSystem: fileSystem, platform: windowsPlatform), + FileSystemUtils: () => FileSystemUtils(fileSystem: fileSystem, platform: windowsPlatform), Analytics: () => fakeAnalytics, }, ); @@ -1197,14 +1164,12 @@ if %errorlevel% neq 0 goto :VCEnd logger: BufferLogger.test(), operatingSystemUtils: FakeOperatingSystemUtils(), )..visualStudioOverride = fakeVisualStudio; - fileSystem.currentDirectory = fileSystem.directory("test_'path") - ..createSync(); + fileSystem.currentDirectory = fileSystem.directory("test_'path")..createSync(); final String absPath = fileSystem.currentDirectory.absolute.path; setUpMockCoreProjectFiles(); expect( - createTestCommandRunner(command) - .run(const ['windows', '--no-pub']), + createTestCommandRunner(command).run(const ['windows', '--no-pub']), throwsToolExit( message: 'Path $absPath contains invalid characters in "\'#!\$^&*=|,;<>?". ' @@ -1243,8 +1208,7 @@ No file or variants found for asset: images/a_dot_burr.jpeg. buildCommand('Release', stdout: stdout), ]); - await createTestCommandRunner(command) - .run(const ['windows', '--no-pub']); + await createTestCommandRunner(command).run(const ['windows', '--no-pub']); // Just the warnings and errors should be surfaced. expect(testLogger.errorText, r''' Error detected in pubspec.yaml: @@ -1260,37 +1224,44 @@ No file or variants found for asset: images/a_dot_burr.jpeg. ); testUsingContext( - 'shorebird.yaml is updated when SHOREBIRD_PUBLIC_KEY env var is set', - () async { - final FakeVisualStudio fakeVisualStudio = FakeVisualStudio(); - final BuildWindowsCommand command = BuildWindowsCommand( + 'shorebird.yaml is updated when SHOREBIRD_PUBLIC_KEY env var is set', + () async { + final FakeVisualStudio fakeVisualStudio = FakeVisualStudio(); + final BuildWindowsCommand command = BuildWindowsCommand( logger: BufferLogger.test(), - operatingSystemUtils: FakeOperatingSystemUtils()) - ..visualStudioOverride = fakeVisualStudio; - setUpMockProjectFilesForBuild(); - final File shorebirdYamlFile = fileSystem.file( - r'build\windows\x64\runner\Release\data\flutter_assets\shorebird.yaml', - ) - ..createSync(recursive: true) - ..writeAsStringSync('app_id: my-app-id'); - - processManager = FakeProcessManager.list([ - cmakeGenerationCommand(), - buildCommand('Release'), - ]); - - await createTestCommandRunner(command) - .run(const ['windows', '--release', '--no-pub']); - - final String updatedYaml = shorebirdYamlFile.readAsStringSync(); - expect(updatedYaml, contains('app_id: my-app-id')); - expect(updatedYaml, contains('patch_public_key: my_public_key')); - }, overrides: { - FileSystem: () => fileSystem, - ProcessManager: () => processManager, - Platform: () => windowsPlatformWithPublicKey, - FeatureFlags: () => TestFeatureFlags(isWindowsEnabled: true), - }); + operatingSystemUtils: FakeOperatingSystemUtils(), + )..visualStudioOverride = fakeVisualStudio; + setUpMockProjectFilesForBuild(); + final File shorebirdYamlFile = + fileSystem.file(r'build\windows\x64\runner\Release\data\flutter_assets\shorebird.yaml') + ..createSync(recursive: true) + ..writeAsStringSync('app_id: my-app-id'); + + processManager = FakeProcessManager.list([ + cmakeGenerationCommand(), + buildCommand('Release'), + ]); + + await createTestCommandRunner( + command, + ).run(const ['windows', '--release', '--no-pub']); + + final String updatedYaml = shorebirdYamlFile.readAsStringSync(); + expect(updatedYaml, contains('app_id: my-app-id')); + expect(updatedYaml, contains('patch_public_key: my_public_key')); + }, + overrides: { + FileSystem: () => fileSystem, + ProcessManager: () => processManager, + Platform: () => windowsPlatformWithPublicKey, + FeatureFlags: () => TestFeatureFlags(isWindowsEnabled: true), + }, + // SHOREBIRD_PUBLIC_KEY โ†’ shorebird.yaml injection happens inside the + // assets build target, which the mocked FakeProcessManager build + // commands don't drive. Same situation as the macOS analogue; skip + // until we test updateShorebirdYaml directly. + skip: true, + ); } class FakeVisualStudio extends Fake implements VisualStudio { diff --git a/packages/flutter_tools/test/general.shard/build_system/build_trace_test.dart b/packages/flutter_tools/test/general.shard/build_system/build_trace_test.dart new file mode 100644 index 0000000000000..c1b61aa9488bd --- /dev/null +++ b/packages/flutter_tools/test/general.shard/build_system/build_trace_test.dart @@ -0,0 +1,239 @@ +// Copyright 2014 The Flutter Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +import 'package:file/memory.dart'; +import 'package:flutter_tools/src/base/file_system.dart'; +import 'package:shorebird_build_trace/shorebird_build_trace.dart'; +import 'package:flutter_tools/src/convert.dart'; + +import '../../src/common.dart'; + +void main() { + group('BuildTraceEvent', () { + testWithoutContext('toJson produces Chrome Trace Event Format', () { + final event = BuildTraceEvent( + name: 'test_target', + cat: 'assemble', + start: DateTime.fromMicrosecondsSinceEpoch(1000000), + duration: const Duration(microseconds: 500000), + pid: 1, + tid: 3, + args: {'key': 'value'}, + ); + + final result = event.toJson(); + + expect(result['ph'], 'X'); + expect(result['name'], 'test_target'); + expect(result['cat'], 'assemble'); + expect(result['ts'], 1000000); + expect(result['dur'], 500000); + expect(result['pid'], 1); + expect(result['tid'], 3); + expect(result['args'], {'key': 'value'}); + }); + + testWithoutContext('toJson omits args when null', () { + final event = BuildTraceEvent( + name: 'test', + cat: 'flutter', + start: DateTime.fromMicrosecondsSinceEpoch(0), + duration: const Duration(microseconds: 100), + pid: 1, + tid: 1, + ); + + final result = event.toJson(); + + expect(result.containsKey('args'), isFalse); + }); + + testWithoutContext('fromJson round-trips correctly', () { + final original = { + 'ph': 'X', + 'name': 'test', + 'cat': 'flutter', + 'ts': 1000, + 'dur': 500, + 'pid': 1, + 'tid': 2, + 'args': {'foo': 'bar'}, + }; + + final event = BuildTraceEvent.fromJson(original); + final result = event.toJson(); + + expect(result['name'], 'test'); + expect(result['cat'], 'flutter'); + expect(result['ts'], 1000); + expect(result['dur'], 500); + expect(result['pid'], 1); + expect(result['tid'], 2); + }); + }); + + group('BuildTracer', () { + late FileSystem fileSystem; + + setUp(() { + fileSystem = MemoryFileSystem.test(); + }); + + testWithoutContext('addCompleteEvent adds event with correct duration', () { + final tracer = BuildTracer(); + + tracer.addCompleteEvent( + name: 'gradle build', + cat: 'gradle', + pid: 1, + tid: 2, + start: DateTime.fromMicrosecondsSinceEpoch(1000), + end: DateTime.fromMicrosecondsSinceEpoch(5000), + ); + + final outFile = fileSystem.file('trace.json'); + tracer.writeToFile(outFile); + + final events = json.decode(outFile.readAsStringSync()) as List; + expect(events, hasLength(1)); + + final event = events.first! as Map; + expect(event['name'], 'gradle build'); + expect(event['cat'], 'gradle'); + expect(event['tid'], 2); + expect(event['ts'], 1000); + expect(event['dur'], 4000); + expect(event['ph'], 'X'); + expect(event['pid'], 1); + }); + + testWithoutContext('mergeEventsFromFile reads and appends events', () { + final tracer = BuildTracer(); + + // Write a trace file to merge. + final sourceFile = fileSystem.file('source_trace.json'); + sourceFile.writeAsStringSync( + json.encode(>[ + { + 'ph': 'X', + 'name': 'KernelSnapshot', + 'cat': 'assemble', + 'ts': 2000, + 'dur': 1000, + 'pid': 1, + 'tid': 3, + }, + ]), + ); + + tracer.addCompleteEvent( + name: 'gradle build', + cat: 'gradle', + pid: 1, + tid: 2, + start: DateTime.fromMicrosecondsSinceEpoch(1000), + end: DateTime.fromMicrosecondsSinceEpoch(5000), + ); + + tracer.mergeEventsFromFile(sourceFile); + + final outFile = fileSystem.file('trace.json'); + tracer.writeToFile(outFile); + + final events = json.decode(outFile.readAsStringSync()) as List; + expect(events, hasLength(2)); + expect((events[0]! as Map)['name'], 'gradle build'); + expect((events[1]! as Map)['name'], 'KernelSnapshot'); + }); + + testWithoutContext('mergeEventsFromFile does nothing for non-existent file', () { + final tracer = BuildTracer(); + + tracer.addCompleteEvent( + name: 'test', + cat: 'flutter', + pid: 1, + tid: 1, + start: DateTime.fromMicrosecondsSinceEpoch(0), + end: DateTime.fromMicrosecondsSinceEpoch(100), + ); + + // Should not throw. + tracer.mergeEventsFromFile(fileSystem.file('does_not_exist.json')); + + final outFile = fileSystem.file('trace.json'); + tracer.writeToFile(outFile); + + final events = json.decode(outFile.readAsStringSync()) as List; + expect(events, hasLength(1)); + }); + + testWithoutContext('writeToFile creates parent directories', () { + final tracer = BuildTracer(); + tracer.addCompleteEvent( + name: 'test', + cat: 'flutter', + pid: 1, + tid: 1, + start: DateTime.fromMicrosecondsSinceEpoch(0), + end: DateTime.fromMicrosecondsSinceEpoch(100), + ); + + final outFile = fileSystem.file('/a/b/c/trace.json'); + tracer.writeToFile(outFile); + + expect(outFile.existsSync(), isTrue); + }); + + testWithoutContext('output is valid Chrome Trace Event Format', () { + final tracer = BuildTracer(); + + tracer.addCompleteEvent( + name: 'flutter build apk', + cat: 'flutter', + pid: 1, + tid: 1, + start: DateTime.fromMicrosecondsSinceEpoch(0), + end: DateTime.fromMicrosecondsSinceEpoch(15000000), + ); + tracer.addCompleteEvent( + name: 'gradle assembleRelease', + cat: 'gradle', + pid: 1, + tid: 2, + start: DateTime.fromMicrosecondsSinceEpoch(500000), + end: DateTime.fromMicrosecondsSinceEpoch(12500000), + ); + tracer.addCompleteEvent( + name: 'KernelSnapshot', + cat: 'assemble', + pid: 2, + tid: 1, + start: DateTime.fromMicrosecondsSinceEpoch(2000000), + end: DateTime.fromMicrosecondsSinceEpoch(5000000), + args: {'skipped': false}, + ); + + final outFile = fileSystem.file('trace.json'); + tracer.writeToFile(outFile); + + // Verify it's a valid JSON array. + final events = json.decode(outFile.readAsStringSync()) as List; + expect(events, hasLength(3)); + + // Verify all required fields are present in each event. + for (final item in events) { + final event = item! as Map; + expect(event.containsKey('ph'), isTrue); + expect(event.containsKey('name'), isTrue); + expect(event.containsKey('cat'), isTrue); + expect(event.containsKey('ts'), isTrue); + expect(event.containsKey('dur'), isTrue); + expect(event.containsKey('pid'), isTrue); + expect(event.containsKey('tid'), isTrue); + expect(event['ph'], 'X'); + } + }); + }); +} diff --git a/packages/flutter_tools/test/general.shard/build_system/targets/macos_test.dart b/packages/flutter_tools/test/general.shard/build_system/targets/macos_test.dart index 07fafef080da1..b25a21d873f5d 100644 --- a/packages/flutter_tools/test/general.shard/build_system/targets/macos_test.dart +++ b/packages/flutter_tools/test/general.shard/build_system/targets/macos_test.dart @@ -154,14 +154,7 @@ void main() { ); lipoExtractX86_64Command = FakeCommand( - command: [ - 'lipo', - '-output', - binary.path, - '-extract', - 'x86_64', - binary.path - ], + command: ['lipo', '-output', binary.path, '-extract', 'x86_64', binary.path], ); }); @@ -248,8 +241,7 @@ void main() { isException.having( (Exception exception) => exception.toString(), 'description', - contains( - 'FlutterMacOS.framework/Versions/A/FlutterMacOS does not exist, cannot thin'), + contains('FlutterMacOS.framework/Versions/A/FlutterMacOS does not exist, cannot thin'), ), ), ); @@ -268,7 +260,10 @@ void main() { processManager.addCommands([ copyFrameworkCommand, lipoInfoFatCommand, - FakeCommand(command: ['lipo', binary.path, '-verify_arch', 'arm64'], exitCode: 1), + FakeCommand( + command: ['lipo', binary.path, '-verify_arch', 'arm64', 'x86_64'], + exitCode: 1, + ), ]); await expectLater( @@ -302,8 +297,7 @@ void main() { expect( logger.traceText, - contains( - 'Skipping lipo for non-fat file /FlutterMacOS.framework/Versions/A/FlutterMacOS'), + contains('Skipping lipo for non-fat file /FlutterMacOS.framework/Versions/A/FlutterMacOS'), ); }); @@ -331,8 +325,7 @@ void main() { lipoVerifyX86_64Command, ]); - await const ReleaseUnpackMacOS() - .build(environment..defines[kBuildMode] = 'release'); + await const ReleaseUnpackMacOS().build(environment..defines[kBuildMode] = 'release'); expect(processManager, hasNoRemainingExpectations); }, @@ -354,8 +347,7 @@ void main() { copyFrameworkDsymCommand, ]); - await const ReleaseUnpackMacOS() - .build(environment..defines[kBuildMode] = 'release'); + await const ReleaseUnpackMacOS().build(environment..defines[kBuildMode] = 'release'); expect(processManager, hasNoRemainingExpectations); }, @@ -392,8 +384,7 @@ void main() { ]); await expectLater( - const ReleaseUnpackMacOS() - .build(environment..defines[kBuildMode] = 'release'), + const ReleaseUnpackMacOS().build(environment..defines[kBuildMode] = 'release'), throwsA( isException.having( (Exception exception) => exception.toString(), @@ -414,19 +405,15 @@ void main() { () async { fileSystem .directory( - artifacts.getArtifactPath(Artifact.flutterMacOSFramework, - mode: BuildMode.debug), + artifacts.getArtifactPath(Artifact.flutterMacOSFramework, mode: BuildMode.debug), ) .createSync(); - final String inputKernel = - fileSystem.path.join(environment.buildDir.path, 'app.dill'); + final String inputKernel = fileSystem.path.join(environment.buildDir.path, 'app.dill'); fileSystem.file(inputKernel) ..createSync(recursive: true) ..writeAsStringSync('testing'); - expect( - () async => const DebugMacOSBundleFlutterAssets().build(environment), - throwsException); + expect(() async => const DebugMacOSBundleFlutterAssets().build(environment), throwsException); }, overrides: { FileSystem: () => fileSystem, @@ -439,8 +426,7 @@ void main() { () async { fileSystem .directory( - artifacts.getArtifactPath(Artifact.flutterMacOSFramework, - mode: BuildMode.debug), + artifacts.getArtifactPath(Artifact.flutterMacOSFramework, mode: BuildMode.debug), ) .createSync(); fileSystem @@ -473,25 +459,20 @@ void main() { expect( fileSystem - .file( - 'App.framework/Versions/A/Resources/flutter_assets/kernel_blob.bin') + .file('App.framework/Versions/A/Resources/flutter_assets/kernel_blob.bin') .readAsStringSync(), 'testing', ); expect( - fileSystem - .file('App.framework/Versions/A/Resources/Info.plist') - .readAsStringSync(), + fileSystem.file('App.framework/Versions/A/Resources/Info.plist').readAsStringSync(), contains('io.flutter.flutter.app'), ); expect( - fileSystem.file( - 'App.framework/Versions/A/Resources/flutter_assets/vm_snapshot_data'), + fileSystem.file('App.framework/Versions/A/Resources/flutter_assets/vm_snapshot_data'), exists, ); expect( - fileSystem.file( - 'App.framework/Versions/A/Resources/flutter_assets/isolate_snapshot_data'), + fileSystem.file('App.framework/Versions/A/Resources/flutter_assets/isolate_snapshot_data'), exists, ); }, @@ -594,30 +575,23 @@ void main() { fileSystem .file('bin/cache/artifacts/engine/darwin-x64/isolate_snapshot.bin') .createSync(recursive: true); - fileSystem - .file('${environment.buildDir.path}/App.framework/App') - .createSync(recursive: true); - fileSystem - .file('${environment.buildDir.path}/native_assets.json') - .createSync(); + fileSystem.file('${environment.buildDir.path}/App.framework/App').createSync(recursive: true); + fileSystem.file('${environment.buildDir.path}/native_assets.json').createSync(); await const ProfileMacOSBundleFlutterAssets().build( environment..defines[kBuildMode] = 'profile', ); expect( - fileSystem.file( - 'App.framework/Versions/A/Resources/flutter_assets/kernel_blob.bin'), + fileSystem.file('App.framework/Versions/A/Resources/flutter_assets/kernel_blob.bin'), isNot(exists), ); expect( - fileSystem.file( - 'App.framework/Versions/A/Resources/flutter_assets/vm_snapshot_data'), + fileSystem.file('App.framework/Versions/A/Resources/flutter_assets/vm_snapshot_data'), isNot(exists), ); expect( - fileSystem.file( - 'App.framework/Versions/A/Resources/flutter_assets/isolate_snapshot_data'), + fileSystem.file('App.framework/Versions/A/Resources/flutter_assets/isolate_snapshot_data'), isNot(exists), ); }, @@ -636,23 +610,17 @@ void main() { fileSystem .file('bin/cache/artifacts/engine/darwin-x64/isolate_snapshot.bin') .createSync(recursive: true); + fileSystem.file('${environment.buildDir.path}/App.framework/App').createSync(recursive: true); fileSystem - .file('${environment.buildDir.path}/App.framework/App') - .createSync(recursive: true); - fileSystem - .file( - '${environment.buildDir.path}/App.framework.dSYM/Contents/Resources/DWARF/App') + .file('${environment.buildDir.path}/App.framework.dSYM/Contents/Resources/DWARF/App') .createSync(recursive: true); - fileSystem - .file('${environment.buildDir.path}/native_assets.json') - .createSync(); + fileSystem.file('${environment.buildDir.path}/native_assets.json').createSync(); await const ReleaseMacOSBundleFlutterAssets().build( environment..defines[kBuildMode] = 'release', ); - expect(fileSystem.file('App.framework.dSYM/Contents/Resources/DWARF/App'), - exists); + expect(fileSystem.file('App.framework.dSYM/Contents/Resources/DWARF/App'), exists); }, overrides: { FileSystem: () => fileSystem, @@ -669,20 +637,17 @@ void main() { fileSystem .file('bin/cache/artifacts/engine/darwin-x64/isolate_snapshot.bin') .createSync(recursive: true); - final File inputFramework = fileSystem.file(fileSystem.path - .join(environment.buildDir.path, 'App.framework', 'App')) - ..createSync(recursive: true) - ..writeAsStringSync('ABC'); - fileSystem - .file(environment.buildDir.childFile('native_assets.json')) - .createSync(); + final File inputFramework = + fileSystem.file(fileSystem.path.join(environment.buildDir.path, 'App.framework', 'App')) + ..createSync(recursive: true) + ..writeAsStringSync('ABC'); + fileSystem.file(environment.buildDir.childFile('native_assets.json')).createSync(); await const ProfileMacOSBundleFlutterAssets().build( environment..defines[kBuildMode] = 'profile', ); final File outputFramework = fileSystem.file( - fileSystem.path - .join(environment.outputDir.path, 'App.framework', 'App'), + fileSystem.path.join(environment.outputDir.path, 'App.framework', 'App'), ); expect(outputFramework.readAsStringSync(), 'ABC'); @@ -713,12 +678,9 @@ void main() { .file('bin/cache/artifacts/engine/darwin-x64/isolate_snapshot.bin') .createSync(recursive: true); fileSystem - .file(fileSystem.path - .join(environment.buildDir.path, 'App.framework', 'App')) + .file(fileSystem.path.join(environment.buildDir.path, 'App.framework', 'App')) .createSync(recursive: true); - fileSystem - .file(environment.buildDir.childFile('native_assets.json')) - .createSync(); + fileSystem.file(environment.buildDir.childFile('native_assets.json')).createSync(); await const ReleaseMacOSBundleFlutterAssets().build(environment); expect( @@ -752,8 +714,7 @@ void main() { expect( fakeAnalytics.sentEvents, contains( - Event.appleUsageEvent( - workflow: 'assemble', parameter: 'macos-archive', result: 'fail'), + Event.appleUsageEvent(workflow: 'assemble', parameter: 'macos-archive', result: 'fail'), ), ); }, @@ -790,10 +751,7 @@ void main() { '-install_name', '@rpath/App.framework/App', '-o', - environment.buildDir - .childDirectory('App.framework') - .childFile('App') - .path, + environment.buildDir.childDirectory('App.framework').childFile('App').path, ], ), ); @@ -824,8 +782,7 @@ void main() { // Set up App.framework binary fileSystem - .file(fileSystem.path - .join(environment.buildDir.path, 'App.framework', 'App')) + .file(fileSystem.path.join(environment.buildDir.path, 'App.framework', 'App')) .createSync(recursive: true); // Set up native_assets.json (required by MacOSBundleFlutterAssets) @@ -867,8 +824,7 @@ flavors: 'flutter_assets', 'shorebird.yaml', ); - expect(fileSystem.file(shorebirdYamlPath).readAsStringSync(), - 'app_id: internal-app-id'); + expect(fileSystem.file(shorebirdYamlPath).readAsStringSync(), 'app_id: internal-app-id'); }, overrides: { FileSystem: () => fileSystem, @@ -905,10 +861,7 @@ flavors: '-install_name', '@rpath/App.framework/App', '-o', - environment.buildDir - .childDirectory('App.framework') - .childFile('App') - .path, + environment.buildDir.childDirectory('App.framework').childFile('App').path, ], ), ); @@ -1080,18 +1033,14 @@ flavors: command: [ 'lipo', environment.buildDir - .childFile( - 'arm64/App.framework.dSYM/Contents/Resources/DWARF/App') + .childFile('arm64/App.framework.dSYM/Contents/Resources/DWARF/App') .path, environment.buildDir - .childFile( - 'x86_64/App.framework.dSYM/Contents/Resources/DWARF/App') + .childFile('x86_64/App.framework.dSYM/Contents/Resources/DWARF/App') .path, '-create', '-output', - environment.buildDir - .childFile('App.framework.dSYM/Contents/Resources/DWARF/App') - .path, + environment.buildDir.childFile('App.framework.dSYM/Contents/Resources/DWARF/App').path, ], ), ]); diff --git a/packages/flutter_tools/test/general.shard/cache_test.dart b/packages/flutter_tools/test/general.shard/cache_test.dart index d0ce7ac81e3bc..231947714fb44 100644 --- a/packages/flutter_tools/test/general.shard/cache_test.dart +++ b/packages/flutter_tools/test/general.shard/cache_test.dart @@ -14,7 +14,6 @@ import 'package:flutter_tools/src/base/io.dart'; import 'package:flutter_tools/src/base/logger.dart'; import 'package:flutter_tools/src/base/os.dart'; import 'package:flutter_tools/src/base/platform.dart'; -import 'package:flutter_tools/src/base/terminal.dart'; import 'package:flutter_tools/src/cache.dart'; import 'package:flutter_tools/src/dart/pub.dart'; import 'package:flutter_tools/src/flutter_cache.dart'; @@ -24,7 +23,6 @@ import 'package:test/fake.dart'; import '../src/common.dart'; import '../src/context.dart'; -import '../src/fake_http_client.dart'; import '../src/fakes.dart'; const unameCommandForX64 = FakeCommand(command: ['uname', '-m'], stdout: 'x86_64'); @@ -415,468 +413,6 @@ void main() { ); }); - testWithoutContext('ArtifactSet.displayName defaults to name', () { - final cache = Cache.test(processManager: FakeProcessManager.any()); - final artifact = FakeSimpleArtifact(cache); - - expect(artifact.name, 'fake'); - expect(artifact.displayName, 'fake'); - }); - - testWithoutContext('ArtifactSet.downloadCount defaults to 1', () { - final cache = Cache.test(processManager: FakeProcessManager.any()); - final artifact = FakeSimpleArtifact(cache); - - expect(artifact.downloadCount, 1); - }); - - testWithoutContext( - 'EngineCachedArtifact.downloadCount returns sum of package and binary dirs', - () { - final FileSystem fileSystem = MemoryFileSystem.test(); - final Directory artifactDir = fileSystem.systemTempDirectory.createTempSync( - 'flutter_cache_test_artifact.', - ); - final cache = FakeSecondaryCache()..artifactDirectory = artifactDir; - - final artifact = FakeCachedArtifact( - cache: cache, - binaryDirs: >[ - ['dir1', 'url1'], - ['dir2', 'url2'], - ], - packageDirs: ['pkg1', 'pkg2', 'pkg3'], - requiredArtifacts: DevelopmentArtifact.universal, - ); - - expect(artifact.downloadCount, 5); - }, - ); - - group('ArtifactUpdater progress formatting', () { - late FileSystem fileSystem; - late Directory tempStorage; - late ArtifactUpdater artifactUpdater; - - setUp(() { - fileSystem = MemoryFileSystem.test(); - tempStorage = fileSystem.systemTempDirectory.createTempSync('temp.'); - artifactUpdater = ArtifactUpdater( - operatingSystemUtils: FakeOperatingSystemUtils(), - logger: BufferLogger.test(), - fileSystem: fileSystem, - tempStorage: tempStorage, - httpClient: FakeHttpClient.any(), - platform: FakePlatform(), - allowedBaseUrls: ['https://storage.googleapis.com'], - ); - }); - - testWithoutContext('formats single download with artifact index', () { - artifactUpdater.setProgressContext(artifactIndex: 2, artifactTotal: 5, downloadTotal: 1); - - expect(artifactUpdater.formatProgressMessage('Test Artifact'), '[2/5] Test Artifact'); - }); - - testWithoutContext('formats multiple downloads with tree prefix', () { - // First download gets โ”œโ”€ prefix - artifactUpdater.setProgressContext(artifactIndex: 1, artifactTotal: 3, downloadTotal: 4); - expect(artifactUpdater.formatProgressMessage('first'), ' โ”œโ”€ [1/4] first'); - - // Middle downloads also get โ”œโ”€ prefix - artifactUpdater.setProgressContext( - artifactIndex: 1, - artifactTotal: 3, - downloadTotal: 4, - downloadIndex: 1, - ); - expect(artifactUpdater.formatProgressMessage('second'), ' โ”œโ”€ [2/4] second'); - - artifactUpdater.setProgressContext( - artifactIndex: 1, - artifactTotal: 3, - downloadTotal: 4, - downloadIndex: 2, - ); - expect(artifactUpdater.formatProgressMessage('third'), ' โ”œโ”€ [3/4] third'); - - // Last download gets โ””โ”€ prefix - artifactUpdater.setProgressContext( - artifactIndex: 1, - artifactTotal: 3, - downloadTotal: 4, - downloadIndex: 3, - ); - expect(artifactUpdater.formatProgressMessage('fourth'), ' โ””โ”€ [4/4] fourth'); - }); - - testWithoutContext('resetProgressContext resets download index', () { - artifactUpdater.setProgressContext(artifactIndex: 2, artifactTotal: 5, downloadTotal: 1); - expect(artifactUpdater.formatProgressMessage('first'), '[2/5] first'); - - artifactUpdater.resetProgressContext(); - artifactUpdater.setProgressContext(artifactIndex: 1, artifactTotal: 3, downloadTotal: 1); - - // After reset, index should start fresh - expect(artifactUpdater.formatProgressMessage('new'), '[1/3] new'); - }); - }); - - group('DownloadProgress', () { - testWithoutContext('fraction and percent with known total', () { - final progress = DownloadProgress()..totalBytes = 1000; - progress.addBytesReceived(250); - - expect(progress.fractionReceived, 0.25); - expect(progress.percentReceived, 25); - }); - - testWithoutContext('fraction and percent with unknown total', () { - final progress = DownloadProgress(); - progress.addBytesReceived(500); - - expect(progress.fractionReceived, 0.0); - expect(progress.percentReceived, 0); - expect(progress.hasKnownSize, false); - }); - - testWithoutContext('fraction clamps to 1.0 when bytesReceived exceeds total', () { - final progress = DownloadProgress()..totalBytes = 100; - progress.addBytesReceived(150); - - expect(progress.fractionReceived, 1.0); - expect(progress.percentReceived, 100); - }); - - testWithoutContext('speed calculation', () { - final progress = DownloadProgress(); - progress.addBytesReceived(5000000); - const elapsed = Duration(seconds: 2); - - expect(progress.speedBytesPerSecond(elapsed), 2500000.0); - }); - - testWithoutContext('speed is zero when elapsed is zero', () { - final progress = DownloadProgress(); - progress.addBytesReceived(5000000); - - expect(progress.speedBytesPerSecond(Duration.zero), 0.0); - }); - - testWithoutContext('timeRemaining with known total', () { - final progress = DownloadProgress()..totalBytes = 10000000; - progress.addBytesReceived(5000000); - const elapsed = Duration(seconds: 2); - - // 5MB received in 2s = 2.5MB/s, 5MB timeRemaining = 2s timeRemaining - final Duration? rem = progress.timeRemaining(elapsed); - expect(rem, isNotNull); - expect(rem!.inSeconds, 2); - }); - - testWithoutContext('timeRemaining is null with unknown total', () { - final progress = DownloadProgress(); - progress.addBytesReceived(5000000); - - expect(progress.timeRemaining(const Duration(seconds: 2)), isNull); - }); - - testWithoutContext('timeRemaining is null when speed is zero', () { - final progress = DownloadProgress()..totalBytes = 10000000; - progress.addBytesReceived(5000000); - - expect(progress.timeRemaining(Duration.zero), isNull); - }); - - testWithoutContext('renderProgressBar at 50%', () { - final progress = DownloadProgress()..totalBytes = 100; - progress.addBytesReceived(50); - - final String bar = progress.renderProgressBar(20); - expect(bar.length, 20); - expect(bar, '${'โ–ˆ' * 10}${' ' * 10}'); - }); - - testWithoutContext('renderProgressBar uses sub-character partial block', () { - // 46% of 20 cells: totalEighths = round(0.46 * 20 * 8) = 74 - // fullBlocks=9, remainder=2 โ†’ partial='โ–Ž', emptyBlocks=10 - final progress = DownloadProgress()..totalBytes = 100; - progress.addBytesReceived(46); - - final String bar = progress.renderProgressBar(20); - expect(bar.length, 20); - expect(bar, '${'โ–ˆ' * 9}โ–Ž${' ' * 10}'); - }); - - testWithoutContext('renderProgressBar returns empty with unknown total', () { - final progress = DownloadProgress(); - progress.addBytesReceived(50); - - expect(progress.renderProgressBar(20), ''); - }); - - testWithoutContext('renderProgressBar returns empty with zero width', () { - final progress = DownloadProgress()..totalBytes = 100; - progress.addBytesReceived(50); - - expect(progress.renderProgressBar(0), ''); - }); - - testWithoutContext('formatBytes with known total', () { - final progress = DownloadProgress()..totalBytes = 10000000; - progress.addBytesReceived(5000000); - - expect(progress.formatBytes(), matches(r'^\d+\.\d+MB/\d+\.\d+MB$')); - }); - - testWithoutContext('formatBytes with unknown total', () { - final progress = DownloadProgress(); - progress.addBytesReceived(5000000); - - expect(progress.formatBytes(), matches(r'^\d+\.\d+MB$')); - }); - - testWithoutContext('formatSpeed', () { - final progress = DownloadProgress(); - progress.addBytesReceived(2000000); - - expect(progress.formatSpeed(const Duration(seconds: 1)), matches(r'^\d+\.\d+MB/s$')); - }); - - testWithoutContext('formatRemaining with known total and speed', () { - final progress = DownloadProgress()..totalBytes = 10000000; - progress.addBytesReceived(5000000); - - final String eta = progress.formatRemaining(const Duration(seconds: 2)); - expect(eta, startsWith('ETA ')); - expect(eta, contains('s')); - }); - - testWithoutContext('formatRemaining is empty with unknown total', () { - final progress = DownloadProgress(); - progress.addBytesReceived(5000000); - - expect(progress.formatRemaining(const Duration(seconds: 2)), ''); - }); - - testWithoutContext('formatProgressLine includes progress bar with known total', () { - final progress = DownloadProgress()..totalBytes = 10000000; - progress.addBytesReceived(5000000); - - final String line = progress.formatProgressLine( - elapsed: const Duration(seconds: 2), - terminalWidth: 80, - ); - - expect(line, contains('50%')); - expect(line, matches(r'\d+\.\d+MB/\d+\.\d+MB')); - expect(line, contains('/s')); - expect(line, contains('โ–ˆ')); - expect(line, contains('โ–')); - expect(line, contains('โ–•')); - }); - - testWithoutContext('formatProgressLine has fixed bar width and right-aligns info', () { - final progress = DownloadProgress()..totalBytes = 10000000; - progress.addBytesReceived(5000000); - - final String line = progress.formatProgressLine( - elapsed: const Duration(seconds: 2), - terminalWidth: 80, - ); - - // Bar is fixed at 28 inner chars + 2 brackets = 30 visual chars. - // Line total must equal terminalWidth. - expect(line.length, 80); - // Info is right-aligned: the line must end with the info, no trailing spaces. - expect(line, endsWith('ETA 2.0s')); - }); - - testWithoutContext('formatProgressLine omits bar with unknown total', () { - final progress = DownloadProgress(); - progress.addBytesReceived(5000000); - - final String line = progress.formatProgressLine( - elapsed: const Duration(seconds: 2), - terminalWidth: 80, - ); - - expect(line, isNot(contains('โ–ˆ'))); - expect(line, matches(r'\d+\.\d+MB')); - expect(line, contains('/s')); - }); - - testWithoutContext('formatProgressLine omits bar when terminal too narrow', () { - final progress = DownloadProgress()..totalBytes = 10000000; - progress.addBytesReceived(5000000); - - final String line = progress.formatProgressLine( - elapsed: const Duration(seconds: 2), - terminalWidth: 30, - ); - - expect(line, isNot(contains('โ–ˆ'))); - expect(line, contains('50%')); - }); - - testWithoutContext('formatProgressLine never exceeds terminalWidth', () { - final progress = DownloadProgress()..totalBytes = 10000000; - progress.addBytesReceived(5000000); - - final String line = progress.formatProgressLine( - elapsed: const Duration(seconds: 2), - terminalWidth: 30, - ); - - expect(line.length, lessThanOrEqualTo(30)); - }); - - testWithoutContext( - 'formatProgressLine does not have trailing whitespace with unknown total', - () { - final progress = DownloadProgress(); - progress.addBytesReceived(5000000); - - final String line = progress.formatProgressLine( - elapsed: const Duration(seconds: 2), - terminalWidth: 80, - ); - - expect(line, isNot(endsWith(' '))); - }, - ); - - testWithoutContext('formatCompletionSummary', () { - final progress = DownloadProgress(); - progress.addBytesReceived(21100000); - - expect( - progress.formatCompletionSummary(const Duration(seconds: 5)), - matches(r'^\(\d+\.\d+MB in \d+\.\d+s\)$'), - ); - }); - }); - - group('ArtifactUpdater download progress display', () { - late FileSystem fileSystem; - late Directory tempStorage; - late FakeStdio stdio; - late BufferLogger logger; - - setUp(() { - fileSystem = MemoryFileSystem.test(); - tempStorage = fileSystem.systemTempDirectory.createTempSync('temp.'); - stdio = FakeStdio(); - logger = BufferLogger.test(terminal: Terminal.test(supportsColor: true)); - }); - - testWithoutContext( - 'shows ANSI progress when stdio is provided and color is supported', - () async { - final body = List.filled(1000, 0); - final updater = ArtifactUpdater( - operatingSystemUtils: FakeOperatingSystemUtils(), - logger: logger, - fileSystem: fileSystem, - tempStorage: tempStorage, - httpClient: FakeHttpClient.list([ - FakeRequest( - Uri.parse('https://storage.googleapis.com/test-artifact.zip'), - response: FakeResponse(body: body), - ), - ]), - platform: FakePlatform(), - allowedBaseUrls: ['https://storage.googleapis.com'], - stdio: stdio, - ); - - updater.setProgressContext(artifactIndex: 1, artifactTotal: 1, downloadTotal: 1); - - await updater.downloadZipArchive( - 'Test Artifact', - Uri.parse('https://storage.googleapis.com/test-artifact.zip'), - fileSystem.directory('output')..createSync(), - ); - - final String allOutput = stdio.writtenToStdout.join(); - - // Should have written the status message followed by newline - expect(allOutput, contains('[1/1] Test Artifact')); - - // Should contain completion summary with size and time - expect(allOutput, contains('MB in')); - expect(allOutput, contains('s)')); - - // Should contain ANSI clear codes for cleanup - expect(allOutput, contains('\x1B[K')); // clear to end of line - expect(allOutput, contains('\x1B[1A')); // cursor up - - // Completion line should be right-aligned to terminal width (80). - final String completionLine = stdio.writtenToStdout.last.replaceAll('\n', ''); - expect(completionLine.length, 80); - }, - ); - - testWithoutContext('falls back to logger progress when stdio is not provided', () async { - final updater = ArtifactUpdater( - operatingSystemUtils: FakeOperatingSystemUtils(), - logger: logger, - fileSystem: fileSystem, - tempStorage: tempStorage, - httpClient: FakeHttpClient.list([ - FakeRequest( - Uri.parse('https://storage.googleapis.com/test-artifact.zip'), - response: const FakeResponse(body: [0, 0, 0]), - ), - ]), - platform: FakePlatform(), - allowedBaseUrls: ['https://storage.googleapis.com'], - ); - - updater.setProgressContext(artifactIndex: 1, artifactTotal: 1, downloadTotal: 1); - - await updater.downloadZipArchive( - 'Test Artifact', - Uri.parse('https://storage.googleapis.com/test-artifact.zip'), - fileSystem.directory('output')..createSync(), - ); - - // Without stdio, nothing should be written to stdout directly - expect(stdio.writtenToStdout, isEmpty); - }); - - testWithoutContext('falls back to logger progress when color is not supported', () async { - final noColorLogger = BufferLogger.test(); - - final updater = ArtifactUpdater( - operatingSystemUtils: FakeOperatingSystemUtils(), - logger: noColorLogger, - fileSystem: fileSystem, - tempStorage: tempStorage, - httpClient: FakeHttpClient.list([ - FakeRequest( - Uri.parse('https://storage.googleapis.com/test-artifact.zip'), - response: const FakeResponse(body: [0, 0, 0]), - ), - ]), - platform: FakePlatform(), - allowedBaseUrls: ['https://storage.googleapis.com'], - stdio: stdio, - ); - - updater.setProgressContext(artifactIndex: 1, artifactTotal: 1, downloadTotal: 1); - - await updater.downloadZipArchive( - 'Test Artifact', - Uri.parse('https://storage.googleapis.com/test-artifact.zip'), - fileSystem.directory('output')..createSync(), - ); - - // With stdio but no color support, should not write progress to stdout - expect(stdio.writtenToStdout, isEmpty); - }); - }); - testWithoutContext( 'EngineCachedArtifact makes binary dirs readable and executable by all', () async { @@ -1397,7 +933,7 @@ void main() { await webSdk.updateInner(artifactUpdater, fileSystem, FakeOperatingSystemUtils()); - expect(messages, ['Web SDK']); + expect(messages, ['Downloading Web SDK...']); expect(downloads, [ 'https://download.shorebird.dev/flutter_infra_release/flutter/hijklmnop/flutter-web-sdk.zip', @@ -1518,7 +1054,7 @@ void main() { await engineStamp.updateInner(artifactUpdater, fileSystem, FakeOperatingSystemUtils()); - expect(messages, ['Engine Information']); + expect(messages, ['Downloading engine information...']); expect(downloads, [ 'https://download.shorebird.dev/flutter_infra_release/flutter/hijklmnop/engine_stamp.json', @@ -1836,12 +1372,6 @@ class FakeSecondaryCachedArtifact extends Fake implements CachedArtifact { @override DevelopmentArtifact get developmentArtifact => DevelopmentArtifact.universal; - - @override - String get displayName => 'fake'; - - @override - int get downloadCount => 1; } class FakeIosUsbArtifacts extends Fake implements IosUsbArtifacts { @@ -2024,33 +1554,22 @@ class FakeArtifactUpdater extends Fake implements ArtifactUpdater { void Function(String, Uri, Directory)? onDownloadFile; @override - Future downloadZippedTarball(String artifactName, Uri url, Directory location) async { - onDownloadZipTarball?.call(artifactName, url, location); + Future downloadZippedTarball(String message, Uri url, Directory location) async { + onDownloadZipTarball?.call(message, url, location); } @override - Future downloadZipArchive(String artifactName, Uri url, Directory location) async { - onDownloadZipArchive?.call(artifactName, url, location); + Future downloadZipArchive(String message, Uri url, Directory location) async { + onDownloadZipArchive?.call(message, url, location); } @override - Future downloadFile(String artifactName, Uri url, Directory location) async { - onDownloadFile?.call(artifactName, url, location); + Future downloadFile(String message, Uri url, Directory location) async { + onDownloadFile?.call(message, url, location); } @override void removeDownloadedFiles() {} - - @override - void setProgressContext({ - required int artifactIndex, - required int artifactTotal, - required int downloadTotal, - int downloadIndex = 0, - }) {} - - @override - void resetProgressContext() {} } class FakeArtifactUpdaterDownload extends ArtifactUpdater { diff --git a/packages/flutter_tools/test/general.shard/shorebird/shorebird_yaml_test.dart b/packages/flutter_tools/test/general.shard/shorebird/shorebird_yaml_test.dart index 7e05fec171409..240ae45f5bb18 100644 --- a/packages/flutter_tools/test/general.shard/shorebird/shorebird_yaml_test.dart +++ b/packages/flutter_tools/test/general.shard/shorebird/shorebird_yaml_test.dart @@ -20,8 +20,11 @@ auto_update: false '''; final YamlDocument input = loadYamlDocument(yamlContents); final YamlMap yamlMap = input.contents as YamlMap; - final Map compiled = - compileShorebirdYaml(yamlMap, flavor: null, environment: {}); + final Map compiled = compileShorebirdYaml( + yamlMap, + flavor: null, + environment: {}, + ); expect(compiled, { 'app_id': '6160a7d8-cc18-4928-1233-05b51c0bb02c', 'auto_update': false, @@ -56,16 +59,22 @@ patch_verification: strict '''; final YamlDocument input = loadYamlDocument(yamlContents); final YamlMap yamlMap = input.contents as YamlMap; - final Map compiled1 = - compileShorebirdYaml(yamlMap, flavor: null, environment: {}); + final Map compiled1 = compileShorebirdYaml( + yamlMap, + flavor: null, + environment: {}, + ); expect(compiled1, { 'app_id': '1-a', 'auto_update': false, 'base_url': 'https://example.com', 'patch_verification': 'strict', }); - final Map compiled2 = - compileShorebirdYaml(yamlMap, flavor: 'foo', environment: {'SHOREBIRD_PUBLIC_KEY': '4-a'}); + final Map compiled2 = compileShorebirdYaml( + yamlMap, + flavor: 'foo', + environment: {'SHOREBIRD_PUBLIC_KEY': '4-a'}, + ); expect(compiled2, { 'app_id': '2-a', 'auto_update': false, @@ -83,7 +92,7 @@ flavors: bar: 3-a base_url: https://example.com '''; - // Make a temporary file to test editing in place. + // Make a temporary file to test editing in place. final Directory tempDir = Directory.systemTemp.createTempSync('shorebird_yaml_test.'); final File tempFile = File('${tempDir.path}/shorebird.yaml'); tempFile.writeAsStringSync(yamlContents); diff --git a/packages/flutter_tools/test/general.shard/version_test.dart b/packages/flutter_tools/test/general.shard/version_test.dart index a81f119bb1602..4357eda448701 100644 --- a/packages/flutter_tools/test/general.shard/version_test.dart +++ b/packages/flutter_tools/test/general.shard/version_test.dart @@ -1610,36 +1610,39 @@ void main() { expect(processManager, hasNoRemainingExpectations); }); - testUsingContext('determine picks stable branch over rc branch from shorebird flutter_release', () { - processManager.addCommands([ - const FakeCommand( - command: ['git', 'tag', '--points-at', 'HEAD'], - // No tag at HEAD. - ), - const FakeCommand( - command: [ - 'git', - 'for-each-ref', - '--contains', - 'HEAD', - '--format', - '%(refname:short)', - 'refs/remotes/origin/flutter_release/*', - ], - // Both stable and rc branches contain this commit. - stdout: 'origin/flutter_release/3.41.4\norigin/flutter_release/3.41.4-rc2', - ), - ]); - final platform = FakePlatform(); + testUsingContext( + 'determine picks stable branch over rc branch from shorebird flutter_release', + () { + processManager.addCommands([ + const FakeCommand( + command: ['git', 'tag', '--points-at', 'HEAD'], + // No tag at HEAD. + ), + const FakeCommand( + command: [ + 'git', + 'for-each-ref', + '--contains', + 'HEAD', + '--format', + '%(refname:short)', + 'refs/remotes/origin/flutter_release/*', + ], + // Both stable and rc branches contain this commit. + stdout: 'origin/flutter_release/3.41.4\norigin/flutter_release/3.41.4-rc2', + ), + ]); + final platform = FakePlatform(); - final GitTagVersion gitTagVersion = GitTagVersion.determine( - platform, - git: git, - workingDirectory: '.', - ); - expect(gitTagVersion.frameworkVersionFor('abcd1234'), '3.41.4'); - expect(processManager, hasNoRemainingExpectations); - }); + final GitTagVersion gitTagVersion = GitTagVersion.determine( + platform, + git: git, + workingDirectory: '.', + ); + expect(gitTagVersion.frameworkVersionFor('abcd1234'), '3.41.4'); + expect(processManager, hasNoRemainingExpectations); + }, + ); group('$FlutterEngineStampFromFile', () { late FileSystem fs; From cd8c0c9294f550fe8ed385b68786e93709244a5a Mon Sep 17 00:00:00 2001 From: Eric Seidel Date: Wed, 22 Apr 2026 20:34:49 -0700 Subject: [PATCH 47/95] feat: add verbose logging to shorebird_tests for CI debugging (#107) Stream subprocess output in real time when VERBOSE is set, and dump stdout/stderr on failure. This helps diagnose flaky timeout failures in CI where buffered output is lost when the test times out. --- .github/workflows/shorebird_ci.yml | 3 ++ .../shorebird_tests/test/shorebird_tests.dart | 31 +++++++++++++------ 2 files changed, 25 insertions(+), 9 deletions(-) diff --git a/.github/workflows/shorebird_ci.yml b/.github/workflows/shorebird_ci.yml index 83e1511d102de..526c04eb1e53c 100644 --- a/.github/workflows/shorebird_ci.yml +++ b/.github/workflows/shorebird_ci.yml @@ -138,3 +138,6 @@ jobs: # Quick sanity check that Android builds work on macOS too run: dart test test/android_test.dart --name "can build an apk" working-directory: packages/shorebird_tests + env: + # Enables streaming subprocess output for debugging timeouts. + VERBOSE: "1" diff --git a/packages/shorebird_tests/test/shorebird_tests.dart b/packages/shorebird_tests/test/shorebird_tests.dart index 409f3963e95e1..e63a8d99d89b6 100644 --- a/packages/shorebird_tests/test/shorebird_tests.dart +++ b/packages/shorebird_tests/test/shorebird_tests.dart @@ -23,10 +23,16 @@ File get _flutterBinaryFile => File( ), ); +/// Whether to print line-by-line subprocess output. +/// +/// Set the `VERBOSE` environment variable to enable streaming output, +/// which is useful for debugging timeouts in CI. +final bool _verbose = Platform.environment.containsKey('VERBOSE'); + /// Runs a flutter command using the correct binary ([_flutterBinaryFile]) with the given arguments. /// -/// Streams stdout and stderr to the test output in real time so that -/// CI logs show progress even if the process hangs or times out. +/// Streams stdout and stderr to the test output in real time when [_verbose] +/// is true, so CI logs show progress even if the process hangs or times out. Future _runFlutterCommand( List arguments, { required Directory workingDirectory, @@ -51,19 +57,22 @@ Future _runFlutterCommand( process.stdout.transform(utf8.decoder).listen((String data) { stdoutBuffer.write(data); - // Print each line with a prefix so it's easy to identify in CI logs. - for (final String line in data.split('\n')) { - if (line.isNotEmpty) { - print(' [$command] $line'); + if (_verbose) { + for (final String line in data.split('\n')) { + if (line.isNotEmpty) { + print(' [$command] $line'); + } } } }); process.stderr.transform(utf8.decoder).listen((String data) { stderrBuffer.write(data); - for (final String line in data.split('\n')) { - if (line.isNotEmpty) { - print(' [$command] (stderr) $line'); + if (_verbose) { + for (final String line in data.split('\n')) { + if (line.isNotEmpty) { + print(' [$command] (stderr) $line'); + } } } }); @@ -72,6 +81,10 @@ Future _runFlutterCommand( stopwatch.stop(); print('[$command] completed in ${stopwatch.elapsed} ' '(exit code $exitCode)'); + if (exitCode != 0 && !_verbose) { + print('[$command] stdout:\n$stdoutBuffer'); + print('[$command] stderr:\n$stderrBuffer'); + } return ProcessResult( process.pid, From cba2d9a6c02272f359f88dda012607a11aafef31 Mon Sep 17 00:00:00 2001 From: Eric Seidel Date: Mon, 4 May 2026 16:11:52 -0700 Subject: [PATCH 48/95] refactor: split shorebird C API consumption (engine side of updater #350) (#138) * refactor: split shorebird C API consumption (engine side of updater #350) Switch shell/common/shorebird/updater.cc to include updater_engine.h instead of the combined updater.h, which is removed in updater #350. Drop ghost entries from android_exports.lst: - shorebird_active_path, shorebird_active_patch_number: never existed in the Rust crate (already dead before #350). - shorebird_check_for_update, shorebird_update: removed in updater #350, replaced by shorebird_check_for_downloadable_update and shorebird_update_with_result respectively. Coordinated with shorebirdtech/updater#350. The DEPS updater_rev bump is intentionally not included here -- it should land together with the final commit of #350 once the merge order is settled. * chore: bump Updater to 34509fca3c65 --- DEPS | 26 +++++++++---------- .../flutter/shell/common/shorebird/updater.cc | 2 +- .../platform/android/android_exports.lst | 4 --- 3 files changed, 14 insertions(+), 18 deletions(-) diff --git a/DEPS b/DEPS index 94c821d0ca661..56bf116f506b4 100644 --- a/DEPS +++ b/DEPS @@ -20,7 +20,7 @@ vars = { "dart_sdk_revision": "b65ce89c8057d6880e00693a7b0ecd7b9e5f61ca", "dart_sdk_git": "git@github.com:shorebirdtech/dart-sdk.git", "updater_git": "https://github.com/shorebirdtech/updater.git", - "updater_rev": "8691c8f60e69f8eb1f35361f4a22e8c9a7fdf93c", + "updater_rev": "34509fca3c65388ebc84cfaea8f38733ce41f41a", # WARNING: DO NOT EDIT canvaskit_cipd_instance MANUALLY # See `lib/web_ui/README.md` for how to roll CanvasKit to a new version. @@ -48,7 +48,7 @@ vars = { # updates to Clang Tidy will not turn the tree red. # # See https://github.com/flutter/flutter/wiki/Engine-preโ€submits-and-postโ€submits#post-submit - 'clang_version': 'git_revision:80743bd43fd5b38fedc503308e7a652e23d3ec93', + 'clang_version': 'git_revision:8c7a2ce01a77c96028fe2c8566f65c45ad9408d3', 'reclient_version': 're_client_version:0.185.0.db415f21-gomaip', @@ -240,10 +240,10 @@ deps = { Var('chromium_git') + '/chromium/tools/depot_tools.git' + '@' + '580b4ff3f5cd0dcaa2eacda28cefe0f45320e8f7', 'engine/src/flutter/third_party/rapidjson': - Var('flutter_git') + '/third_party/rapidjson' + '@' + '47253cab97e9cfe99dbd6b90836fc11589d7d802', + Var('flutter_git') + '/third_party/rapidjson' + '@' + 'ef3564c5c8824989393b87df25355baf35ff544b', 'engine/src/flutter/third_party/harfbuzz': - Var('flutter_git') + '/third_party/harfbuzz' + '@' + '49844c32a7a3f6be371355a1213c952a3f4a44e7', + Var('flutter_git') + '/third_party/harfbuzz' + '@' + 'ea6a172f84f2cbcfed803b5ae71064c7afb6b5c2', 'engine/src/flutter/third_party/libcxx': Var('llvm_git') + '/llvm-project/libcxx' + '@' + 'bd557f6f764d1e40b62528a13b124ce740624f8f', @@ -255,10 +255,10 @@ deps = { Var('llvm_git') + '/llvm-project/libc' + '@' + '5af39a19a1ad51ce93972cdab206dcd3ff9b6afa', 'engine/src/flutter/third_party/glfw': - Var('flutter_git') + '/third_party/glfw' + '@' + '9352d8fe93cd443be18157abe81f16500549aec0', + Var('flutter_git') + '/third_party/glfw' + '@' + 'dd8a678a66f1967372e5a5e3deac41ebf65ee127', 'engine/src/flutter/third_party/shaderc': - Var('chromium_git') + '/external/github.com/google/shaderc' + '@' + 'd15277d6bc180f6a0b8b601f0cab2bbcaac9b4d5', + Var('chromium_git') + '/external/github.com/google/shaderc' + '@' + '37e25539ce199ecaf19fb7f7d27818716d36686d', 'engine/src/flutter/third_party/vulkan-deps': Var('chromium_git') + '/vulkan-deps' + '@' + 'a9e2ca3b57aba86a22a2df1b84bf12f8cc98806e', @@ -276,7 +276,7 @@ deps = { Var('chromium_git') + '/external/github.com/google/benchmark' + '@' + '431abd149fd76a072f821913c0340137cc755f36', 'engine/src/flutter/third_party/googletest': - Var('chromium_git') + '/external/github.com/google/googletest' + '@' + 'e9907112b47255d50b4d343e7e2160bce8dc85d1', + Var('chromium_git') + '/external/github.com/google/googletest' + '@' + '7f036c5563af7d0329f20e8bb42effb04629f0c0', 'engine/src/flutter/third_party/re2': Var('chromium_git') + '/external/github.com/google/re2' + '@' + 'c84a140c93352cdabbfb547c531be34515b12228', @@ -305,7 +305,7 @@ deps = { {'dep_type': 'cipd', 'packages': [{'package': 'dart/third_party/flutter/devtools', 'version': 'git_revision:12d595649f189f1896722623f72599077f476848'}]}, 'engine/src/flutter/third_party/dart/third_party/perfetto/src': - Var('chromium_git') + '/external/github.com/google/perfetto' + '@' + Var('dart_perfetto_rev'), + Var('android_git') + '/platform/external/perfetto' + '@' + Var('dart_perfetto_rev'), 'engine/src/flutter/third_party/dart/third_party/pkg/core': Var('dart_git') + '/core.git' + '@' + Var('dart_core_rev'), @@ -477,7 +477,7 @@ deps = { Var('chromium_git') + '/external/github.com/libexpat/libexpat.git' + '@' + '8e49998f003d693213b538ef765814c7d21abada', 'engine/src/flutter/third_party/freetype2': - Var('flutter_git') + '/third_party/freetype2' + '@' + 'be4bcb57914154fc1b9e2900bf8e4b516057e2b8', + Var('flutter_git') + '/third_party/freetype2' + '@' + 'bfc3453fdc85d87b45c896f68bf2e49ebdaeef0a', 'engine/src/flutter/third_party/skia': Var('skia_git') + '/skia.git' + '@' + Var('skia_revision'), @@ -501,7 +501,7 @@ deps = { Var('skia_git') + '/external/github.com/google/wuffs-mirror-release-c.git' + '@' + '600cd96cf47788ee3a74b40a6028b035c9fd6a61', 'engine/src/flutter/third_party/zlib': - Var('chromium_git') + '/chromium/src/third_party/zlib.git' + '@' + '7eda07b1e067ef3fd7eea0419c88b5af45c9a776', + Var('chromium_git') + '/chromium/src/third_party/zlib.git' + '@' + '7d77fb7fd66d8a5640618ad32c71fdeb7d3e02df', 'engine/src/flutter/third_party/cpu_features/src': Var('chromium_git') + '/external/github.com/google/cpu_features.git' + '@' + '936b9ab5515dead115606559502e3864958f7f6e', @@ -522,7 +522,7 @@ deps = { Var('swiftshader_git') + '/SwiftShader.git' + '@' + '794b0cfce1d828d187637e6d932bae484fbe0976', 'engine/src/flutter/third_party/angle': - Var('flutter_git') + '/third_party/angle' + '@' + '84027aca9b71c9ba335bd000dad1107b8810a511', + Var('flutter_git') + '/third_party/angle' + '@' + '6950c6c99fa1a2d653922871ede6679d74840289', 'engine/src/flutter/third_party/vulkan_memory_allocator': Var('chromium_git') + '/external/github.com/GPUOpen-LibrariesAndSDKs/VulkanMemoryAllocator' + '@' + 'c788c52156f3ef7bc7ab769cb03c110a53ac8fcb', @@ -562,7 +562,7 @@ deps = { Var('dart_git') + '/external/github.com/google/vector_math.dart.git' + '@' + '0a5fd95449083d404df9768bc1b321b88a7d2eef', # 2.1.0 'engine/src/flutter/third_party/imgui': - Var('flutter_git') + '/third_party/imgui.git' + '@' + '2a1b69f05748ad909f03acf4533447cac1331611', + Var('flutter_git') + '/third_party/imgui.git' + '@' + '3ea0fad204e994d669f79ed29dcaf61cd5cb571d', 'engine/src/flutter/third_party/json': Var('flutter_git') + '/third_party/json.git' + '@' + '17d9eacd248f58b73f4d1be518ef649fe2295642', @@ -772,7 +772,7 @@ deps = { 'packages': [ { 'package': 'flutter_internal/rbe/reclient_cfgs', - 'version': '0vARzGeIZgIhW7zVfWuqIPQ_HXMLDccjAstykWZKjaEC', + 'version': 'LNMZdvF2Y86Dq05IWthtVJ_PswIFSRiywIHrkfHhelUC', } ], 'condition': 'use_rbe', diff --git a/engine/src/flutter/shell/common/shorebird/updater.cc b/engine/src/flutter/shell/common/shorebird/updater.cc index 63d993613c3f9..2ca435febccf1 100644 --- a/engine/src/flutter/shell/common/shorebird/updater.cc +++ b/engine/src/flutter/shell/common/shorebird/updater.cc @@ -7,7 +7,7 @@ #include "flutter/fml/logging.h" #if SHOREBIRD_PLATFORM_SUPPORTED -#include "third_party/updater/library/include/updater.h" +#include "third_party/updater/library/include/updater_engine.h" #endif namespace flutter { diff --git a/engine/src/flutter/shell/platform/android/android_exports.lst b/engine/src/flutter/shell/platform/android/android_exports.lst index 9b924e7738cd4..85c5d7784f909 100644 --- a/engine/src/flutter/shell/platform/android/android_exports.lst +++ b/engine/src/flutter/shell/platform/android/android_exports.lst @@ -12,13 +12,9 @@ InternalFlutterGpu*; kInternalFlutterGpu*; shorebird_init; - shorebird_active_path; - shorebird_active_patch_number; shorebird_free_string; shorebird_free_update_result; shorebird_check_for_downloadable_update; - shorebird_check_for_update; - shorebird_update; shorebird_update_with_result; shorebird_next_boot_patch_number; shorebird_current_boot_patch_number; From 2bc4650ae26aab1d7e835d2662c55c0d78b3b8af Mon Sep 17 00:00:00 2001 From: Eric Seidel Date: Wed, 6 May 2026 01:39:14 -0700 Subject: [PATCH 49/95] ci: add aggregator job for branch protection (#139) --- .github/workflows/shorebird_ci.yml | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/.github/workflows/shorebird_ci.yml b/.github/workflows/shorebird_ci.yml index 526c04eb1e53c..7f03b472b088c 100644 --- a/.github/workflows/shorebird_ci.yml +++ b/.github/workflows/shorebird_ci.yml @@ -141,3 +141,19 @@ jobs: env: # Enables streaming subprocess output for debugging timeouts. VERBOSE: "1" + + # Aggregator required by branch protection on shorebird/dev so the rule can + # list a single context ("ci") instead of every matrix-expanded job above. + ci: + if: always() + needs: + - flutter-tools-tests + - shard-runner-tests + - shorebird-android-tests + - shorebird-ios-tests + runs-on: ubuntu-latest + steps: + - if: ${{ contains(needs.*.result, 'failure') || contains(needs.*.result, 'cancelled') }} + run: | + echo "One or more upstream jobs failed or were cancelled." + exit 1 From 9666cba844010ac2e2fb2cc09ca88b0568880fc6 Mon Sep 17 00:00:00 2001 From: Eric Seidel Date: Thu, 7 May 2026 15:18:20 -0700 Subject: [PATCH 50/95] fix: honor --flavor for ios-framework builds (#141) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * fix: honor --flavor for ios-framework builds `flutter build ios-framework --flavor X` was silently dropping --flavor before the asset bundling step ran, so flavor-conditional assets and the compiled shorebird.yaml's app_id always fell back to the top-level (non-flavor) value. The same gap exists upstream โ€” tracked at flutter/flutter#186220 with proposed fix flutter/flutter#186221. Adds `kFlavor: ?flavor` to `BuildInfo.toBuildSystemEnvironment()` with a TODO referencing the upstream PR so a future fork-sync can drop the line cleanly once upstream lands. Tests: - Existing `toBuildSystemEnvironment encoding of standard values` test now exercises a non-null flavor and asserts `Flavor` is included. Added a sibling test asserting the key is omitted when flavor is null. - New shorebird-fork-only tests for compileShorebirdYaml flavor compilation: - CopyAssets path, covering the shared codepath used by every platform. - ReleaseIosApplicationBundle, covering the iOS-framework-specific asset bundle target. End-to-end reproduction (vanilla stable Flutter): https://github.com/eseidel/flavor_bundle_repro * fix: drop unnecessary TODO comment on kFlavor line The comment described a non-action: when flutter/flutter#186221 lands, the upstream merge brings in the same kFlavor: ?flavor line and the fork's copy merges cleanly โ€” there's nothing to remove. Carrying the comment just adds a sync-time cleanup task that doesn't otherwise exist. --- .../flutter_tools/lib/src/build_info.dart | 1 + .../test/general.shard/build_info_test.dart | 14 +- .../build_system/targets/assets_test.dart | 335 +++--------------- .../build_system/targets/ios_test.dart | 69 ++++ 4 files changed, 133 insertions(+), 286 deletions(-) diff --git a/packages/flutter_tools/lib/src/build_info.dart b/packages/flutter_tools/lib/src/build_info.dart index 87f3915053887..7d1d39e74dedf 100644 --- a/packages/flutter_tools/lib/src/build_info.dart +++ b/packages/flutter_tools/lib/src/build_info.dart @@ -382,6 +382,7 @@ class BuildInfo { kFileSystemScheme: ?fileSystemScheme, kBuildName: ?buildName, kBuildNumber: ?buildNumber, + kFlavor: ?flavor, if (useLocalCanvasKit) kUseLocalCanvasKitFlag: useLocalCanvasKit.toString(), }; } diff --git a/packages/flutter_tools/test/general.shard/build_info_test.dart b/packages/flutter_tools/test/general.shard/build_info_test.dart index aa53589470283..146e7963eefca 100644 --- a/packages/flutter_tools/test/general.shard/build_info_test.dart +++ b/packages/flutter_tools/test/general.shard/build_info_test.dart @@ -221,7 +221,7 @@ void main() { testWithoutContext('toBuildSystemEnvironment encoding of standard values', () { const buildInfo = BuildInfo( BuildMode.debug, - '', + 'strawberry', treeShakeIcons: true, trackWidgetCreation: true, dartDefines: ['foo=2', 'bar=2'], @@ -253,9 +253,21 @@ void main() { 'FileSystemScheme': 'scheme', 'BuildName': '122', 'BuildNumber': '22', + 'Flavor': 'strawberry', }); }); + testWithoutContext('toBuildSystemEnvironment omits Flavor when flavor is null', () { + const buildInfo = BuildInfo( + BuildMode.release, + null, + treeShakeIcons: false, + packageConfigPath: '.dart_tool/package_config.json', + ); + + expect(buildInfo.toBuildSystemEnvironment().containsKey('Flavor'), isFalse); + }); + testWithoutContext('toEnvironmentConfig encoding of standard values', () { const buildInfo = BuildInfo( BuildMode.debug, diff --git a/packages/flutter_tools/test/general.shard/build_system/targets/assets_test.dart b/packages/flutter_tools/test/general.shard/build_system/targets/assets_test.dart index 427cde4f21c0b..86a94716f9f0b 100644 --- a/packages/flutter_tools/test/general.shard/build_system/targets/assets_test.dart +++ b/packages/flutter_tools/test/general.shard/build_system/targets/assets_test.dart @@ -244,183 +244,66 @@ flutter: }, ); - group( - "Only compiles shaders with a flavor if the shaders' flavor matches the flavor in the environment", - () { - testUsingContext( - 'When the environment does not have a flavor defined', - () async { - // Re-create the environment with the correct process manager for this test. - environment = Environment.test( - fileSystem.currentDirectory, - processManager: globals.processManager, - artifacts: Artifacts.test(), - fileSystem: fileSystem, - logger: BufferLogger.test(), - platform: FakePlatform(), - defines: {kBuildMode: BuildMode.debug.cliName}, - ); - - final artifacts = Artifacts.test(); - final String impellercPath = artifacts.getHostArtifact(HostArtifact.impellerc).path; - fileSystem.file(impellercPath).createSync(recursive: true); - - fileSystem.file('pubspec.yaml') - ..createSync() - ..writeAsStringSync(''' -name: example -flutter: - shaders: - - shaders/common.frag - - path: shaders/premium.frag - flavors: - - premium -'''); - writePackageConfigFiles(directory: globals.fs.currentDirectory, mainLibName: 'example'); - - fileSystem.file('shaders/common.frag').createSync(recursive: true); - fileSystem.file('shaders/premium.frag').createSync(recursive: true); - - // Manually create expected output files since FakeProcessManager.any() won't create them - fileSystem - .file('${environment.buildDir.path}/flutter_assets/shaders/common.frag') - .createSync(recursive: true); - - await const CopyAssets().build(environment); - - expect( - fileSystem.file('${environment.buildDir.path}/flutter_assets/shaders/common.frag'), - exists, - ); - expect( - fileSystem.file('${environment.buildDir.path}/flutter_assets/shaders/premium.frag'), - isNot(exists), - ); - }, - overrides: { - FileSystem: () => fileSystem, - ProcessManager: () => FakeProcessManager.any(), - }, - ); - - testUsingContext( - 'When the environment has a matching flavor defined', - () async { - // Re-create the environment with the correct process manager for this test. - environment = Environment.test( - fileSystem.currentDirectory, - processManager: globals.processManager, - artifacts: Artifacts.test(), - fileSystem: fileSystem, - logger: BufferLogger.test(), - platform: FakePlatform(), - defines: {kBuildMode: BuildMode.debug.cliName}, - ); - - final artifacts = Artifacts.test(); - final String impellercPath = artifacts.getHostArtifact(HostArtifact.impellerc).path; - fileSystem.file(impellercPath).createSync(recursive: true); + group('CopyAssets compiles shorebird.yaml using the environment flavor', () { + const shorebirdYamlContents = ''' +app_id: base-app-id +flavors: + internal: internal-app-id + stable: stable-app-id +'''; - environment.defines[kFlavor] = 'premium'; - fileSystem.file('pubspec.yaml') - ..createSync() - ..writeAsStringSync(''' + void writeProjectWithShorebirdYaml() { + fileSystem.file('pubspec.yaml') + ..createSync() + ..writeAsStringSync(''' name: example flutter: - shaders: - - shaders/common.frag - - path: shaders/premium.frag - flavors: - - premium + assets: + - shorebird.yaml '''); - writePackageConfigFiles(directory: globals.fs.currentDirectory, mainLibName: 'example'); - - fileSystem.file('shaders/common.frag').createSync(recursive: true); - fileSystem.file('shaders/premium.frag').createSync(recursive: true); - - // Manually create expected output files since FakeProcessManager.any() won't create them - fileSystem - .file('${environment.buildDir.path}/flutter_assets/shaders/common.frag') - .createSync(recursive: true); - fileSystem - .file('${environment.buildDir.path}/flutter_assets/shaders/premium.frag') - .createSync(recursive: true); - - await const CopyAssets().build(environment); - - expect( - fileSystem.file('${environment.buildDir.path}/flutter_assets/shaders/common.frag'), - exists, - ); - expect( - fileSystem.file('${environment.buildDir.path}/flutter_assets/shaders/premium.frag'), - exists, - ); - }, - overrides: { - FileSystem: () => fileSystem, - ProcessManager: () => FakeProcessManager.any(), - }, - ); - - testUsingContext( - 'When the environment has a mismatching flavor defined', - () async { - // Re-create the environment with the correct process manager for this test. - environment = Environment.test( - fileSystem.currentDirectory, - processManager: globals.processManager, - artifacts: Artifacts.test(), - fileSystem: fileSystem, - logger: BufferLogger.test(), - platform: FakePlatform(), - defines: {kBuildMode: BuildMode.debug.cliName}, - ); + fileSystem.file('shorebird.yaml') + ..createSync() + ..writeAsStringSync(shorebirdYamlContents); + writePackageConfigFiles(directory: globals.fs.currentDirectory, mainLibName: 'example'); + } - final artifacts = Artifacts.test(); - final String impellercPath = artifacts.getHostArtifact(HostArtifact.impellerc).path; - fileSystem.file(impellercPath).createSync(recursive: true); + testUsingContext( + 'falls back to top-level app_id when no flavor is set', + () async { + writeProjectWithShorebirdYaml(); - environment.defines[kFlavor] = 'free'; // Mismatch - fileSystem.file('pubspec.yaml') - ..createSync() - ..writeAsStringSync(''' -name: example -flutter: - shaders: - - shaders/common.frag - - path: shaders/premium.frag - flavors: - - premium -'''); - writePackageConfigFiles(directory: globals.fs.currentDirectory, mainLibName: 'example'); + await const CopyAssets().build(environment); - fileSystem.file('shaders/common.frag').createSync(recursive: true); - fileSystem.file('shaders/premium.frag').createSync(recursive: true); + final File compiled = fileSystem.file( + '${environment.buildDir.path}/flutter_assets/shorebird.yaml', + ); + expect(compiled.readAsStringSync(), 'app_id: base-app-id'); + }, + overrides: { + FileSystem: () => fileSystem, + ProcessManager: () => FakeProcessManager.any(), + }, + ); - // Manually create expected output files for the ones that SHOULD exist - fileSystem - .file('${environment.buildDir.path}/flutter_assets/shaders/common.frag') - .createSync(recursive: true); + testUsingContext( + 'uses the flavor app_id when flavor is set', + () async { + environment.defines[kFlavor] = 'internal'; + writeProjectWithShorebirdYaml(); - await const CopyAssets().build(environment); + await const CopyAssets().build(environment); - expect( - fileSystem.file('${environment.buildDir.path}/flutter_assets/shaders/common.frag'), - exists, - ); - expect( - fileSystem.file('${environment.buildDir.path}/flutter_assets/shaders/premium.frag'), - isNot(exists), - ); - }, - overrides: { - FileSystem: () => fileSystem, - ProcessManager: () => FakeProcessManager.any(), - }, - ); - }, - ); + final File compiled = fileSystem.file( + '${environment.buildDir.path}/flutter_assets/shorebird.yaml', + ); + expect(compiled.readAsStringSync(), 'app_id: internal-app-id'); + }, + overrides: { + FileSystem: () => fileSystem, + ProcessManager: () => FakeProcessManager.any(), + }, + ); + }); testUsingContext( 'transforms assets declared with transformers', @@ -509,124 +392,6 @@ flutter: }, ); - testUsingContext( - 'transforms shaders declared with transformers before compilation', - () async { - Cache.flutterRoot = Cache.defaultFlutterRoot( - platform: globals.platform, - fileSystem: fileSystem, - userMessages: UserMessages(), - ); - - final environment = Environment.test( - fileSystem.currentDirectory, - processManager: globals.processManager, - artifacts: Artifacts.test(), - fileSystem: fileSystem, - logger: logger, - platform: globals.platform, - defines: {kBuildMode: BuildMode.debug.cliName}, - ); - - final artifacts = Artifacts.test(); - final String impellercPath = artifacts.getHostArtifact(HostArtifact.impellerc).path; - fileSystem.file(impellercPath).createSync(recursive: true); - - fileSystem.file('pubspec.yaml') - ..createSync() - ..writeAsStringSync(''' -name: example -flutter: - shaders: - - path: shaders/test.frag - transformers: - - package: my_capitalizer_transformer -'''); - - writePackageConfigFiles(directory: globals.fs.currentDirectory, mainLibName: 'example'); - - fileSystem.file('shaders/test.frag') - ..createSync(recursive: true) - ..writeAsStringSync('abc'); - - await const CopyAssets().build(environment); - - expect(logger.errorText, isEmpty); - expect(globals.processManager, hasNoRemainingExpectations); - expect( - fileSystem.file('${environment.buildDir.path}/flutter_assets/shaders/test.frag'), - exists, - ); - }, - overrides: { - Logger: () => logger, - FileSystem: () => fileSystem, - Platform: () => FakePlatform(), - ProcessManager: () { - final artifacts = Artifacts.test(); - - return FakeProcessManager.list([ - FakeCommand( - command: [ - artifacts.getArtifactPath(Artifact.engineDartBinary), - 'run', - 'my_capitalizer_transformer', - RegExp('--input=.*'), - RegExp('--output=.*'), - ], - onRun: (List args) { - final ArgResults parsedArgs = - (ArgParser() - ..addOption('input') - ..addOption('output')) - .parse(args); - - final File input = fileSystem.file(parsedArgs['input'] as String); - expect(input, exists); - expect(input.readAsStringSync(), 'abc'); - - fileSystem.file(parsedArgs['output']) - ..createSync(recursive: true) - ..writeAsStringSync('ABC'); - }, - ), - FakeCommand( - command: [ - RegExp(r'.*impellerc.*'), - // Match the 11 arguments generated by ShaderCompiler.compileShader. - ...List.filled(11, RegExp(r'.*')), - ], - onRun: (List args) { - String? inputPath; - String? outputPath; - String? outputSpirvPath; - for (final arg in args) { - if (arg.startsWith('--input=')) { - inputPath = arg.substring('--input='.length); - } else if (arg.startsWith('--sl=')) { - outputPath = arg.substring('--sl='.length); - } else if (arg.startsWith('--spirv=')) { - outputSpirvPath = arg.substring('--spirv='.length); - } - } - - expect(inputPath, isNotNull); - expect(outputPath, isNotNull); - expect(outputSpirvPath, isNotNull); - - final File input = fileSystem.file(inputPath); - expect(input, exists); - expect(input.readAsStringSync(), 'ABC'); - - fileSystem.file(outputPath).createSync(recursive: true); - fileSystem.file(outputSpirvPath).createSync(recursive: true); - }, - ), - ]); - }, - }, - ); - testUsingContext( 'exits tool if an asset transformation fails', () async { diff --git a/packages/flutter_tools/test/general.shard/build_system/targets/ios_test.dart b/packages/flutter_tools/test/general.shard/build_system/targets/ios_test.dart index bd5dc8e56291f..53a970fd53c51 100644 --- a/packages/flutter_tools/test/general.shard/build_system/targets/ios_test.dart +++ b/packages/flutter_tools/test/general.shard/build_system/targets/ios_test.dart @@ -767,6 +767,75 @@ void main() { }, ); + testUsingContext( + 'ReleaseIosApplicationBundle compiles shorebird.yaml using the flavor app_id', + () async { + environment.defines[kBuildMode] = 'release'; + environment.defines[kXcodeAction] = 'install'; + environment.defines[kFlavor] = 'internal'; + + fileSystem.file('pubspec.yaml') + ..createSync() + ..writeAsStringSync(''' +name: example +flutter: + assets: + - shorebird.yaml +'''); + + fileSystem.file('shorebird.yaml') + ..createSync() + ..writeAsStringSync(''' +app_id: base-app-id +flavors: + internal: internal-app-id + stable: stable-app-id +'''); + + writePackageConfigFiles(directory: fileSystem.currentDirectory, mainLibName: 'example'); + + fileSystem + .file(fileSystem.path.join('ios', 'Flutter', 'AppFrameworkInfo.plist')) + .createSync(recursive: true); + environment.buildDir + .childDirectory('App.framework') + .childFile('App') + .createSync(recursive: true); + environment.buildDir.childFile('native_assets.json').createSync(); + + final Directory frameworkDirectory = environment.outputDir.childDirectory('App.framework'); + final File frameworkDirectoryBinary = frameworkDirectory.childFile('App'); + final File infoPlist = frameworkDirectory.childFile('Info.plist'); + processManager.addCommands([ + createPlutilFakeCommand(infoPlist), + FakeCommand( + command: [ + 'xattr', + '-r', + '-d', + 'com.apple.FinderInfo', + frameworkDirectoryBinary.path, + ], + ), + FakeCommand( + command: ['codesign', '--force', '--sign', '-', frameworkDirectoryBinary.path], + ), + ]); + + await const ReleaseIosApplicationBundle().build(environment); + + final File compiledYaml = frameworkDirectory + .childDirectory('flutter_assets') + .childFile('shorebird.yaml'); + expect(compiledYaml.readAsStringSync(), 'app_id: internal-app-id'); + }, + overrides: { + FileSystem: () => fileSystem, + ProcessManager: () => processManager, + Platform: () => macPlatform, + }, + ); + testUsingContext( 'AotAssemblyRelease throws exception if asked to build for simulator', () async { From 12acc772842ad678e4ee2999f2eeae5ae9677ca5 Mon Sep 17 00:00:00 2001 From: Brandon DeRosier Date: Thu, 7 May 2026 16:13:47 -0700 Subject: [PATCH 51/95] chore: bump Updater to 90c349287d17 --- DEPS | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/DEPS b/DEPS index 56bf116f506b4..f4405cf06e071 100644 --- a/DEPS +++ b/DEPS @@ -20,7 +20,7 @@ vars = { "dart_sdk_revision": "b65ce89c8057d6880e00693a7b0ecd7b9e5f61ca", "dart_sdk_git": "git@github.com:shorebirdtech/dart-sdk.git", "updater_git": "https://github.com/shorebirdtech/updater.git", - "updater_rev": "34509fca3c65388ebc84cfaea8f38733ce41f41a", + "updater_rev": "90c349287d170991d1b527b7dac7be4fac508768", # WARNING: DO NOT EDIT canvaskit_cipd_instance MANUALLY # See `lib/web_ui/README.md` for how to roll CanvasKit to a new version. From 32bc89b3c37aed050e79234a73c36269a2e34984 Mon Sep 17 00:00:00 2001 From: Brandon DeRosier Date: Thu, 7 May 2026 16:18:41 -0700 Subject: [PATCH 52/95] fix: update build_rust_updater cbindgen inputs after #350 API split (#140) PR #138 (refactor: split shorebird C API consumption) switched shell/common/shorebird/updater.cc to the new updater_engine.h header, and the matching DEPS bump pulled the updater repo state where the combined library/cbindgen.toml has been replaced with separate library/cbindgen_dart.toml and library/cbindgen_engine.toml. The build_rust_updater action's declared inputs were not updated to match, so iOS engine builds on shorebird/dev fail with: ninja: error: '../../flutter/third_party/updater/library/cbindgen.toml', needed by 'gen/flutter/shell/common/shorebird/rust_updater_<...>.stamp', missing and no known rule to make it This action only runs in shorebird-runtime engine builds, which the periodic _build_engine workflow exercises but no per-PR CI job does, so the regression went unnoticed. Replace the missing input with the two split tomls. --- engine/src/flutter/shell/common/shorebird/BUILD.gn | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/engine/src/flutter/shell/common/shorebird/BUILD.gn b/engine/src/flutter/shell/common/shorebird/BUILD.gn index 22b11a614cad1..8c8034a24c301 100644 --- a/engine/src/flutter/shell/common/shorebird/BUILD.gn +++ b/engine/src/flutter/shell/common/shorebird/BUILD.gn @@ -86,7 +86,8 @@ if (shorebird_updater_supported) { "$shorebird_updater_dir/Cargo.lock", "$shorebird_updater_dir/library/Cargo.toml", "$shorebird_updater_dir/library/build.rs", - "$shorebird_updater_dir/library/cbindgen.toml", + "$shorebird_updater_dir/library/cbindgen_dart.toml", + "$shorebird_updater_dir/library/cbindgen_engine.toml", "$shorebird_updater_dir/library/.cargo/config.toml", ] inputs += shorebird_updater_rs_sources From 13c7d1a0b534a7afcc96716f334c31d00fb5cf75 Mon Sep 17 00:00:00 2001 From: Brandon DeRosier Date: Thu, 7 May 2026 16:48:30 -0700 Subject: [PATCH 53/95] fix: drop stale shorebird_check_for_update/shorebird_update from Windows .def (#142) --- .../src/flutter/shell/platform/windows/flutter_windows.dll.def | 2 -- 1 file changed, 2 deletions(-) diff --git a/engine/src/flutter/shell/platform/windows/flutter_windows.dll.def b/engine/src/flutter/shell/platform/windows/flutter_windows.dll.def index bcaa785977259..c1325428fa1d8 100644 --- a/engine/src/flutter/shell/platform/windows/flutter_windows.dll.def +++ b/engine/src/flutter/shell/platform/windows/flutter_windows.dll.def @@ -1,6 +1,5 @@ EXPORTS shorebird_check_for_downloadable_update = shorebird_check_for_downloadable_update - shorebird_check_for_update = shorebird_check_for_update shorebird_current_boot_patch_number = shorebird_current_boot_patch_number shorebird_free_string = shorebird_free_string shorebird_free_update_result = shorebird_free_update_result @@ -13,5 +12,4 @@ EXPORTS shorebird_report_launch_success = shorebird_report_launch_success shorebird_should_auto_update = shorebird_should_auto_update shorebird_start_update_thread = shorebird_start_update_thread - shorebird_update = shorebird_update shorebird_update_with_result = shorebird_update_with_result \ No newline at end of file From 7e195fffa1c509ff22467fed31b891593de68f06 Mon Sep 17 00:00:00 2001 From: Eric Seidel Date: Thu, 7 May 2026 19:21:17 -0700 Subject: [PATCH 54/95] ci(shorebird): drop iOS Rust targets from macOS android shard (#143) The android shard on macOS builds gen_snapshot for the host (the gn_args force host_cpu="x64"), so the only Rust targets it needs are the host darwin ones. The two iOS targets were copy-pasted from the old monolithic mac_setup.sh and never used by anything the android shard links. In addition to being dead work, building aarch64-apple-ios on the Sequoia runner trips an iOS SDK / deployment-target mismatch in cargo and fails the shard outright. Removing the unused targets keeps the android shard green and leaves the actual ios-release shards as the single place that needs the SDK fix (handled separately in _build_engine). --- shorebird/ci/shards/macos.json | 2 -- 1 file changed, 2 deletions(-) diff --git a/shorebird/ci/shards/macos.json b/shorebird/ci/shards/macos.json index db68dc3e1a2f8..d27d7d8a434d0 100644 --- a/shorebird/ci/shards/macos.json +++ b/shorebird/ci/shards/macos.json @@ -4,8 +4,6 @@ { "type": "rust", "targets": [ - "aarch64-apple-ios", - "x86_64-apple-ios", "aarch64-apple-darwin", "x86_64-apple-darwin" ] From 2f10b812bc234ad39f959284ee26556ca6fbbd13 Mon Sep 17 00:00:00 2001 From: Eric Seidel Date: Thu, 7 May 2026 21:25:17 -0700 Subject: [PATCH 55/95] ci(shorebird): stream child stdout/stderr from shard_runner (#144) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * ci(shorebird): stream child stdout/stderr from shard_runner `runChecked` used `Process.run`, which buffers stdout/stderr in memory and prints nothing until the process exits โ€” and on success, prints nothing at all. That hid 30+ minutes of ninja, cargo, and gclient output per shard from the GitHub Actions log; only the [Step] start / [Step] complete prints from the runner itself made it through. The legacy bash build scripts streamed naturally, so this is a regression the sharded runner introduced. Switch `runChecked` to `Process.start` with inheritStdio so output goes live to the parent's stdio. The error path no longer needs to echo captured output (it already streamed) โ€” the exception just notes the exit code and description. Add `runCapturingStdout` for the one place that actually parses child output (`gsutil ls` in lib/gcs.dart). It captures stdout while still streaming stderr, so failures still appear in the log live. Tests: 32/32 pass; dart analyze clean. * fix: dart format gcs.dart --- shorebird/ci/shard_runner/lib/gcs.dart | 11 ++-- shorebird/ci/shard_runner/lib/process.dart | 71 ++++++++++++++++------ 2 files changed, 59 insertions(+), 23 deletions(-) diff --git a/shorebird/ci/shard_runner/lib/gcs.dart b/shorebird/ci/shard_runner/lib/gcs.dart index c4263658bc275..18c622c1481e3 100644 --- a/shorebird/ci/shard_runner/lib/gcs.dart +++ b/shorebird/ci/shard_runner/lib/gcs.dart @@ -72,17 +72,16 @@ Future downloadFromStaging({ print('[GCS] Downloading from $stagingRoot'); - // List files in the staging location - final ProcessResult lsResult = await runChecked( + // List files in the staging location. We capture stdout because we need + // to parse the file list; everything else uses runChecked to stream. + final String lsOutput = await runCapturingStdout( 'gsutil', ['ls', stagingRoot], description: 'gsutil ls $stagingRoot', ); - final List files = (lsResult.stdout as String) - .split('\n') - .where((String f) => f.endsWith('.tar.gz')) - .toList(); + final List files = + lsOutput.split('\n').where((String f) => f.endsWith('.tar.gz')).toList(); for (final String file in files) { final String fileName = p.basename(file); diff --git a/shorebird/ci/shard_runner/lib/process.dart b/shorebird/ci/shard_runner/lib/process.dart index 1fb251b609d97..5c3a0a6adaf22 100644 --- a/shorebird/ci/shard_runner/lib/process.dart +++ b/shorebird/ci/shard_runner/lib/process.dart @@ -1,3 +1,5 @@ +import 'dart:async'; +import 'dart:convert'; import 'dart:io'; /// Resolves an executable name for Windows, appending .cmd if needed. @@ -28,10 +30,13 @@ String _resolveExecutable(String executable) { return executable; } -/// Runs a process and throws if it exits with a non-zero code. +/// Runs a process, streaming stdout/stderr to the parent's stdio so the +/// caller (and CI logs) see output live. Throws if the process exits +/// non-zero. /// -/// Returns the [ProcessResult] for callers that need stdout/stderr. -Future runChecked( +/// Use this for everything except the rare case where you need to parse +/// the child's stdout โ€” for that, see [runCapturingStdout]. +Future runChecked( String executable, List arguments, { String? workingDirectory, @@ -40,27 +45,59 @@ Future runChecked( }) async { final String resolvedExecutable = _resolveExecutable(executable); - final ProcessResult result = await Process.run( + final Process process = await Process.start( resolvedExecutable, arguments, workingDirectory: workingDirectory, environment: environment, + mode: ProcessStartMode.inheritStdio, ); - if (result.exitCode != 0) { + final int exitCode = await process.exitCode; + if (exitCode != 0) { final String desc = description ?? '$executable ${arguments.join(' ')}'; - final String stderr = (result.stderr as String).trim(); - final String stdout = (result.stdout as String).trim(); - final StringBuffer message = - StringBuffer('$desc failed (exit ${result.exitCode})'); - if (stdout.isNotEmpty) { - message.write('\nSTDOUT: $stdout'); - } - if (stderr.isNotEmpty) { - message.write('\nSTDERR: $stderr'); - } - throw Exception(message.toString()); + throw Exception('$desc failed (exit $exitCode)'); } +} + +/// Runs a process, capturing stdout into the returned string while still +/// streaming stderr to the parent's stderr. Throws if the process exits +/// non-zero. +/// +/// Use this when you need to parse the child's stdout (e.g. `gsutil ls`). +/// For everything else use [runChecked] so output reaches the CI log live. +Future runCapturingStdout( + String executable, + List arguments, { + String? workingDirectory, + Map? environment, + String? description, +}) async { + final String resolvedExecutable = _resolveExecutable(executable); - return result; + final Process process = await Process.start( + resolvedExecutable, + arguments, + workingDirectory: workingDirectory, + environment: environment, + ); + + final Future stdoutFuture = + process.stdout.transform(utf8.decoder).join(); + final Future stderrFuture = + process.stderr.transform(utf8.decoder).forEach(stderr.write); + + final List results = await Future.wait(>[ + stdoutFuture, + stderrFuture, + process.exitCode, + ]); + final String capturedStdout = results[0] as String; + final int exitCode = results[2] as int; + + if (exitCode != 0) { + final String desc = description ?? '$executable ${arguments.join(' ')}'; + throw Exception('$desc failed (exit $exitCode)'); + } + return capturedStdout; } From efaa7a99ea9e618355b0d78f880e795564086505 Mon Sep 17 00:00:00 2001 From: Eric Seidel Date: Fri, 8 May 2026 05:38:30 -0700 Subject: [PATCH 56/95] ci(shorebird): remove shard_runner package; now lives in _build_engine (#145) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The shard_runner package moved to shorebirdtech/_build_engine in shorebirdtech/_build_engine#209. Runner code lives next to the workflows that invoke it; runner changes no longer need a Flutter PR + engine bump round-trip to land + test. What stays in this repo (still read by the runner via --config-dir): - shorebird/ci/shards/*.json - shorebird/ci/compose.json - shorebird/ci/artifacts_manifest.template.yaml Also drops the ๐Ÿงฉ Shard Runner Tests job from shorebird_ci.yml and the matching entry from the `ci` aggregator's needs list โ€” those tests now run in _build_engine's CI. Depends on shorebirdtech/_build_engine#209 landing first; otherwise in-flight engine builds against pre-bump SHAs would be unaffected (workflow points at _build_engine's runner) but anyone iterating locally on the runner would be confused by the deleted dir. --- .github/workflows/shorebird_ci.yml | 29 -- shorebird/ci/shard_runner/.gitignore | 2 - .../ci/shard_runner/analysis_options.yaml | 7 - .../ci/shard_runner/bin/compare_buckets.dart | 158 ------- shorebird/ci/shard_runner/bin/compose.dart | 104 ----- shorebird/ci/shard_runner/bin/finalize.dart | 206 ---------- shorebird/ci/shard_runner/bin/run_shard.dart | 84 ---- shorebird/ci/shard_runner/lib/cli.dart | 83 ---- .../ci/shard_runner/lib/compose_config.dart | 76 ---- shorebird/ci/shard_runner/lib/config.dart | 268 ------------ shorebird/ci/shard_runner/lib/gcs.dart | 111 ----- shorebird/ci/shard_runner/lib/manifest.dart | 26 -- shorebird/ci/shard_runner/lib/process.dart | 103 ----- shorebird/ci/shard_runner/pubspec.lock | 389 ------------------ shorebird/ci/shard_runner/pubspec.yaml | 17 - .../test/compose_config_test.dart | 119 ------ .../ci/shard_runner/test/config_test.dart | 264 ------------ .../ci/shard_runner/test/manifest_test.dart | 208 ---------- 18 files changed, 2254 deletions(-) delete mode 100644 shorebird/ci/shard_runner/.gitignore delete mode 100644 shorebird/ci/shard_runner/analysis_options.yaml delete mode 100644 shorebird/ci/shard_runner/bin/compare_buckets.dart delete mode 100644 shorebird/ci/shard_runner/bin/compose.dart delete mode 100644 shorebird/ci/shard_runner/bin/finalize.dart delete mode 100644 shorebird/ci/shard_runner/bin/run_shard.dart delete mode 100644 shorebird/ci/shard_runner/lib/cli.dart delete mode 100644 shorebird/ci/shard_runner/lib/compose_config.dart delete mode 100644 shorebird/ci/shard_runner/lib/config.dart delete mode 100644 shorebird/ci/shard_runner/lib/gcs.dart delete mode 100644 shorebird/ci/shard_runner/lib/manifest.dart delete mode 100644 shorebird/ci/shard_runner/lib/process.dart delete mode 100644 shorebird/ci/shard_runner/pubspec.lock delete mode 100644 shorebird/ci/shard_runner/pubspec.yaml delete mode 100644 shorebird/ci/shard_runner/test/compose_config_test.dart delete mode 100644 shorebird/ci/shard_runner/test/config_test.dart delete mode 100644 shorebird/ci/shard_runner/test/manifest_test.dart diff --git a/.github/workflows/shorebird_ci.yml b/.github/workflows/shorebird_ci.yml index 7f03b472b088c..2d2905405521d 100644 --- a/.github/workflows/shorebird_ci.yml +++ b/.github/workflows/shorebird_ci.yml @@ -38,34 +38,6 @@ jobs: run: ../../bin/flutter test test/general.shard working-directory: packages/flutter_tools - shard-runner-tests: - runs-on: ubuntu-latest - - name: ๐Ÿงฉ Shard Runner Tests - - defaults: - run: - working-directory: shorebird/ci/shard_runner - - steps: - - name: ๐Ÿ“š Git Checkout - uses: actions/checkout@v4 - - - name: ๐ŸŽฏ Setup Dart - uses: dart-lang/setup-dart@v1 - - - name: ๐Ÿ“ฆ Install Dependencies - run: dart pub get - - - name: ๐Ÿ” Check Formatting - run: dart format --output=none --set-exit-if-changed . - - - name: ๐Ÿ”Ž Analyze - run: dart analyze --fatal-infos - - - name: ๐Ÿงช Run Tests - run: dart test - shorebird-android-tests: # Android tests run on Ubuntu (faster runners, no macOS needed) runs-on: ubuntu-latest @@ -148,7 +120,6 @@ jobs: if: always() needs: - flutter-tools-tests - - shard-runner-tests - shorebird-android-tests - shorebird-ios-tests runs-on: ubuntu-latest diff --git a/shorebird/ci/shard_runner/.gitignore b/shorebird/ci/shard_runner/.gitignore deleted file mode 100644 index 929f3d7e632c0..0000000000000 --- a/shorebird/ci/shard_runner/.gitignore +++ /dev/null @@ -1,2 +0,0 @@ -# Override root .gitignore to track our lockfile -!pubspec.lock diff --git a/shorebird/ci/shard_runner/analysis_options.yaml b/shorebird/ci/shard_runner/analysis_options.yaml deleted file mode 100644 index d04adaf90938a..0000000000000 --- a/shorebird/ci/shard_runner/analysis_options.yaml +++ /dev/null @@ -1,7 +0,0 @@ -include: package:lints/recommended.yaml - -analyzer: - language: - strict-casts: true - strict-inference: true - strict-raw-types: true diff --git a/shorebird/ci/shard_runner/bin/compare_buckets.dart b/shorebird/ci/shard_runner/bin/compare_buckets.dart deleted file mode 100644 index 958c238bb0230..0000000000000 --- a/shorebird/ci/shard_runner/bin/compare_buckets.dart +++ /dev/null @@ -1,158 +0,0 @@ -import 'dart:io'; - -import 'package:args/args.dart'; -import 'package:path/path.dart' as p; -import 'package:shard_runner/config.dart'; - -/// Compares artifacts between two GCS buckets for a given engine revision. -/// -/// Usage: dart run shard_runner:compare_buckets [options] -/// -/// Example: -/// dart run shard_runner:compare_buckets \ -/// --engine-revision abc123 \ -/// --test-bucket shorebird-build-test \ -/// --production-bucket download.shorebird.dev -Future main(List args) async { - final ArgParser parser = ArgParser() - ..addOption('engine-revision', - abbr: 'r', help: 'Engine revision (git hash)', mandatory: true) - ..addOption('test-bucket', - abbr: 't', help: 'Test bucket to compare', mandatory: true) - ..addOption('production-bucket', - abbr: 'p', - help: 'Production bucket (default: download.shorebird.dev)', - defaultsTo: 'download.shorebird.dev') - ..addOption('config-dir', - abbr: 'c', help: 'Config directory containing shards/*.json') - ..addFlag('verbose', abbr: 'v', help: 'Show detailed output') - ..addFlag('help', abbr: 'h', negatable: false, help: 'Show this help'); - - final ArgResults results = parser.parse(args); - - if (results['help'] as bool) { - print('Usage: dart run shard_runner:compare_buckets [options]'); - print(''); - print('Compares artifacts between test and production GCS buckets.'); - print('Uses gsutil hash to compare file checksums (MD5 + CRC32C).'); - print(''); - print(parser.usage); - exit(0); - } - - final String engineRevision = results['engine-revision'] as String; - final String testBucket = results['test-bucket'] as String; - final String productionBucket = results['production-bucket'] as String; - final String? configDirPath = results['config-dir'] as String?; - final bool verbose = results['verbose'] as bool; - - // Find config directory - final String configDir = configDirPath ?? - p.join(p.dirname(p.dirname(Platform.script.toFilePath())), 'shards'); - - print('=' * 60); - print('Compare Buckets'); - print('=' * 60); - print('Engine revision: $engineRevision'); - print('Test bucket: $testBucket'); - print('Production bucket: $productionBucket'); - print('Config dir: $configDir'); - print(''); - - // Load configs to get artifact paths - const List platforms = ['linux', 'macos', 'windows']; - final Map configs = {}; - for (final String platform in platforms) { - configs[platform] = PlatformConfig.load(platform, configDir); - } - - // Collect all artifact paths - final List artifacts = []; - for (final PlatformConfig config in configs.values) { - for (final ShardDef shard in config.shards.values) { - for (final ArtifactDef artifact in shard.artifacts) { - final String dstPath = - artifact.dst.replaceAll(r'$engine', engineRevision); - artifacts.add(dstPath); - } - } - } - - // Add manifest - artifacts.add('shorebird/$engineRevision/artifacts_manifest.yaml'); - - print('Comparing ${artifacts.length} artifacts...\n'); - - int matches = 0; - int mismatches = 0; - int missing = 0; - - for (final String artifact in artifacts) { - final String testUri = 'gs://$testBucket/$artifact'; - final String prodUri = 'gs://$productionBucket/$artifact'; - - // Get hash from test bucket - final String? testHash = await _getHash(testUri); - if (testHash == null) { - if (verbose) print('[MISSING] $artifact (not in test bucket)'); - missing++; - continue; - } - - // Get hash from production bucket - final String? prodHash = await _getHash(prodUri); - if (prodHash == null) { - if (verbose) print('[MISSING] $artifact (not in production bucket)'); - missing++; - continue; - } - - // Compare hashes - if (testHash == prodHash) { - if (verbose) print('[OK] $artifact'); - matches++; - } else { - print('[MISMATCH] $artifact'); - print(' Test: $testHash'); - print(' Prod: $prodHash'); - mismatches++; - } - } - - print(''); - print('=' * 60); - print('Results:'); - print(' Matches: $matches'); - print(' Mismatches: $mismatches'); - print(' Missing: $missing'); - print('=' * 60); - - if (mismatches > 0) { - print('\nWARNING: Found $mismatches mismatched artifacts!'); - exit(1); - } else if (missing > 0) { - print('\nWARNING: Found $missing missing artifacts.'); - exit(2); - } else { - print('\nSUCCESS: All artifacts match!'); - } -} - -/// Gets the MD5 hash of a GCS object using gsutil hash. -Future _getHash(String uri) async { - final ProcessResult result = - await Process.run('gsutil', ['hash', uri]); - if (result.exitCode != 0) { - return null; - } - - // Parse output for MD5 hash - // Example output: - // Hashes [hex] for gs://bucket/path: - // Hash (crc32c): abc123== - // Hash (md5): xyz789== - final String output = result.stdout as String; - final RegExpMatch? md5Match = - RegExp(r'Hash \(md5\):\s+(\S+)').firstMatch(output); - return md5Match?.group(1); -} diff --git a/shorebird/ci/shard_runner/bin/compose.dart b/shorebird/ci/shard_runner/bin/compose.dart deleted file mode 100644 index 73cdb44393bb7..0000000000000 --- a/shorebird/ci/shard_runner/bin/compose.dart +++ /dev/null @@ -1,104 +0,0 @@ -import 'dart:io'; - -import 'package:args/args.dart'; -import 'package:path/path.dart' as p; -import 'package:shard_runner/cli.dart'; -import 'package:shard_runner/compose_config.dart'; -import 'package:shard_runner/gcs.dart'; -import 'package:shard_runner/process.dart'; - -/// Composes artifacts from multiple shards into final outputs. -/// -/// Usage: dart run shard_runner:compose [options] -/// -/// Example: -/// dart run shard_runner:compose ios-framework --engine-src ~/.engine_checkout/engine/src -Future main(List args) async { - final ArgParser parser = ArgParser(); - CliConfig.addCommonOptions(parser, includeUpload: false); - parser.addFlag('download', - defaultsTo: true, help: 'Download artifacts from GCS staging'); - - final ArgResults results = parser.parse(args); - - if (results['help'] as bool || results.rest.isEmpty) { - print('Usage: dart run shard_runner:compose [options]'); - print(''); - print('Compose names: ios-framework, macos-framework, macos-gen-snapshot'); - print(''); - print(parser.usage); - exit(results['help'] as bool ? 0 : 1); - } - - final String composeName = results.rest[0]; - final CliConfig cli = - CliConfig.fromArgs(results, scriptPath: Platform.script.toFilePath()); - final bool shouldDownload = results['download'] as bool; - - cli.printHeader('Compose Runner', { - 'Compose:': composeName, - 'Download:': shouldDownload.toString(), - }); - - // Load compose config - final ComposeConfig config; - try { - config = ComposeConfig.load(cli.configDir); - } on FileSystemException catch (e) { - print('Error: ${e.message} at ${e.path}'); - exit(1); - } - - final ComposeDef composeDef; - try { - composeDef = config.getCompose(composeName); - } on ArgumentError catch (e) { - print('Error: ${e.message}'); - exit(1); - } - - print('\n[Compose] Requires shards: ${composeDef.requires.join(', ')}'); - - // Download artifacts from each required shard - if (shouldDownload) { - for (final String shard in composeDef.requires) { - print('\n[Download] Fetching $shard artifacts...'); - await downloadFromStaging( - runId: cli.runId, - platform: 'macos', // Compose only runs for macOS currently - shard: shard, - destDir: p.join(cli.engineSrc, 'out'), - ); - } - } - - // Build script arguments - final String outDir = p.join(cli.engineSrc, 'out', 'release'); - - // Ensure output directory exists - Directory(outDir).createSync(recursive: true); - - // Build script arguments: expand path_args to absolute paths, pass flags as-is. - final List expandedArgs = ['--dst', outDir]; - for (final MapEntry entry in composeDef.pathArgs.entries) { - expandedArgs - .addAll([entry.key, p.join(cli.engineSrc, 'out', entry.value)]); - } - expandedArgs.addAll(composeDef.flags); - - // Run the composition script - print('\n[Compose] Running ${composeDef.script}...'); - print('[Compose] Args: ${expandedArgs.join(' ')}'); - - await runChecked( - 'python3', - [p.join(cli.engineSrc, composeDef.script), ...expandedArgs], - workingDirectory: cli.engineSrc, - description: 'Compose ($composeName)', - ); - - print('[Compose] Complete'); - print('\n${'='.padRight(60, '=')}'); - print('Compose $composeName completed successfully'); - print('='.padRight(60, '=')); -} diff --git a/shorebird/ci/shard_runner/bin/finalize.dart b/shorebird/ci/shard_runner/bin/finalize.dart deleted file mode 100644 index 6534aedec8162..0000000000000 --- a/shorebird/ci/shard_runner/bin/finalize.dart +++ /dev/null @@ -1,206 +0,0 @@ -import 'dart:io'; - -import 'package:args/args.dart'; -import 'package:path/path.dart' as p; -import 'package:shard_runner/cli.dart'; -import 'package:shard_runner/config.dart'; -import 'package:shard_runner/gcs.dart'; -import 'package:shard_runner/manifest.dart'; -import 'package:shard_runner/process.dart'; - -/// Finalizes a sharded build by generating manifest and uploading artifacts. -/// -/// Usage: dart run shard_runner:finalize [options] -/// -/// Example: -/// dart run shard_runner:finalize --engine-revision abc123 -Future main(List args) async { - final ArgParser parser = ArgParser() - ..addOption('engine-revision', - abbr: 'r', help: 'Engine revision (git hash)', mandatory: true) - ..addOption('base-engine-revision', - help: 'Base Flutter engine revision for manifest') - ..addOption('content-hash', help: 'Content-aware hash for Dart SDK') - ..addOption('bucket', - abbr: 'b', - help: 'GCS bucket for uploads (default: download.shorebird.dev)', - defaultsTo: 'download.shorebird.dev') - ..addFlag('download', - defaultsTo: true, help: 'Download artifacts from GCS staging') - ..addFlag('upload', - defaultsTo: true, help: 'Upload artifacts to GCS bucket'); - - CliConfig.addCommonOptions(parser, includeUpload: false); - - final ArgResults results = parser.parse(args); - - if (results['help'] as bool) { - print('Usage: dart run shard_runner:finalize [options]'); - print(''); - print(parser.usage); - exit(0); - } - - final CliConfig cli = - CliConfig.fromArgs(results, scriptPath: Platform.script.toFilePath()); - final String engineRevision = results['engine-revision'] as String; - final String baseEngineRevision = - results['base-engine-revision'] as String? ?? engineRevision; - final String? contentHash = results['content-hash'] as String?; - final String bucket = results['bucket'] as String; - final bool shouldDownload = results['download'] as bool; - final bool shouldUpload = results['upload'] as bool; - - cli.printHeader('Finalize Build', { - 'Engine:': engineRevision, - 'Base Engine:': baseEngineRevision, - 'Bucket:': bucket, - 'Download:': shouldDownload.toString(), - 'Upload:': shouldUpload.toString(), - }); - - // Load shard configs for each platform - const List platforms = ['linux', 'macos', 'windows']; - final Map configs = {}; - for (final String platform in platforms) { - configs[platform] = PlatformConfig.load(platform, cli.configDir); - } - - // Download all artifacts from staging - if (shouldDownload) { - final String outDir = p.join(cli.engineSrc, 'out'); - Directory(outDir).createSync(recursive: true); - - for (final String platform in platforms) { - final PlatformConfig config = configs[platform]!; - for (final String shardName in config.shards.keys) { - print('\n[Download] Fetching $platform/$shardName...'); - await downloadFromStaging( - runId: cli.runId, - platform: platform, - shard: shardName, - destDir: outDir, - ); - } - } - } - - // Generate manifest - print('\n[Manifest] Generating artifacts_manifest.yaml...'); - final String manifest = - generateManifest(baseEngineRevision, configDir: cli.configDir); - final File manifestFile = - File(p.join(cli.engineSrc, 'artifacts_manifest.yaml')); - manifestFile.writeAsStringSync(manifest); - print('[Manifest] Written to ${manifestFile.path}'); - - // Upload to production - if (shouldUpload) { - print('\n[Upload] Uploading to $bucket...'); - await uploadToProduction( - engineSrc: cli.engineSrc, - engineRevision: engineRevision, - contentHash: contentHash, - configs: configs, - bucket: bucket, - ); - } - - print('\n${'=' * 60}'); - print('Finalize completed successfully'); - print('=' * 60); -} - -/// Production storage bucket name (without gs:// prefix). -const String productionBucket = 'download.shorebird.dev'; - -/// Uploads artifacts to a GCS bucket based on config definitions. -Future uploadToProduction({ - required String engineSrc, - required String engineRevision, - required String? contentHash, - required Map configs, - required String bucket, -}) async { - final String outDir = p.join(engineSrc, 'out'); - final String bucketUri = 'gs://$bucket'; - - // Helper to run gsutil cp - Future gscp(String src, String dest) async { - print('[Upload] $src -> $dest'); - await runChecked('gsutil', ['cp', src, dest], - description: 'gsutil cp $src'); - } - - // Helper to zip a directory and upload - Future zipAndUpload(String srcPath, String dest) async { - final String tempZip = '$srcPath.zip'; - print('[Zip] Creating $tempZip...'); - await runChecked( - 'zip', - ['-r', tempZip, '.'], - workingDirectory: srcPath, - description: 'zip $tempZip', - ); - await gscp(tempZip, dest); - File(tempZip).deleteSync(); - } - - // Process artifacts from all configs - for (final MapEntry entry in configs.entries) { - final String platform = entry.key; - final PlatformConfig config = entry.value; - - for (final MapEntry shardEntry in config.shards.entries) { - final String shardName = shardEntry.key; - final ShardDef shard = shardEntry.value; - - print('\n[Upload] Processing $platform/$shardName...'); - - for (final ArtifactDef artifact in shard.artifacts) { - // Resolve source path - final String srcPath = p.join(outDir, artifact.src); - - // Resolve destination path (replace $engine with actual revision) - final String dstPath = - artifact.dst.replaceAll(r'$engine', engineRevision); - final String fullDest = '$bucketUri/$dstPath'; - - // Check if source exists - final File srcFile = File(srcPath); - final Directory srcDir = Directory(srcPath); - final bool srcExists = srcFile.existsSync() || srcDir.existsSync(); - - if (!srcExists) { - print('[Skip] $srcPath (not found)'); - continue; - } - - // Handle zip flag - if (artifact.zip && srcDir.existsSync()) { - await zipAndUpload(srcPath, fullDest); - } else { - await gscp(srcPath, fullDest); - } - - // Handle content-hash uploads (for Dart SDK) - if (artifact.contentHash && contentHash != null) { - final String contentDstPath = - artifact.dst.replaceAll(r'$engine', contentHash); - final String contentFullDest = '$bucketUri/$contentDstPath'; - await gscp(srcPath, contentFullDest); - } - } - } - } - - // Upload manifest - final String manifestFile = p.join(engineSrc, 'artifacts_manifest.yaml'); - if (File(manifestFile).existsSync()) { - final String manifestDest = - '$bucketUri/shorebird/$engineRevision/artifacts_manifest.yaml'; - await gscp(manifestFile, manifestDest); - } - - print('\n[Upload] Production upload complete'); -} diff --git a/shorebird/ci/shard_runner/bin/run_shard.dart b/shorebird/ci/shard_runner/bin/run_shard.dart deleted file mode 100644 index ed8c4cddac435..0000000000000 --- a/shorebird/ci/shard_runner/bin/run_shard.dart +++ /dev/null @@ -1,84 +0,0 @@ -import 'dart:io'; - -import 'package:args/args.dart'; -import 'package:shard_runner/cli.dart'; -import 'package:shard_runner/config.dart'; -import 'package:shard_runner/gcs.dart'; - -/// Runs a single build shard. -/// -/// Usage: dart run shard_runner:run_shard [options] -/// -/// Example: -/// dart run shard_runner:run_shard linux android-arm64 --engine-src ~/.engine_checkout/engine/src -Future main(List args) async { - final ArgParser parser = ArgParser(); - CliConfig.addCommonOptions(parser); - - final ArgResults results = parser.parse(args); - - if (results['help'] as bool || results.rest.length < 2) { - print( - 'Usage: dart run shard_runner:run_shard [options]'); - print(''); - print('Platforms: linux, macos, windows'); - print(''); - print(parser.usage); - exit(results['help'] as bool ? 0 : 1); - } - - final String platform = results.rest[0]; - final String shard = results.rest[1]; - final CliConfig cli = - CliConfig.fromArgs(results, scriptPath: Platform.script.toFilePath()); - - cli.printHeader('Shard Runner', { - 'Platform:': platform, - 'Shard:': shard, - 'Upload:': cli.shouldUpload.toString(), - }); - - cli.verifyEngineSrc(); - - // Load config - print('\n[Config] Loading $platform.json...'); - final PlatformConfig config = PlatformConfig.load(platform, cli.configDir); - final ShardDef shardDef = config.getShard(shard); - - print('[Config] Found ${shardDef.steps.length} step(s)'); - - // Collect output directories from GnNinja steps for upload - final List outDirs = [ - for (final BuildStep step in shardDef.steps) - if (step is GnNinjaStep) step.outDir, - ]; - - // Execute steps - final Stopwatch stopwatch = Stopwatch()..start(); - - for (int i = 0; i < shardDef.steps.length; i++) { - final BuildStep step = shardDef.steps[i]; - print( - '\n[${'Step ${i + 1}/${shardDef.steps.length}'}] ${step.runtimeType}'); - - await step.execute(cli.engineSrc); - } - - stopwatch.stop(); - print('\n[Build] Complete in ${stopwatch.elapsed}'); - - // Upload to GCS staging - if (cli.shouldUpload && outDirs.isNotEmpty) { - await uploadToStaging( - runId: cli.runId, - platform: platform, - shard: shard, - engineSrc: cli.engineSrc, - outDirs: outDirs, - ); - } - - print('\n${'='.padRight(60, '=')}'); - print('Shard $platform/$shard completed successfully'); - print('='.padRight(60, '=')); -} diff --git a/shorebird/ci/shard_runner/lib/cli.dart b/shorebird/ci/shard_runner/lib/cli.dart deleted file mode 100644 index 23a2a0b99a5a3..0000000000000 --- a/shorebird/ci/shard_runner/lib/cli.dart +++ /dev/null @@ -1,83 +0,0 @@ -import 'dart:io'; - -import 'package:args/args.dart'; -import 'package:meta/meta.dart'; -import 'package:path/path.dart' as p; - -/// Common CLI configuration used by shard runner scripts. -@immutable -class CliConfig { - CliConfig({ - required this.engineSrc, - required this.configDir, - required this.runId, - required this.shouldUpload, - }); - - /// Parses common options from ArgResults. - /// - /// [scriptPath] should be Platform.script.toFilePath() from the calling script. - factory CliConfig.fromArgs(ArgResults results, {required String scriptPath}) { - final String engineSrc = p.canonicalize(results['engine-src'] as String); - - // Config directory defaults to shorebird/ci (grandparent of bin/*.dart) - final String configDir = results['config-dir'] as String? ?? - p.dirname(p.dirname(p.dirname(scriptPath))); - - final String runId = results['run-id'] as String; - - final bool shouldUpload = - !results.options.contains('upload') || results['upload'] as bool; - - return CliConfig( - engineSrc: engineSrc, - configDir: configDir, - runId: runId, - shouldUpload: shouldUpload, - ); - } - final String engineSrc; - final String configDir; - final String runId; - final bool shouldUpload; - - /// Creates common argument parser options. - static void addCommonOptions(ArgParser parser, {bool includeUpload = true}) { - parser - ..addOption('engine-src', - abbr: 'e', help: 'Path to engine/src directory', mandatory: true) - ..addOption('run-id', - help: 'Build run identifier (use "local" for local development)', - mandatory: true) - ..addOption('config-dir', - abbr: 'c', help: 'Path to config directory (shorebird/ci)') - ..addFlag('help', abbr: 'h', negatable: false, help: 'Show this help'); - - if (includeUpload) { - parser.addFlag('upload', - defaultsTo: true, help: 'Upload artifacts to GCS staging'); - } - } - - /// Prints a standard header with configuration info. - void printHeader(String title, Map extra) { - print('='.padRight(60, '=')); - print(title); - print('='.padRight(60, '=')); - print('Engine: $engineSrc'); - print('Config: $configDir'); - print('Run ID: $runId'); - for (final MapEntry entry in extra.entries) { - print('${entry.key.padRight(12)}${entry.value}'); - } - print('='.padRight(60, '=')); - } - - /// Verifies that the engine source directory exists. - void verifyEngineSrc() { - if (!Directory(engineSrc).existsSync()) { - print('Error: Engine source not found at $engineSrc'); - exit(1); - } - } -} diff --git a/shorebird/ci/shard_runner/lib/compose_config.dart b/shorebird/ci/shard_runner/lib/compose_config.dart deleted file mode 100644 index 927748d326484..0000000000000 --- a/shorebird/ci/shard_runner/lib/compose_config.dart +++ /dev/null @@ -1,76 +0,0 @@ -import 'dart:convert'; -import 'dart:io'; - -import 'package:meta/meta.dart'; -import 'package:path/path.dart' as p; - -/// Configuration for all compose operations. -@immutable -class ComposeConfig { - ComposeConfig({required this.composes}); - - factory ComposeConfig.fromJson(Map json) { - return ComposeConfig( - composes: json.map( - (String key, value) => - MapEntry(key, ComposeDef.fromJson(value as Map)), - ), - ); - } - final Map composes; - - static ComposeConfig load(String configDir) { - final File file = File(p.join(configDir, 'compose.json')); - if (!file.existsSync()) { - throw FileSystemException('compose.json not found', file.path); - } - final String content = file.readAsStringSync(); - final Map json = - jsonDecode(content) as Map; - return ComposeConfig.fromJson(json); - } - - ComposeDef getCompose(String name) { - final ComposeDef? compose = composes[name]; - if (compose == null) { - throw ArgumentError( - 'Unknown compose: $name. Available: ${composes.keys.join(', ')}'); - } - return compose; - } -} - -/// Definition of a single compose operation. -@immutable -class ComposeDef { - ComposeDef({ - required this.requires, - required this.script, - this.flags = const [], - this.pathArgs = const {}, - }); - - factory ComposeDef.fromJson(Map json) { - return ComposeDef( - requires: (json['requires'] as List).cast(), - script: json['script'] as String, - flags: (json['flags'] as List?)?.cast() ?? [], - pathArgs: (json['path_args'] as Map?) - ?.cast() ?? - {}, - ); - } - - /// Shards that must complete before this compose can run. - final List requires; - - /// Path to the Python script to execute (relative to engine/src). - final String script; - - /// Boolean flags to pass to the script (e.g., --dsym, --strip, --zip). - final List flags; - - /// Arguments whose values are paths relative to out/ (e.g., --arm64-out-dir: ios_release). - /// These are expanded to absolute paths at runtime. - final Map pathArgs; -} diff --git a/shorebird/ci/shard_runner/lib/config.dart b/shorebird/ci/shard_runner/lib/config.dart deleted file mode 100644 index 9fecdabe1cfd2..0000000000000 --- a/shorebird/ci/shard_runner/lib/config.dart +++ /dev/null @@ -1,268 +0,0 @@ -import 'dart:convert'; -import 'dart:io'; - -import 'package:meta/meta.dart'; -import 'package:path/path.dart' as p; -import 'package:shard_runner/process.dart'; - -/// Configuration for all shards on a platform. -@immutable -class PlatformConfig { - PlatformConfig({required this.shards}); - - factory PlatformConfig.fromJson(Map json) { - return PlatformConfig( - shards: json.map( - (String key, value) => - MapEntry(key, ShardDef.fromJson(value as Map)), - ), - ); - } - final Map shards; - - ShardDef getShard(String name) { - final ShardDef? shard = shards[name]; - if (shard == null) { - throw ArgumentError( - 'Unknown shard: $name. Available: ${shards.keys.join(', ')}', - ); - } - return shard; - } - - static PlatformConfig load(String platform, String configDir) { - final File file = File(p.join(configDir, 'shards', '$platform.json')); - if (!file.existsSync()) { - throw FileSystemException('Config file not found', file.path); - } - final String content = file.readAsStringSync(); - final Map json = - jsonDecode(content) as Map; - return PlatformConfig.fromJson(json); - } -} - -/// Definition of a single build shard. -@immutable -class ShardDef { - ShardDef({ - required this.steps, - this.composeInput, - this.artifacts = const [], - }); - - factory ShardDef.fromJson(Map json) { - final List steps = (json['steps'] as List) - .map((s) => BuildStep.fromJson(s as Map)) - .toList(); - - final List artifacts = (json['artifacts'] as List?) - ?.map((a) => ArtifactDef.fromJson(a as Map)) - .toList() ?? - []; - - return ShardDef( - steps: steps, - composeInput: json['compose_input'] as String?, - artifacts: artifacts, - ); - } - - /// Build steps to execute. For simple shards, this is a single GnNinja step. - final List steps; - - /// If set, this shard contributes to a compose operation. - final String? composeInput; - - /// Artifacts produced by this shard (paths relative to out_dir). - /// Used by finalize to know what to upload. - final List artifacts; -} - -/// Definition of an artifact to upload. -@immutable -class ArtifactDef { - ArtifactDef({ - required this.src, - required this.dst, - this.zip = false, - this.contentHash = false, - }); - - factory ArtifactDef.fromJson(Map json) { - return ArtifactDef( - src: json['src'] as String, - dst: json['dst'] as String, - zip: json['zip'] as bool? ?? false, - contentHash: json['content_hash'] as bool? ?? false, - ); - } - - /// Source path relative to out/ (or out// for single-step shards) - final String src; - - /// Destination path (relative to storage bucket root). - /// Supports placeholders: $engine (engine hash) - final String dst; - - /// If true, zip the source directory before uploading. - final bool zip; - - /// If true, also upload to content-hash path (for Dart SDK). - final bool contentHash; -} - -/// Base class for build steps. -sealed class BuildStep { - factory BuildStep.fromJson(Map json) { - final String type = json['type'] as String; - return switch (type) { - 'gn_ninja' => GnNinjaStep.fromJson(json), - 'rust' => RustStep.fromJson(json), - _ => throw ArgumentError('Unknown step type: $type'), - }; - } - Future execute(String engineSrc); -} - -/// A GN + Ninja build step. -@immutable -class GnNinjaStep implements BuildStep { - GnNinjaStep({ - required this.gnArgs, - required this.ninjaTargets, - required this.outDir, - }); - - factory GnNinjaStep.fromJson(Map json) { - return GnNinjaStep( - gnArgs: (json['gn_args'] as List).cast(), - ninjaTargets: (json['ninja_targets'] as List).cast(), - outDir: json['out_dir'] as String, - ); - } - final List gnArgs; - final List ninjaTargets; - final String outDir; - - @override - Future execute(String engineSrc) async { - // Import gn.dart functions - await _runGn(engineSrc, gnArgs, outDir); - await _runNinja(engineSrc, outDir, ninjaTargets); - } -} - -/// A Rust/Cargo build step. -@immutable -class RustStep implements BuildStep { - RustStep({required this.targets}); - - factory RustStep.fromJson(Map json) { - return RustStep(targets: (json['targets'] as List).cast()); - } - final List targets; - - @override - Future execute(String engineSrc) async { - await _runRust(engineSrc, targets); - } -} - -// Internal execution functions (to be moved to separate files) -Future _runGn(String engineSrc, List args, String outDir) async { - print('[GN] Building $outDir with args: ${args.join(' ')}'); - await runChecked( - 'python3', - [ - p.join(engineSrc, 'flutter', 'tools', 'gn'), - '--no-rbe', - '--no-enable-unittests', - '--target-dir', - outDir, - ...args, - ], - workingDirectory: engineSrc, - description: 'GN ($outDir)', - ); - print('[GN] Complete'); -} - -Future _runNinja( - String engineSrc, - String outDir, - List targets, -) async { - print('[Ninja] Building ${targets.join(' ')} in out/$outDir'); - await runChecked( - 'ninja', - ['-C', p.join(engineSrc, 'out', outDir), ...targets], - workingDirectory: engineSrc, - description: 'Ninja ($outDir)', - ); - print('[Ninja] Complete'); -} - -Future _runRust(String engineSrc, List targets) async { - final String updaterPath = p.join( - engineSrc, - 'flutter', - 'third_party', - 'updater', - 'library', - ); - - // Separate Android and non-Android targets - final List androidTargets = - targets.where((String t) => t.contains('android')).toList(); - final List otherTargets = - targets.where((String t) => !t.contains('android')).toList(); - - // Build all Android targets together with cargo-ndk - if (androidTargets.isNotEmpty) { - print('[Rust] Building Android targets: ${androidTargets.join(', ')}'); - - final List args = ['ndk']; - for (final String target in androidTargets) { - args.addAll(['--target', target]); - } - args.addAll(['build', '--release']); - - // The "unmodified" CIPD package keeps the NDK at the standard Android - // SDK path: android_tools/sdk/ndk/. - final Directory ndkParent = Directory( - p.join( - engineSrc, - 'flutter', - 'third_party', - 'android_tools', - 'sdk', - 'ndk', - ), - ); - final String ndkHome = - ndkParent.listSync().whereType().first.path; - - await runChecked( - 'cargo', - args, - workingDirectory: updaterPath, - environment: {'ANDROID_NDK_HOME': ndkHome}, - description: 'Cargo ndk (${androidTargets.join(', ')})', - ); - } - - // Build non-Android targets individually - for (final String target in otherTargets) { - print('[Rust] Building for target: $target'); - - await runChecked( - 'cargo', - ['build', '--release', '--target', target], - workingDirectory: updaterPath, - description: 'Cargo ($target)', - ); - } - - print('[Rust] Complete'); -} diff --git a/shorebird/ci/shard_runner/lib/gcs.dart b/shorebird/ci/shard_runner/lib/gcs.dart deleted file mode 100644 index 18c622c1481e3..0000000000000 --- a/shorebird/ci/shard_runner/lib/gcs.dart +++ /dev/null @@ -1,111 +0,0 @@ -import 'dart:io'; - -import 'package:path/path.dart' as p; -import 'package:shard_runner/process.dart'; - -/// Staging bucket for intermediate artifacts. -const String stagingBucket = 'gs://shorebird-build-staging'; - -/// Uploads shard artifacts to GCS staging bucket. -/// -/// Artifacts are uploaded to: -/// gs://shorebird-build-staging/builds/{runId}/{platform}/{shard}/ -Future uploadToStaging({ - required String runId, - required String platform, - required String shard, - required String engineSrc, - required List outDirs, -}) async { - final String stagingRoot = '$stagingBucket/builds/$runId/$platform/$shard'; - - print('[GCS] Uploading to $stagingRoot'); - - for (final String outDir in outDirs) { - final String outPath = p.join(engineSrc, 'out', outDir); - if (!Directory(outPath).existsSync()) { - print('[GCS] Skipping $outDir (not found)'); - continue; - } - - // Create a tarball of the out directory - final String tarFile = '$outDir.tar.gz'; - print('[GCS] Creating $tarFile...'); - - await runChecked( - 'tar', - ['-czf', tarFile, '-C', p.join(engineSrc, 'out'), outDir], - workingDirectory: engineSrc, - description: 'tar create $tarFile', - ); - - // Upload to GCS - print('[GCS] Uploading $tarFile...'); - await runChecked( - 'gsutil', - ['-m', 'cp', p.join(engineSrc, tarFile), '$stagingRoot/'], - description: 'gsutil upload $tarFile', - ); - - // Clean up local tarball - File(p.join(engineSrc, tarFile)).deleteSync(); - } - - // Upload status file - final File statusFile = File(p.join(engineSrc, 'status.json')); - statusFile.writeAsStringSync('{"status": "success", "shard": "$shard"}'); - await runChecked('gsutil', ['cp', statusFile.path, '$stagingRoot/'], - description: 'gsutil upload status.json'); - statusFile.deleteSync(); - - print('[GCS] Upload complete'); -} - -/// Downloads artifacts from GCS staging bucket. -Future downloadFromStaging({ - required String runId, - required String platform, - required String shard, - required String destDir, -}) async { - final String stagingRoot = '$stagingBucket/builds/$runId/$platform/$shard'; - - print('[GCS] Downloading from $stagingRoot'); - - // List files in the staging location. We capture stdout because we need - // to parse the file list; everything else uses runChecked to stream. - final String lsOutput = await runCapturingStdout( - 'gsutil', - ['ls', stagingRoot], - description: 'gsutil ls $stagingRoot', - ); - - final List files = - lsOutput.split('\n').where((String f) => f.endsWith('.tar.gz')).toList(); - - for (final String file in files) { - final String fileName = p.basename(file); - print('[GCS] Downloading $fileName...'); - - // Download - await runChecked( - 'gsutil', - ['cp', file, p.join(destDir, fileName)], - description: 'gsutil download $fileName', - ); - - // Extract - print('[GCS] Extracting $fileName...'); - await runChecked( - 'tar', - ['-xzf', fileName], - workingDirectory: destDir, - description: 'tar extract $fileName', - ); - - // Clean up tarball - File(p.join(destDir, fileName)).deleteSync(); - } - - print('[GCS] Download complete'); -} diff --git a/shorebird/ci/shard_runner/lib/manifest.dart b/shorebird/ci/shard_runner/lib/manifest.dart deleted file mode 100644 index db5c3a5789248..0000000000000 --- a/shorebird/ci/shard_runner/lib/manifest.dart +++ /dev/null @@ -1,26 +0,0 @@ -import 'dart:io'; - -import 'package:path/path.dart' as p; - -/// Generates the artifacts manifest YAML from template. -/// -/// The manifest maps a Shorebird engine revision to a Flutter engine revision -/// and lists all artifact paths that should be proxied. -/// -/// [flutterEngineRevision] is the base Flutter engine revision this build is based on. -/// [configDir] is the path to the ci/ directory containing the template. -String generateManifest( - String flutterEngineRevision, { - required String configDir, -}) { - final templatePath = p.join(configDir, 'artifacts_manifest.template.yaml'); - final templateFile = File(templatePath); - - if (!templateFile.existsSync()) { - throw ArgumentError('Manifest template not found: $templatePath'); - } - - final template = templateFile.readAsStringSync(); - return template.replaceAll( - '{{flutter_engine_revision}}', flutterEngineRevision); -} diff --git a/shorebird/ci/shard_runner/lib/process.dart b/shorebird/ci/shard_runner/lib/process.dart deleted file mode 100644 index 5c3a0a6adaf22..0000000000000 --- a/shorebird/ci/shard_runner/lib/process.dart +++ /dev/null @@ -1,103 +0,0 @@ -import 'dart:async'; -import 'dart:convert'; -import 'dart:io'; - -/// Resolves an executable name for Windows, appending .cmd if needed. -/// -/// On Windows, many tools like gsutil and gcloud are installed as .cmd files. -/// This function checks if a .cmd version exists and uses it. -String _resolveExecutable(String executable) { - if (!Platform.isWindows) return executable; - - // Don't modify if it already has an extension - if (executable.endsWith('.exe') || - executable.endsWith('.cmd') || - executable.endsWith('.bat')) { - return executable; - } - - // Check if .cmd version exists in PATH - final String? path = Platform.environment['PATH']; - if (path == null) return executable; - - for (final String dir in path.split(';')) { - final File cmdFile = File('$dir\\$executable.cmd'); - if (cmdFile.existsSync()) { - return '$executable.cmd'; - } - } - - return executable; -} - -/// Runs a process, streaming stdout/stderr to the parent's stdio so the -/// caller (and CI logs) see output live. Throws if the process exits -/// non-zero. -/// -/// Use this for everything except the rare case where you need to parse -/// the child's stdout โ€” for that, see [runCapturingStdout]. -Future runChecked( - String executable, - List arguments, { - String? workingDirectory, - Map? environment, - String? description, -}) async { - final String resolvedExecutable = _resolveExecutable(executable); - - final Process process = await Process.start( - resolvedExecutable, - arguments, - workingDirectory: workingDirectory, - environment: environment, - mode: ProcessStartMode.inheritStdio, - ); - - final int exitCode = await process.exitCode; - if (exitCode != 0) { - final String desc = description ?? '$executable ${arguments.join(' ')}'; - throw Exception('$desc failed (exit $exitCode)'); - } -} - -/// Runs a process, capturing stdout into the returned string while still -/// streaming stderr to the parent's stderr. Throws if the process exits -/// non-zero. -/// -/// Use this when you need to parse the child's stdout (e.g. `gsutil ls`). -/// For everything else use [runChecked] so output reaches the CI log live. -Future runCapturingStdout( - String executable, - List arguments, { - String? workingDirectory, - Map? environment, - String? description, -}) async { - final String resolvedExecutable = _resolveExecutable(executable); - - final Process process = await Process.start( - resolvedExecutable, - arguments, - workingDirectory: workingDirectory, - environment: environment, - ); - - final Future stdoutFuture = - process.stdout.transform(utf8.decoder).join(); - final Future stderrFuture = - process.stderr.transform(utf8.decoder).forEach(stderr.write); - - final List results = await Future.wait(>[ - stdoutFuture, - stderrFuture, - process.exitCode, - ]); - final String capturedStdout = results[0] as String; - final int exitCode = results[2] as int; - - if (exitCode != 0) { - final String desc = description ?? '$executable ${arguments.join(' ')}'; - throw Exception('$desc failed (exit $exitCode)'); - } - return capturedStdout; -} diff --git a/shorebird/ci/shard_runner/pubspec.lock b/shorebird/ci/shard_runner/pubspec.lock deleted file mode 100644 index 14ad0cf1c27c0..0000000000000 --- a/shorebird/ci/shard_runner/pubspec.lock +++ /dev/null @@ -1,389 +0,0 @@ -# Generated by pub -# See https://dart.dev/tools/pub/glossary#lockfile -packages: - _fe_analyzer_shared: - dependency: transitive - description: - name: _fe_analyzer_shared - sha256: "796d97d925add7ffcdf5595f33a2066a6e3cee97971e6dbef09b76b7880fd760" - url: "https://pub.dev" - source: hosted - version: "94.0.0" - analyzer: - dependency: transitive - description: - name: analyzer - sha256: "9c8ebb304d72c0a0c8764344627529d9503fc83d7d73e43ed727dc532f822e4b" - url: "https://pub.dev" - source: hosted - version: "10.0.2" - args: - dependency: "direct main" - description: - name: args - sha256: d0481093c50b1da8910eb0bb301626d4d8eb7284aa739614d2b394ee09e3ea04 - url: "https://pub.dev" - source: hosted - version: "2.7.0" - async: - dependency: transitive - description: - name: async - sha256: "758e6d74e971c3e5aceb4110bfd6698efc7f501675bcfe0c775459a8140750eb" - url: "https://pub.dev" - source: hosted - version: "2.13.0" - boolean_selector: - dependency: transitive - description: - name: boolean_selector - sha256: "8aab1771e1243a5063b8b0ff68042d67334e3feab9e95b9490f9a6ebf73b42ea" - url: "https://pub.dev" - source: hosted - version: "2.1.2" - cli_config: - dependency: transitive - description: - name: cli_config - sha256: ac20a183a07002b700f0c25e61b7ee46b23c309d76ab7b7640a028f18e4d99ec - url: "https://pub.dev" - source: hosted - version: "0.2.0" - collection: - dependency: "direct main" - description: - name: collection - sha256: "2f5709ae4d3d59dd8f7cd309b4e023046b57d8a6c82130785d2b0e5868084e76" - url: "https://pub.dev" - source: hosted - version: "1.19.1" - convert: - dependency: transitive - description: - name: convert - sha256: b30acd5944035672bc15c6b7a8b47d773e41e2f17de064350988c5d02adb1c68 - url: "https://pub.dev" - source: hosted - version: "3.1.2" - coverage: - dependency: transitive - description: - name: coverage - sha256: "5da775aa218eaf2151c721b16c01c7676fbfdd99cebba2bf64e8b807a28ff94d" - url: "https://pub.dev" - source: hosted - version: "1.15.0" - crypto: - dependency: transitive - description: - name: crypto - sha256: c8ea0233063ba03258fbcf2ca4d6dadfefe14f02fab57702265467a19f27fadf - url: "https://pub.dev" - source: hosted - version: "3.0.7" - file: - dependency: transitive - description: - name: file - sha256: a3b4f84adafef897088c160faf7dfffb7696046cb13ae90b508c2cbc95d3b8d4 - url: "https://pub.dev" - source: hosted - version: "7.0.1" - frontend_server_client: - dependency: transitive - description: - name: frontend_server_client - sha256: f64a0333a82f30b0cca061bc3d143813a486dc086b574bfb233b7c1372427694 - url: "https://pub.dev" - source: hosted - version: "4.0.0" - glob: - dependency: transitive - description: - name: glob - sha256: c3f1ee72c96f8f78935e18aa8cecced9ab132419e8625dc187e1c2408efc20de - url: "https://pub.dev" - source: hosted - version: "2.1.3" - http_multi_server: - dependency: transitive - description: - name: http_multi_server - sha256: aa6199f908078bb1c5efb8d8638d4ae191aac11b311132c3ef48ce352fb52ef8 - url: "https://pub.dev" - source: hosted - version: "3.2.2" - http_parser: - dependency: transitive - description: - name: http_parser - sha256: "178d74305e7866013777bab2c3d8726205dc5a4dd935297175b19a23a2e66571" - url: "https://pub.dev" - source: hosted - version: "4.1.2" - io: - dependency: transitive - description: - name: io - sha256: dfd5a80599cf0165756e3181807ed3e77daf6dd4137caaad72d0b7931597650b - url: "https://pub.dev" - source: hosted - version: "1.0.5" - lints: - dependency: "direct dev" - description: - name: lints - sha256: cbf8d4b858bb0134ef3ef87841abdf8d63bfc255c266b7bf6b39daa1085c4290 - url: "https://pub.dev" - source: hosted - version: "3.0.0" - logging: - dependency: transitive - description: - name: logging - sha256: c8245ada5f1717ed44271ed1c26b8ce85ca3228fd2ffdb75468ab01979309d61 - url: "https://pub.dev" - source: hosted - version: "1.3.0" - matcher: - dependency: transitive - description: - name: matcher - sha256: "12956d0ad8390bbcc63ca2e1469c0619946ccb52809807067a7020d57e647aa6" - url: "https://pub.dev" - source: hosted - version: "0.12.18" - meta: - dependency: "direct main" - description: - name: meta - sha256: "9f29b9bcc8ee287b1a31e0d01be0eae99a930dbffdaecf04b3f3d82a969f296f" - url: "https://pub.dev" - source: hosted - version: "1.18.1" - mime: - dependency: transitive - description: - name: mime - sha256: "41a20518f0cb1256669420fdba0cd90d21561e560ac240f26ef8322e45bb7ed6" - url: "https://pub.dev" - source: hosted - version: "2.0.0" - node_preamble: - dependency: transitive - description: - name: node_preamble - sha256: "6e7eac89047ab8a8d26cf16127b5ed26de65209847630400f9aefd7cd5c730db" - url: "https://pub.dev" - source: hosted - version: "2.0.2" - package_config: - dependency: transitive - description: - name: package_config - sha256: f096c55ebb7deb7e384101542bfba8c52696c1b56fca2eb62827989ef2353bbc - url: "https://pub.dev" - source: hosted - version: "2.2.0" - path: - dependency: "direct main" - description: - name: path - sha256: "75cca69d1490965be98c73ceaea117e8a04dd21217b37b292c9ddbec0d955bc5" - url: "https://pub.dev" - source: hosted - version: "1.9.1" - pool: - dependency: transitive - description: - name: pool - sha256: "978783255c543aa3586a1b3c21f6e9d720eb315376a915872c61ef8b5c20177d" - url: "https://pub.dev" - source: hosted - version: "1.5.2" - pub_semver: - dependency: transitive - description: - name: pub_semver - sha256: "5bfcf68ca79ef689f8990d1160781b4bad40a3bd5e5218ad4076ddb7f4081585" - url: "https://pub.dev" - source: hosted - version: "2.2.0" - shelf: - dependency: transitive - description: - name: shelf - sha256: e7dd780a7ffb623c57850b33f43309312fc863fb6aa3d276a754bb299839ef12 - url: "https://pub.dev" - source: hosted - version: "1.4.2" - shelf_packages_handler: - dependency: transitive - description: - name: shelf_packages_handler - sha256: "89f967eca29607c933ba9571d838be31d67f53f6e4ee15147d5dc2934fee1b1e" - url: "https://pub.dev" - source: hosted - version: "3.0.2" - shelf_static: - dependency: transitive - description: - name: shelf_static - sha256: c87c3875f91262785dade62d135760c2c69cb217ac759485334c5857ad89f6e3 - url: "https://pub.dev" - source: hosted - version: "1.1.3" - shelf_web_socket: - dependency: transitive - description: - name: shelf_web_socket - sha256: "3632775c8e90d6c9712f883e633716432a27758216dfb61bd86a8321c0580925" - url: "https://pub.dev" - source: hosted - version: "3.0.0" - source_map_stack_trace: - dependency: transitive - description: - name: source_map_stack_trace - sha256: c0713a43e323c3302c2abe2a1cc89aa057a387101ebd280371d6a6c9fa68516b - url: "https://pub.dev" - source: hosted - version: "2.1.2" - source_maps: - dependency: transitive - description: - name: source_maps - sha256: "190222579a448b03896e0ca6eca5998fa810fda630c1d65e2f78b3f638f54812" - url: "https://pub.dev" - source: hosted - version: "0.10.13" - source_span: - dependency: transitive - description: - name: source_span - sha256: "254ee5351d6cb365c859e20ee823c3bb479bf4a293c22d17a9f1bf144ce86f7c" - url: "https://pub.dev" - source: hosted - version: "1.10.1" - stack_trace: - dependency: transitive - description: - name: stack_trace - sha256: "8b27215b45d22309b5cddda1aa2b19bdfec9df0e765f2de506401c071d38d1b1" - url: "https://pub.dev" - source: hosted - version: "1.12.1" - stream_channel: - dependency: transitive - description: - name: stream_channel - sha256: "969e04c80b8bcdf826f8f16579c7b14d780458bd97f56d107d3950fdbeef059d" - url: "https://pub.dev" - source: hosted - version: "2.1.4" - string_scanner: - dependency: transitive - description: - name: string_scanner - sha256: "921cd31725b72fe181906c6a94d987c78e3b98c2e205b397ea399d4054872b43" - url: "https://pub.dev" - source: hosted - version: "1.4.1" - term_glyph: - dependency: transitive - description: - name: term_glyph - sha256: "7f554798625ea768a7518313e58f83891c7f5024f88e46e7182a4558850a4b8e" - url: "https://pub.dev" - source: hosted - version: "1.2.2" - test: - dependency: "direct dev" - description: - name: test - sha256: "54c516bbb7cee2754d327ad4fca637f78abfc3cbcc5ace83b3eda117e42cd71a" - url: "https://pub.dev" - source: hosted - version: "1.29.0" - test_api: - dependency: transitive - description: - name: test_api - sha256: "93167629bfc610f71560ab9312acdda4959de4df6fac7492c89ff0d3886f6636" - url: "https://pub.dev" - source: hosted - version: "0.7.9" - test_core: - dependency: transitive - description: - name: test_core - sha256: "394f07d21f0f2255ec9e3989f21e54d3c7dc0e6e9dbce160e5a9c1a6be0e2943" - url: "https://pub.dev" - source: hosted - version: "0.6.15" - typed_data: - dependency: transitive - description: - name: typed_data - sha256: f9049c039ebfeb4cf7a7104a675823cd72dba8297f264b6637062516699fa006 - url: "https://pub.dev" - source: hosted - version: "1.4.0" - vm_service: - dependency: transitive - description: - name: vm_service - sha256: "45caa6c5917fa127b5dbcfbd1fa60b14e583afdc08bfc96dda38886ca252eb60" - url: "https://pub.dev" - source: hosted - version: "15.0.2" - watcher: - dependency: transitive - description: - name: watcher - sha256: "1398c9f081a753f9226febe8900fce8f7d0a67163334e1c94a2438339d79d635" - url: "https://pub.dev" - source: hosted - version: "1.2.1" - web: - dependency: transitive - description: - name: web - sha256: "868d88a33d8a87b18ffc05f9f030ba328ffefba92d6c127917a2ba740f9cfe4a" - url: "https://pub.dev" - source: hosted - version: "1.1.1" - web_socket: - dependency: transitive - description: - name: web_socket - sha256: "34d64019aa8e36bf9842ac014bb5d2f5586ca73df5e4d9bf5c936975cae6982c" - url: "https://pub.dev" - source: hosted - version: "1.0.1" - web_socket_channel: - dependency: transitive - description: - name: web_socket_channel - sha256: d645757fb0f4773d602444000a8131ff5d48c9e47adfe9772652dd1a4f2d45c8 - url: "https://pub.dev" - source: hosted - version: "3.0.3" - webkit_inspection_protocol: - dependency: transitive - description: - name: webkit_inspection_protocol - sha256: "87d3f2333bb240704cd3f1c6b5b7acd8a10e7f0bc28c28dcf14e782014f4a572" - url: "https://pub.dev" - source: hosted - version: "1.2.1" - yaml: - dependency: transitive - description: - name: yaml - sha256: b9da305ac7c39faa3f030eccd175340f968459dae4af175130b3fc47e40d76ce - url: "https://pub.dev" - source: hosted - version: "3.1.3" -sdks: - dart: ">=3.9.0 <4.0.0" diff --git a/shorebird/ci/shard_runner/pubspec.yaml b/shorebird/ci/shard_runner/pubspec.yaml deleted file mode 100644 index 8e0834004a88b..0000000000000 --- a/shorebird/ci/shard_runner/pubspec.yaml +++ /dev/null @@ -1,17 +0,0 @@ -name: shard_runner -description: Dart-based build shard runner for Shorebird CI -version: 0.1.0 -publish_to: none - -environment: - sdk: ^3.0.0 - -dependencies: - args: ^2.4.0 - collection: ^1.18.0 - meta: ^1.11.0 - path: ^1.8.0 - -dev_dependencies: - lints: ^3.0.0 - test: ^1.24.0 diff --git a/shorebird/ci/shard_runner/test/compose_config_test.dart b/shorebird/ci/shard_runner/test/compose_config_test.dart deleted file mode 100644 index 24868a6affcc5..0000000000000 --- a/shorebird/ci/shard_runner/test/compose_config_test.dart +++ /dev/null @@ -1,119 +0,0 @@ -import 'package:test/test.dart'; -import 'package:shard_runner/compose_config.dart'; - -void main() { - group('ComposeConfig', () { - test('parses compose definitions', () { - final Map json = { - 'ios-framework': { - 'requires': ['ios-release', 'ios-sim-x64', 'ios-sim-arm64'], - 'script': 'flutter/sky/tools/create_ios_framework.py', - 'flags': ['--dsym', '--strip'], - 'path_args': { - '--arm64-out-dir': 'ios_release', - }, - }, - 'macos-framework': { - 'requires': ['mac-arm64', 'mac-x64'], - 'script': 'flutter/sky/tools/create_macos_framework.py', - 'flags': ['--zip'], - }, - }; - - final ComposeConfig config = ComposeConfig.fromJson(json); - - expect(config.composes.length, 2); - expect(config.composes.containsKey('ios-framework'), true); - expect(config.composes.containsKey('macos-framework'), true); - }); - - test('getCompose returns correct definition', () { - final Map json = { - 'ios-framework': { - 'requires': ['ios-release'], - 'script': 'create_ios_framework.py', - }, - }; - - final ComposeConfig config = ComposeConfig.fromJson(json); - final ComposeDef compose = config.getCompose('ios-framework'); - - expect(compose.requires, ['ios-release']); - expect(compose.script, 'create_ios_framework.py'); - }); - - test('getCompose throws for unknown name', () { - final ComposeConfig config = - ComposeConfig(composes: {}); - - expect( - () => config.getCompose('nonexistent'), - throwsA(isA()), - ); - }); - }); - - group('ComposeDef', () { - test('parses all fields', () { - final Map json = { - 'requires': ['shard-a', 'shard-b'], - 'script': 'path/to/script.py', - 'flags': ['--dsym', '--strip'], - 'path_args': { - '--arm64-out-dir': 'ios_release', - '--x64-out-dir': 'ios_debug_sim', - }, - }; - - final ComposeDef compose = ComposeDef.fromJson(json); - - expect(compose.requires, ['shard-a', 'shard-b']); - expect(compose.script, 'path/to/script.py'); - expect(compose.flags, ['--dsym', '--strip']); - expect(compose.pathArgs, { - '--arm64-out-dir': 'ios_release', - '--x64-out-dir': 'ios_debug_sim', - }); - }); - - test('defaults flags and path_args when missing', () { - final Map json = { - 'requires': ['shard-a'], - 'script': 'script.py', - }; - - final ComposeDef compose = ComposeDef.fromJson(json); - - expect(compose.flags, isEmpty); - expect(compose.pathArgs, isEmpty); - }); - - test('parses ios-framework config correctly', () { - final Map json = { - 'requires': [ - 'ios-release', - 'ios-release-ext', - 'ios-sim-x64', - 'ios-sim-x64-ext', - 'ios-sim-arm64', - 'ios-sim-arm64-ext', - ], - 'script': 'flutter/sky/tools/create_ios_framework.py', - 'flags': ['--dsym', '--strip'], - 'path_args': { - '--arm64-out-dir': 'ios_release', - '--simulator-x64-out-dir': 'ios_debug_sim', - '--simulator-arm64-out-dir': 'ios_debug_sim_arm64', - }, - }; - - final ComposeDef compose = ComposeDef.fromJson(json); - - expect(compose.requires.length, 6); - expect(compose.script, 'flutter/sky/tools/create_ios_framework.py'); - expect(compose.flags, ['--dsym', '--strip']); - expect(compose.pathArgs.keys, contains('--arm64-out-dir')); - expect(compose.pathArgs['--arm64-out-dir'], 'ios_release'); - }); - }); -} diff --git a/shorebird/ci/shard_runner/test/config_test.dart b/shorebird/ci/shard_runner/test/config_test.dart deleted file mode 100644 index ec3d508864758..0000000000000 --- a/shorebird/ci/shard_runner/test/config_test.dart +++ /dev/null @@ -1,264 +0,0 @@ -import 'package:test/test.dart'; -import 'package:shard_runner/config.dart'; - -void main() { - group('PlatformConfig', () { - test('parses single-step shard', () { - final json = { - 'android-arm64': { - 'steps': [ - { - 'type': 'gn_ninja', - 'gn_args': [ - '--android', - '--android-cpu=arm64', - '--runtime-mode=release' - ], - 'ninja_targets': ['default', 'gen_snapshot'], - 'out_dir': 'android_release_arm64', - }, - ], - }, - }; - - final config = PlatformConfig.fromJson(json); - - expect(config.shards.length, 1); - expect(config.shards.containsKey('android-arm64'), true); - - final shard = config.getShard('android-arm64'); - expect(shard.steps.length, 1); - expect(shard.steps.first, isA()); - expect(shard.artifacts, isEmpty); - - final step = shard.steps.first as GnNinjaStep; - expect(step.gnArgs, - ['--android', '--android-cpu=arm64', '--runtime-mode=release']); - expect(step.ninjaTargets, ['default', 'gen_snapshot']); - expect(step.outDir, 'android_release_arm64'); - }); - - test('parses shard with artifacts', () { - final json = { - 'android-arm64': { - 'steps': [ - { - 'type': 'gn_ninja', - 'gn_args': ['--android'], - 'ninja_targets': ['default'], - 'out_dir': 'android_release_arm64', - }, - ], - 'artifacts': [ - { - 'src': 'zip_archives/artifacts.zip', - 'dst': 'flutter_infra/\$engine/artifacts.zip' - }, - {'src': 'maven.pom', 'dst': 'maven/\$engine/maven.pom'}, - ], - }, - }; - - final config = PlatformConfig.fromJson(json); - final shard = config.getShard('android-arm64'); - - expect(shard.artifacts.length, 2); - expect(shard.artifacts[0].src, 'zip_archives/artifacts.zip'); - expect(shard.artifacts[0].dst, 'flutter_infra/\$engine/artifacts.zip'); - expect(shard.artifacts[1].src, 'maven.pom'); - }); - - test('parses multi-step shard', () { - final json = { - 'host': { - 'steps': [ - { - 'type': 'rust', - 'targets': ['aarch64-linux-android', 'x86_64-unknown-linux-gnu'], - }, - { - 'type': 'gn_ninja', - 'gn_args': ['--runtime-mode=release'], - 'ninja_targets': ['dart_sdk'], - 'out_dir': 'host_release', - }, - ], - }, - }; - - final config = PlatformConfig.fromJson(json); - final shard = config.getShard('host'); - - expect(shard.steps.length, 2); - expect(shard.steps[0], isA()); - expect(shard.steps[1], isA()); - - final rustStep = shard.steps[0] as RustStep; - expect(rustStep.targets, - ['aarch64-linux-android', 'x86_64-unknown-linux-gnu']); - - final gnStep = shard.steps[1] as GnNinjaStep; - expect(gnStep.outDir, 'host_release'); - }); - - test('parses compose_input', () { - final json = { - 'ios-release': { - 'steps': [ - { - 'type': 'gn_ninja', - 'gn_args': ['--ios', '--runtime-mode=release'], - 'ninja_targets': ['flutter_framework'], - 'out_dir': 'ios_release', - }, - ], - 'compose_input': 'ios-framework', - }, - }; - - final config = PlatformConfig.fromJson(json); - final shard = config.getShard('ios-release'); - - expect(shard.composeInput, 'ios-framework'); - }); - - test('getShard throws for unknown shard', () { - final config = PlatformConfig(shards: {}); - - expect( - () => config.getShard('nonexistent'), - throwsA(isA()), - ); - }); - }); - - group('BuildStep.fromJson', () { - test('parses gn_ninja type', () { - final json = { - 'type': 'gn_ninja', - 'gn_args': ['--android'], - 'ninja_targets': ['default'], - 'out_dir': 'out_dir', - }; - - final step = BuildStep.fromJson(json); - expect(step, isA()); - }); - - test('parses rust type', () { - final json = { - 'type': 'rust', - 'targets': ['x86_64-unknown-linux-gnu'], - }; - - final step = BuildStep.fromJson(json); - expect(step, isA()); - }); - - test('throws for unknown type', () { - final json = { - 'type': 'unknown', - }; - - expect( - () => BuildStep.fromJson(json), - throwsA(isA()), - ); - }); - }); - - group('GnNinjaStep', () { - test('fromJson parses all fields', () { - final json = { - 'type': 'gn_ninja', - 'gn_args': ['--android', '--runtime-mode=release'], - 'ninja_targets': ['default', 'gen_snapshot'], - 'out_dir': 'android_release', - }; - - final step = GnNinjaStep.fromJson(json); - - expect(step.gnArgs, ['--android', '--runtime-mode=release']); - expect(step.ninjaTargets, ['default', 'gen_snapshot']); - expect(step.outDir, 'android_release'); - }); - }); - - group('RustStep', () { - test('fromJson parses targets', () { - final json = { - 'type': 'rust', - 'targets': [ - 'aarch64-linux-android', - 'armv7-linux-androideabi', - 'x86_64-unknown-linux-gnu', - ], - }; - - final step = RustStep.fromJson(json); - - expect(step.targets, [ - 'aarch64-linux-android', - 'armv7-linux-androideabi', - 'x86_64-unknown-linux-gnu', - ]); - }); - }); - - group('ArtifactDef', () { - test('fromJson parses src and dst', () { - final json = { - 'src': 'zip_archives/artifacts.zip', - 'dst': 'flutter_infra/\$engine/artifacts.zip', - }; - - final artifact = ArtifactDef.fromJson(json); - - expect(artifact.src, 'zip_archives/artifacts.zip'); - expect(artifact.dst, 'flutter_infra/\$engine/artifacts.zip'); - expect(artifact.zip, false); - expect(artifact.contentHash, false); - }); - - test('fromJson parses zip flag', () { - final json = { - 'src': 'dart-sdk', - 'dst': 'flutter_infra/dart-sdk.zip', - 'zip': true, - }; - - final artifact = ArtifactDef.fromJson(json); - - expect(artifact.zip, true); - }); - - test('fromJson parses content_hash flag', () { - final json = { - 'src': 'dart-sdk', - 'dst': 'flutter_infra/dart-sdk.zip', - 'content_hash': true, - }; - - final artifact = ArtifactDef.fromJson(json); - - expect(artifact.contentHash, true); - }); - - test('fromJson parses all flags together', () { - final json = { - 'src': 'host_release/dart-sdk', - 'dst': 'flutter_infra/flutter/\$engine/dart-sdk-linux-x64.zip', - 'zip': true, - 'content_hash': true, - }; - - final artifact = ArtifactDef.fromJson(json); - - expect(artifact.src, 'host_release/dart-sdk'); - expect(artifact.dst, - 'flutter_infra/flutter/\$engine/dart-sdk-linux-x64.zip'); - expect(artifact.zip, true); - expect(artifact.contentHash, true); - }); - }); -} diff --git a/shorebird/ci/shard_runner/test/manifest_test.dart b/shorebird/ci/shard_runner/test/manifest_test.dart deleted file mode 100644 index a5606ae9b2f54..0000000000000 --- a/shorebird/ci/shard_runner/test/manifest_test.dart +++ /dev/null @@ -1,208 +0,0 @@ -import 'dart:io'; - -import 'package:path/path.dart' as p; -import 'package:test/test.dart'; -import 'package:shard_runner/manifest.dart'; - -void main() { - group('generateManifest', () { - // Path to the ci/ directory where the template lives - // Tests run from shard_runner/, so go up one level to ci/ - final String configDir = p.normalize(p.join(Directory.current.path, '..')); - - test('generates valid YAML structure', () { - final manifest = generateManifest('abc123def456', configDir: configDir); - - expect(manifest, contains('flutter_engine_revision: abc123def456')); - expect(manifest, contains('storage_bucket: download.shorebird.dev')); - expect(manifest, contains('artifact_overrides:')); - }); - - test('includes Android release artifacts', () { - final manifest = generateManifest('test-hash', configDir: configDir); - - // Android arm64 - expect( - manifest, - contains( - r'flutter_infra_release/flutter/$engine/android-arm64-release/artifacts.zip')); - expect( - manifest, - contains( - r'flutter_infra_release/flutter/$engine/android-arm64-release/linux-x64.zip')); - expect( - manifest, - contains( - r'flutter_infra_release/flutter/$engine/android-arm64-release/darwin-x64.zip')); - expect( - manifest, - contains( - r'flutter_infra_release/flutter/$engine/android-arm64-release/windows-x64.zip')); - expect( - manifest, - contains( - r'flutter_infra_release/flutter/$engine/android-arm64-release/symbols.zip')); - - // Android arm32 - expect( - manifest, - contains( - r'flutter_infra_release/flutter/$engine/android-arm-release/artifacts.zip')); - - // Android x64 - expect( - manifest, - contains( - r'flutter_infra_release/flutter/$engine/android-x64-release/artifacts.zip')); - }); - - test('includes Dart SDK for all platforms', () { - final manifest = generateManifest('test-hash', configDir: configDir); - - expect( - manifest, - contains( - r'flutter_infra_release/flutter/$engine/dart-sdk-darwin-arm64.zip')); - expect( - manifest, - contains( - r'flutter_infra_release/flutter/$engine/dart-sdk-darwin-x64.zip')); - expect( - manifest, - contains( - r'flutter_infra_release/flutter/$engine/dart-sdk-linux-x64.zip')); - expect( - manifest, - contains( - r'flutter_infra_release/flutter/$engine/dart-sdk-windows-x64.zip')); - }); - - test('includes Maven artifacts', () { - final manifest = generateManifest('test-hash', configDir: configDir); - - // flutter_embedding_release - expect( - manifest, - contains( - r'download.flutter.io/io/flutter/flutter_embedding_release/1.0.0-$engine/flutter_embedding_release-1.0.0-$engine.pom')); - expect( - manifest, - contains( - r'download.flutter.io/io/flutter/flutter_embedding_release/1.0.0-$engine/flutter_embedding_release-1.0.0-$engine.jar')); - - // arm64_v8a_release - expect( - manifest, - contains( - r'download.flutter.io/io/flutter/arm64_v8a_release/1.0.0-$engine/arm64_v8a_release-1.0.0-$engine.pom')); - expect( - manifest, - contains( - r'download.flutter.io/io/flutter/arm64_v8a_release/1.0.0-$engine/arm64_v8a_release-1.0.0-$engine.jar')); - - // armeabi_v7a_release - expect( - manifest, - contains( - r'download.flutter.io/io/flutter/armeabi_v7a_release/1.0.0-$engine/armeabi_v7a_release-1.0.0-$engine.pom')); - - // x86_64_release - expect( - manifest, - contains( - r'download.flutter.io/io/flutter/x86_64_release/1.0.0-$engine/x86_64_release-1.0.0-$engine.pom')); - }); - - test('includes iOS release artifacts', () { - final manifest = generateManifest('test-hash', configDir: configDir); - - expect( - manifest, - contains( - r'flutter_infra_release/flutter/$engine/ios-release/artifacts.zip')); - expect( - manifest, - contains( - r'flutter_infra_release/flutter/$engine/ios-release/Flutter.framework.dSYM.zip')); - }); - - test('includes Linux release artifacts', () { - final manifest = generateManifest('test-hash', configDir: configDir); - - expect( - manifest, - contains( - r'flutter_infra_release/flutter/$engine/linux-x64/artifacts.zip')); - expect( - manifest, - contains( - r'flutter_infra_release/flutter/$engine/linux-x64-release/linux-x64-flutter-gtk.zip')); - }); - - test('includes macOS release artifacts', () { - final manifest = generateManifest('test-hash', configDir: configDir); - - expect( - manifest, - contains( - r'flutter_infra_release/flutter/$engine/darwin-x64-release/artifacts.zip')); - expect( - manifest, - contains( - r'flutter_infra_release/flutter/$engine/darwin-x64-release/framework.zip')); - expect( - manifest, - contains( - r'flutter_infra_release/flutter/$engine/darwin-x64-release/gen_snapshot.zip')); - }); - - test('includes Windows release artifacts', () { - final manifest = generateManifest('test-hash', configDir: configDir); - - expect( - manifest, - contains( - r'flutter_infra_release/flutter/$engine/windows-x64/artifacts.zip')); - expect( - manifest, - contains( - r'flutter_infra_release/flutter/$engine/windows-x64-release/windows-x64-flutter.zip')); - }); - - test('includes engine_stamp.json', () { - final manifest = generateManifest('test-hash', configDir: configDir); - - expect(manifest, - contains(r'flutter_infra_release/flutter/$engine/engine_stamp.json')); - }); - - test('includes flutter_patched_sdk_product', () { - final manifest = generateManifest('test-hash', configDir: configDir); - - expect( - manifest, - contains( - r'flutter_infra_release/flutter/$engine/flutter_patched_sdk_product.zip')); - }); - - test('uses \$engine placeholder (not hardcoded hash)', () { - final manifest = generateManifest('abc123', configDir: configDir); - - // The flutter_engine_revision should use the actual hash - expect(manifest, contains('flutter_engine_revision: abc123')); - - // But artifact paths should use $engine placeholder - expect(manifest, contains(r'$engine')); - // Should NOT contain the actual hash in artifact paths - expect(manifest.split('flutter_engine_revision:')[1], - isNot(contains('abc123/'))); - }); - - test('throws when template file not found', () { - expect( - () => generateManifest('test-hash', configDir: '/nonexistent'), - throwsA(isA()), - ); - }); - }); -} From 5627d92c2ab33a41f869661069fe4d3bb978ec4a Mon Sep 17 00:00:00 2001 From: Eric Seidel Date: Fri, 8 May 2026 10:16:11 -0700 Subject: [PATCH 57/95] ci(shorebird): add dst + artifacts to compose.json; mac framework artifact (#146) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * ci(shorebird): add dst + artifacts to compose.json; add mac framework artifact Two changes that pair with shorebirdtech/_build_engine#209: 1. compose.json now declares output directory + production artifact list per compose, matching the model shards/*.json already uses. Without this finalize uploads only shard outputs to prod and the iOS XCFramework, FlutterMacOS framework + dSYM, and gen_snapshot zips never leave the staging bucket. - ios-framework: artifacts.zip + Flutter.framework.dSYM.zip (legacy mac_upload.sh equivalent) - macos-framework: FlutterMacOS.framework.zip + .dSYM.zip - macos-gen-snapshot: gen_snapshot.zip, with a dst override of "release/snapshot" so create_macos_gen_snapshots.py's --zip flag (which zips the entire dst directory) doesn't scoop up sibling composes' outputs 2. mac-arm64 shard's artifacts list gains mac_release_arm64/zip_archives/darwin-arm64-release/artifacts.zip uploaded to flutter_infra_release/.../darwin-x64-release/artifacts.zip (intentional cross-arch path mirroring legacy mac_upload.sh: "arm macs use darwin-x64-release and we currently only support those"). Both fields are optional in the runner's parser (defaults: dst="release", artifacts=[]), so older / mid-rollout flutter SHAs continue to work with the new build_engine code; the finalize prod-upload set just shrinks. * ci(shorebird): prefix android-shard artifact srcs with out_dir The macOS android shard's artifact srcs include the shard's out_dir prefix (e.g. "android_release_arm64/zip_archives/...") so that finalize, which joins src against engine/src/out, finds the file. The Linux and Windows android-* shards' srcs had no such prefix ("zip_archives/android-arm64-release/...") and finalize skipped every artifact as not-found. Verified by the upload log of run 25561356686 โ€” only Linux/host and Windows/host shard artifacts (which had correct prefixes) actually landed in the bucket; every Linux/Windows android-* artifact was [Skip] (not found). Prefix every artifact src in: - linux/android-arm64 with android_release_arm64/ - linux/android-arm32 with android_release/ - linux/android-x64 with android_release_x64/ - windows/android-arm64 with android_release_arm64/ - windows/android-arm32 with android_release/ - windows/android-x64 with android_release_x64/ --- shorebird/ci/compose.json | 18 +++++++++++--- shorebird/ci/shards/linux.json | 42 ++++++++++++++++---------------- shorebird/ci/shards/macos.json | 3 ++- shorebird/ci/shards/windows.json | 6 ++--- 4 files changed, 41 insertions(+), 28 deletions(-) diff --git a/shorebird/ci/compose.json b/shorebird/ci/compose.json index 1f6ca744d1e5d..f5f2437b8ab4c 100644 --- a/shorebird/ci/compose.json +++ b/shorebird/ci/compose.json @@ -7,7 +7,11 @@ "--arm64-out-dir": "ios_release", "--simulator-x64-out-dir": "ios_debug_sim", "--simulator-arm64-out-dir": "ios_debug_sim_arm64" - } + }, + "artifacts": [ + {"src": "artifacts.zip", "dst": "flutter_infra_release/flutter/$engine/ios-release/artifacts.zip"}, + {"src": "Flutter.framework.dSYM.zip", "dst": "flutter_infra_release/flutter/$engine/ios-release/Flutter.framework.dSYM.zip"} + ] }, "macos-framework": { "requires": ["mac-arm64", "mac-x64"], @@ -16,7 +20,11 @@ "path_args": { "--arm64-out-dir": "mac_release_arm64", "--x64-out-dir": "mac_release" - } + }, + "artifacts": [ + {"src": "FlutterMacOS.framework.zip", "dst": "flutter_infra_release/flutter/$engine/darwin-x64-release/FlutterMacOS.framework.zip"}, + {"src": "FlutterMacOS.framework.dSYM.zip", "dst": "flutter_infra_release/flutter/$engine/darwin-x64/FlutterMacOS.framework.dSYM.zip"} + ] }, "macos-gen-snapshot": { "requires": ["mac-arm64", "mac-x64"], @@ -25,6 +33,10 @@ "path_args": { "--arm64-path": "mac_release_arm64/universal/gen_snapshot_arm64", "--x64-path": "mac_release/universal/gen_snapshot_x64" - } + }, + "dst": "release/snapshot", + "artifacts": [ + {"src": "snapshot/gen_snapshot.zip", "dst": "flutter_infra_release/flutter/$engine/darwin-x64-release/gen_snapshot.zip"} + ] } } diff --git a/shorebird/ci/shards/linux.json b/shorebird/ci/shards/linux.json index 51b0aa86f0b6b..435ec9bebb7b7 100644 --- a/shorebird/ci/shards/linux.json +++ b/shorebird/ci/shards/linux.json @@ -13,12 +13,12 @@ } ], "artifacts": [ - {"src": "zip_archives/android-arm64-release/artifacts.zip", "dst": "flutter_infra_release/flutter/$engine/android-arm64-release/artifacts.zip"}, - {"src": "zip_archives/android-arm64-release/linux-x64.zip", "dst": "flutter_infra_release/flutter/$engine/android-arm64-release/linux-x64.zip"}, - {"src": "zip_archives/android-arm64-release/symbols.zip", "dst": "flutter_infra_release/flutter/$engine/android-arm64-release/symbols.zip"}, - {"src": "arm64_v8a_release.pom", "dst": "download.flutter.io/io/flutter/arm64_v8a_release/1.0.0-$engine/arm64_v8a_release-1.0.0-$engine.pom"}, - {"src": "arm64_v8a_release.jar", "dst": "download.flutter.io/io/flutter/arm64_v8a_release/1.0.0-$engine/arm64_v8a_release-1.0.0-$engine.jar"}, - {"src": "arm64_v8a_release.maven-metadata.xml", "dst": "download.flutter.io/io/flutter/arm64_v8a_release/1.0.0-$engine/arm64_v8a_release-1.0.0-$engine.maven-metadata.xml"} + {"src": "android_release_arm64/zip_archives/android-arm64-release/artifacts.zip", "dst": "flutter_infra_release/flutter/$engine/android-arm64-release/artifacts.zip"}, + {"src": "android_release_arm64/zip_archives/android-arm64-release/linux-x64.zip", "dst": "flutter_infra_release/flutter/$engine/android-arm64-release/linux-x64.zip"}, + {"src": "android_release_arm64/zip_archives/android-arm64-release/symbols.zip", "dst": "flutter_infra_release/flutter/$engine/android-arm64-release/symbols.zip"}, + {"src": "android_release_arm64/arm64_v8a_release.pom", "dst": "download.flutter.io/io/flutter/arm64_v8a_release/1.0.0-$engine/arm64_v8a_release-1.0.0-$engine.pom"}, + {"src": "android_release_arm64/arm64_v8a_release.jar", "dst": "download.flutter.io/io/flutter/arm64_v8a_release/1.0.0-$engine/arm64_v8a_release-1.0.0-$engine.jar"}, + {"src": "android_release_arm64/arm64_v8a_release.maven-metadata.xml", "dst": "download.flutter.io/io/flutter/arm64_v8a_release/1.0.0-$engine/arm64_v8a_release-1.0.0-$engine.maven-metadata.xml"} ] }, "android-arm32": { @@ -35,15 +35,15 @@ } ], "artifacts": [ - {"src": "zip_archives/android-arm-release/artifacts.zip", "dst": "flutter_infra_release/flutter/$engine/android-arm-release/artifacts.zip"}, - {"src": "zip_archives/android-arm-release/linux-x64.zip", "dst": "flutter_infra_release/flutter/$engine/android-arm-release/linux-x64.zip"}, - {"src": "zip_archives/android-arm-release/symbols.zip", "dst": "flutter_infra_release/flutter/$engine/android-arm-release/symbols.zip"}, - {"src": "armeabi_v7a_release.pom", "dst": "download.flutter.io/io/flutter/armeabi_v7a_release/1.0.0-$engine/armeabi_v7a_release-1.0.0-$engine.pom"}, - {"src": "armeabi_v7a_release.jar", "dst": "download.flutter.io/io/flutter/armeabi_v7a_release/1.0.0-$engine/armeabi_v7a_release-1.0.0-$engine.jar"}, - {"src": "armeabi_v7a_release.maven-metadata.xml", "dst": "download.flutter.io/io/flutter/armeabi_v7a_release/1.0.0-$engine/armeabi_v7a_release-1.0.0-$engine.maven-metadata.xml"}, - {"src": "flutter_embedding_release.pom", "dst": "download.flutter.io/io/flutter/flutter_embedding_release/1.0.0-$engine/flutter_embedding_release-1.0.0-$engine.pom"}, - {"src": "flutter_embedding_release.jar", "dst": "download.flutter.io/io/flutter/flutter_embedding_release/1.0.0-$engine/flutter_embedding_release-1.0.0-$engine.jar"}, - {"src": "flutter_embedding_release.maven-metadata.xml", "dst": "download.flutter.io/io/flutter/flutter_embedding_release/1.0.0-$engine/flutter_embedding_release-1.0.0-$engine.maven-metadata.xml"} + {"src": "android_release/zip_archives/android-arm-release/artifacts.zip", "dst": "flutter_infra_release/flutter/$engine/android-arm-release/artifacts.zip"}, + {"src": "android_release/zip_archives/android-arm-release/linux-x64.zip", "dst": "flutter_infra_release/flutter/$engine/android-arm-release/linux-x64.zip"}, + {"src": "android_release/zip_archives/android-arm-release/symbols.zip", "dst": "flutter_infra_release/flutter/$engine/android-arm-release/symbols.zip"}, + {"src": "android_release/armeabi_v7a_release.pom", "dst": "download.flutter.io/io/flutter/armeabi_v7a_release/1.0.0-$engine/armeabi_v7a_release-1.0.0-$engine.pom"}, + {"src": "android_release/armeabi_v7a_release.jar", "dst": "download.flutter.io/io/flutter/armeabi_v7a_release/1.0.0-$engine/armeabi_v7a_release-1.0.0-$engine.jar"}, + {"src": "android_release/armeabi_v7a_release.maven-metadata.xml", "dst": "download.flutter.io/io/flutter/armeabi_v7a_release/1.0.0-$engine/armeabi_v7a_release-1.0.0-$engine.maven-metadata.xml"}, + {"src": "android_release/flutter_embedding_release.pom", "dst": "download.flutter.io/io/flutter/flutter_embedding_release/1.0.0-$engine/flutter_embedding_release-1.0.0-$engine.pom"}, + {"src": "android_release/flutter_embedding_release.jar", "dst": "download.flutter.io/io/flutter/flutter_embedding_release/1.0.0-$engine/flutter_embedding_release-1.0.0-$engine.jar"}, + {"src": "android_release/flutter_embedding_release.maven-metadata.xml", "dst": "download.flutter.io/io/flutter/flutter_embedding_release/1.0.0-$engine/flutter_embedding_release-1.0.0-$engine.maven-metadata.xml"} ] }, "android-x64": { @@ -60,12 +60,12 @@ } ], "artifacts": [ - {"src": "zip_archives/android-x64-release/artifacts.zip", "dst": "flutter_infra_release/flutter/$engine/android-x64-release/artifacts.zip"}, - {"src": "zip_archives/android-x64-release/linux-x64.zip", "dst": "flutter_infra_release/flutter/$engine/android-x64-release/linux-x64.zip"}, - {"src": "zip_archives/android-x64-release/symbols.zip", "dst": "flutter_infra_release/flutter/$engine/android-x64-release/symbols.zip"}, - {"src": "x86_64_release.pom", "dst": "download.flutter.io/io/flutter/x86_64_release/1.0.0-$engine/x86_64_release-1.0.0-$engine.pom"}, - {"src": "x86_64_release.jar", "dst": "download.flutter.io/io/flutter/x86_64_release/1.0.0-$engine/x86_64_release-1.0.0-$engine.jar"}, - {"src": "x86_64_release.maven-metadata.xml", "dst": "download.flutter.io/io/flutter/x86_64_release/1.0.0-$engine/x86_64_release-1.0.0-$engine.maven-metadata.xml"} + {"src": "android_release_x64/zip_archives/android-x64-release/artifacts.zip", "dst": "flutter_infra_release/flutter/$engine/android-x64-release/artifacts.zip"}, + {"src": "android_release_x64/zip_archives/android-x64-release/linux-x64.zip", "dst": "flutter_infra_release/flutter/$engine/android-x64-release/linux-x64.zip"}, + {"src": "android_release_x64/zip_archives/android-x64-release/symbols.zip", "dst": "flutter_infra_release/flutter/$engine/android-x64-release/symbols.zip"}, + {"src": "android_release_x64/x86_64_release.pom", "dst": "download.flutter.io/io/flutter/x86_64_release/1.0.0-$engine/x86_64_release-1.0.0-$engine.pom"}, + {"src": "android_release_x64/x86_64_release.jar", "dst": "download.flutter.io/io/flutter/x86_64_release/1.0.0-$engine/x86_64_release-1.0.0-$engine.jar"}, + {"src": "android_release_x64/x86_64_release.maven-metadata.xml", "dst": "download.flutter.io/io/flutter/x86_64_release/1.0.0-$engine/x86_64_release-1.0.0-$engine.maven-metadata.xml"} ] }, "host": { diff --git a/shorebird/ci/shards/macos.json b/shorebird/ci/shards/macos.json index d27d7d8a434d0..19173eecfa486 100644 --- a/shorebird/ci/shards/macos.json +++ b/shorebird/ci/shards/macos.json @@ -136,7 +136,8 @@ ], "compose_input": "macos-framework", "artifacts": [ - {"src": "host_release_arm64/dart-sdk", "dst": "flutter_infra_release/flutter/$engine/dart-sdk-darwin-arm64.zip", "zip": true, "content_hash": true} + {"src": "host_release_arm64/dart-sdk", "dst": "flutter_infra_release/flutter/$engine/dart-sdk-darwin-arm64.zip", "zip": true, "content_hash": true}, + {"src": "mac_release_arm64/zip_archives/darwin-arm64-release/artifacts.zip", "dst": "flutter_infra_release/flutter/$engine/darwin-x64-release/artifacts.zip"} ] }, "mac-x64": { diff --git a/shorebird/ci/shards/windows.json b/shorebird/ci/shards/windows.json index 5775e5d0a1ad0..11183cb68611e 100644 --- a/shorebird/ci/shards/windows.json +++ b/shorebird/ci/shards/windows.json @@ -9,7 +9,7 @@ } ], "artifacts": [ - {"src": "zip_archives/android-arm64-release/windows-x64.zip", "dst": "flutter_infra_release/flutter/$engine/android-arm64-release/windows-x64.zip"} + {"src": "android_release_arm64/zip_archives/android-arm64-release/windows-x64.zip", "dst": "flutter_infra_release/flutter/$engine/android-arm64-release/windows-x64.zip"} ] }, "android-arm32": { @@ -22,7 +22,7 @@ } ], "artifacts": [ - {"src": "zip_archives/android-arm-release/windows-x64.zip", "dst": "flutter_infra_release/flutter/$engine/android-arm-release/windows-x64.zip"} + {"src": "android_release/zip_archives/android-arm-release/windows-x64.zip", "dst": "flutter_infra_release/flutter/$engine/android-arm-release/windows-x64.zip"} ] }, "android-x64": { @@ -35,7 +35,7 @@ } ], "artifacts": [ - {"src": "zip_archives/android-x64-release/windows-x64.zip", "dst": "flutter_infra_release/flutter/$engine/android-x64-release/windows-x64.zip"} + {"src": "android_release_x64/zip_archives/android-x64-release/windows-x64.zip", "dst": "flutter_infra_release/flutter/$engine/android-x64-release/windows-x64.zip"} ] }, "host": { From 19809107231eb3effca30f51c76f174db4c037e1 Mon Sep 17 00:00:00 2001 From: Eric Seidel Date: Fri, 8 May 2026 10:57:17 -0700 Subject: [PATCH 58/95] ci(shorebird): add shell steps for engine_stamp.json + aot-tools.dill (#147) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Both files are listed as shard artifacts in the new sharded build but neither is actually produced โ€” they were built outside the gn+ninja graph in the legacy shell scripts (mac_build.sh / linux_build.sh) and the sharded pipeline only knew about gn_ninja and rust step types. shorebirdtech/_build_engine#209 adds a `shell` step type to shard_runner. This PR uses it: - mac-x64: new shell step runs `flutter/bin/et stamp` (writes out/engine_stamp.json) and copies the result into out/mac_release/engine_stamp.json so it ends up in the shard's tarball. The matching artifact src updates from "engine_stamp.json" to "mac_release/engine_stamp.json". - linux/host: new shell step runs the host-built dart against flutter/third_party/dart/pkg/aot_tools to produce out/host_release/aot_tools/aot-tools.dill, matching what linux_build.sh:78-90 does. The existing artifact src ("host_release/aot_tools/aot-tools.dill") is already correct. After this lands and an engine bump produces a SHA carrying it, finalize will upload these two missing files and the sharded build's prod artifact set should match the legacy build's exactly (modulo iOS XCFramework signing). --- shorebird/ci/shards/linux.json | 4 ++++ shorebird/ci/shards/macos.json | 6 +++++- 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/shorebird/ci/shards/linux.json b/shorebird/ci/shards/linux.json index 435ec9bebb7b7..ffd1b4982dbda 100644 --- a/shorebird/ci/shards/linux.json +++ b/shorebird/ci/shards/linux.json @@ -91,6 +91,10 @@ "gn_args": ["--no-prebuilt-dart-sdk"], "ninja_targets": ["flutter/build/archives:artifacts"], "out_dir": "host_debug" + }, + { + "type": "shell", + "command": "mkdir -p out/host_release/aot_tools && out/host_release/dart-sdk/bin/dart pub get --offline --directory=flutter/third_party/dart/pkg/aot_tools && out/host_release/dart-sdk/bin/dart compile kernel flutter/third_party/dart/pkg/aot_tools/bin/aot_tools.dart -o out/host_release/aot_tools/aot-tools.dill" } ], "artifacts": [ diff --git a/shorebird/ci/shards/macos.json b/shorebird/ci/shards/macos.json index 19173eecfa486..a6b7cf37016ab 100644 --- a/shorebird/ci/shards/macos.json +++ b/shorebird/ci/shards/macos.json @@ -157,12 +157,16 @@ "gn_args": ["--runtime-mode=release", "--mac", "--mac-cpu=x64"], "ninja_targets": ["flutter/shell/platform/darwin/macos:zip_macos_flutter_framework", "flutter/lib/snapshot:generate_snapshot_bins", "flutter/build/archives:artifacts"], "out_dir": "mac_release" + }, + { + "type": "shell", + "command": "flutter/bin/et stamp && cp out/engine_stamp.json out/mac_release/engine_stamp.json" } ], "compose_input": "macos-framework", "artifacts": [ {"src": "host_release/dart-sdk", "dst": "flutter_infra_release/flutter/$engine/dart-sdk-darwin-x64.zip", "zip": true, "content_hash": true}, - {"src": "engine_stamp.json", "dst": "flutter_infra_release/flutter/$engine/engine_stamp.json"} + {"src": "mac_release/engine_stamp.json", "dst": "flutter_infra_release/flutter/$engine/engine_stamp.json"} ] } } From 4f1a8bb597fc09acc375777299ec54876cad6a37 Mon Sep 17 00:00:00 2001 From: Eric Seidel Date: Fri, 8 May 2026 12:30:33 -0700 Subject: [PATCH 59/95] ci(shorebird): drop --offline from aot-tools pub get (#148) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The legacy linux_build.sh used `pub get --offline` because gclient hooks pre-populated ~/.pub-cache (run_pub_get.py runs as part of gclient sync). The new sharded path runs run_pub_get.py only in the sync-linux job; per-shard runners have their own ephemeral ~/.pub-cache, so by the time the shell step runs in linux/host the aot_tools deps (e.g. very_good_analysis) aren't cached: Because aot_tools depends on very_good_analysis any which doesn't exist (could not find package very_good_analysis in cache), version solving failed. Drop --offline. pub get will fetch from pub.dev in a few seconds โ€” trivial overhead vs pre-populating the cache or sharing pub cache via namespace cache. Verified failure in run 25571102939 (linux/host). --- shorebird/ci/shards/linux.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/shorebird/ci/shards/linux.json b/shorebird/ci/shards/linux.json index ffd1b4982dbda..5f1b597ceabfe 100644 --- a/shorebird/ci/shards/linux.json +++ b/shorebird/ci/shards/linux.json @@ -94,7 +94,7 @@ }, { "type": "shell", - "command": "mkdir -p out/host_release/aot_tools && out/host_release/dart-sdk/bin/dart pub get --offline --directory=flutter/third_party/dart/pkg/aot_tools && out/host_release/dart-sdk/bin/dart compile kernel flutter/third_party/dart/pkg/aot_tools/bin/aot_tools.dart -o out/host_release/aot_tools/aot-tools.dill" + "command": "mkdir -p out/host_release/aot_tools && out/host_release/dart-sdk/bin/dart pub get --directory=flutter/third_party/dart/pkg/aot_tools && out/host_release/dart-sdk/bin/dart compile kernel flutter/third_party/dart/pkg/aot_tools/bin/aot_tools.dart -o out/host_release/aot_tools/aot-tools.dill" } ], "artifacts": [ From 9ec90c4a6a86bcc9bce3e488cbf2b03e5f08126a Mon Sep 17 00:00:00 2001 From: Eric Seidel Date: Fri, 8 May 2026 12:42:00 -0700 Subject: [PATCH 60/95] Revert "ci(shorebird): drop --offline from aot-tools pub get" (#148) (#149) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Restores the legacy `pub get --offline` form. The original failure (missing very_good_analysis in shard pub cache) was a shard-environment problem โ€” sync.yaml ran tools/run_pub_get.py to populate the cache, but shards' pub cache is per-runner ephemeral. Fixed properly in shorebirdtech/_build_engine#209 (shard-build.yaml now also runs run_pub_get.py post-sync), so shell steps can rely on --offline matching legacy behavior. The JSON config shouldn't carry the env-prep detail. (See FIXME in build_engine's shard-build.yaml: real fix is to pull aot_tools deps via gclient like every other in-tree Dart package.) --- shorebird/ci/shards/linux.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/shorebird/ci/shards/linux.json b/shorebird/ci/shards/linux.json index 5f1b597ceabfe..ffd1b4982dbda 100644 --- a/shorebird/ci/shards/linux.json +++ b/shorebird/ci/shards/linux.json @@ -94,7 +94,7 @@ }, { "type": "shell", - "command": "mkdir -p out/host_release/aot_tools && out/host_release/dart-sdk/bin/dart pub get --directory=flutter/third_party/dart/pkg/aot_tools && out/host_release/dart-sdk/bin/dart compile kernel flutter/third_party/dart/pkg/aot_tools/bin/aot_tools.dart -o out/host_release/aot_tools/aot-tools.dill" + "command": "mkdir -p out/host_release/aot_tools && out/host_release/dart-sdk/bin/dart pub get --offline --directory=flutter/third_party/dart/pkg/aot_tools && out/host_release/dart-sdk/bin/dart compile kernel flutter/third_party/dart/pkg/aot_tools/bin/aot_tools.dart -o out/host_release/aot_tools/aot-tools.dill" } ], "artifacts": [ From 3c26ef9095181645c3d4138f116e15560a39f7ca Mon Sep 17 00:00:00 2001 From: Eric Seidel Date: Fri, 8 May 2026 14:55:54 -0700 Subject: [PATCH 61/95] ci(shorebird): rename macos-framework dst to framework.zip (#150) create_macos_framework.py outputs FlutterMacOS.framework.zip locally, but legacy uploads it to .../darwin-x64-release/framework.zip in the public bucket. Sharded was uploading it as FlutterMacOS.framework.zip, so the legacy/sharded bucket compare showed framework.zip only-in- legacy and FlutterMacOS.framework.zip only-in-sharded for the same artifact. src stays as FlutterMacOS.framework.zip (the actual local filename); only the published dst path changes. --- shorebird/ci/compose.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/shorebird/ci/compose.json b/shorebird/ci/compose.json index f5f2437b8ab4c..7e25d3fc65540 100644 --- a/shorebird/ci/compose.json +++ b/shorebird/ci/compose.json @@ -22,7 +22,7 @@ "--x64-out-dir": "mac_release" }, "artifacts": [ - {"src": "FlutterMacOS.framework.zip", "dst": "flutter_infra_release/flutter/$engine/darwin-x64-release/FlutterMacOS.framework.zip"}, + {"src": "FlutterMacOS.framework.zip", "dst": "flutter_infra_release/flutter/$engine/darwin-x64-release/framework.zip"}, {"src": "FlutterMacOS.framework.dSYM.zip", "dst": "flutter_infra_release/flutter/$engine/darwin-x64/FlutterMacOS.framework.dSYM.zip"} ] }, From 4aaa1112dea40efe526ef131f034dd76b1988b8a Mon Sep 17 00:00:00 2001 From: Eric Seidel Date: Fri, 8 May 2026 16:19:03 -0700 Subject: [PATCH 62/95] ci(shorebird): use framework.zip as macos-framework src (xcframework, not single-arch) (#151) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit create_macos_framework.py with --zip produces two outputs in dst/: - FlutterMacOS.framework.zip โ€” single-arch, double-zipped framework (~17 MB) - framework.zip โ€” the xcframework (both arm64 + x64) plus codesign config files (entitlements.txt, without_entitlements.txt, unsigned_binaries.txt) (~122 MB) Legacy mac_upload.sh uploads framework.zip (the xcframework). Compose was uploading the single-arch FlutterMacOS.framework.zip and renaming it to framework.zip in the bucket via the dst path โ€” same name on disk, completely different bytes. Verified by bucket-comparing engine 93865d602479 (sharded, post-#150) against legacy 25c9b21d638f at the same darwin-x64-release/framework.zip path: 17,237,143 bytes vs 121,918,469 bytes. After this lands, the sharded artifact at that path should be the xcframework matching legacy in size and contents. --- shorebird/ci/compose.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/shorebird/ci/compose.json b/shorebird/ci/compose.json index 7e25d3fc65540..5cda861198e33 100644 --- a/shorebird/ci/compose.json +++ b/shorebird/ci/compose.json @@ -22,7 +22,7 @@ "--x64-out-dir": "mac_release" }, "artifacts": [ - {"src": "FlutterMacOS.framework.zip", "dst": "flutter_infra_release/flutter/$engine/darwin-x64-release/framework.zip"}, + {"src": "framework.zip", "dst": "flutter_infra_release/flutter/$engine/darwin-x64-release/framework.zip"}, {"src": "FlutterMacOS.framework.dSYM.zip", "dst": "flutter_infra_release/flutter/$engine/darwin-x64/FlutterMacOS.framework.dSYM.zip"} ] }, From 56b3b048ad13579c0408bb0dd1aedd96275636ea Mon Sep 17 00:00:00 2001 From: Eric Seidel Date: Tue, 12 May 2026 19:37:04 -0700 Subject: [PATCH 63/95] shorebird/ci: drop iOS extension-safe shards from the engine build (#152) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Removes the three -ext shards (ios-release-ext, ios-sim-x64-ext, ios-sim-arm64-ext) from macos.json and from ios-framework's `requires`. Adds --no-extension-safe-frameworks to the compose flags so create_ios_framework.py doesn't try to read the now-missing *_extension_safe out_dirs. ## Why this is safe In upstream Flutter PR flutter/engine#165346 ("Make iOS Flutter framework extension-safe", March 2025), Victoria Ashworth removed the two-framework design. Before that PR, BUILD.gn gated `-fapplication-extension` and `APPLICATION_EXTENSION_API_ONLY=1` on `if (darwin_extension_safe)`. After it, the `ios_application_extension` gn config is applied unconditionally to every iOS Flutter framework target. Non-extension-safe API calls are guarded at runtime with `NS_EXTENSION_UNAVAILABLE_IOS` annotations. In our current engine (post-PR-165346), the gn arg `darwin_extension_safe` is set when `--darwin-extension-safe` is passed but read by zero BUILD.gn files. Verified: $ grep -rn darwin_extension_safe engine/src engine/src/flutter/tools/gn:62: if args.darwin_extension_safe: engine/src/flutter/tools/gn:839: if args.darwin_extension_safe: engine/src/flutter/tools/gn:840: gn_args['darwin_extension_safe'] = True The only thing the flag still does is append `_extension_safe` to the out_dir. The binaries produced in `ios_release/` and `ios_release_extension_safe/` are byte-identical modulo embedded paths. Upstream Flutter's CI still builds both (see mac_ios_engine.json) and `create_ios_framework.py` still bundles the duplicate into the shipped xcframework by default โ€” but that's dead infrastructure nobody cleaned up. flutter/engine#168955 added `--no-extension-safe-frameworks` as an opt-out for DDM builds; we now use it for all our iOS composes. ## What clients see Customers who manually integrate Flutter.xcframework into an Xcode project and pinned `extension_safe/Flutter.xcframework/...` in their project files will need to drop the `extension_safe/` segment and point at `Flutter.xcframework/...` instead. Same bytes; just one less path indirection. We expect this audience is small (most Shorebird users go through `flutter` + `shorebird` CLI which only references the top-level `Flutter.xcframework`). ## If that breaks more people than expected We can restore source-level backwards compatibility with a single symlink in `create_ios_framework.py`: if args.include_extension_safe_frameworks: os.makedirs(os.path.join(dst, 'extension_safe'), exist_ok=True) os.symlink('../Flutter.xcframework', os.path.join(dst, 'extension_safe', 'Flutter.xcframework')) `sky_utils.create_zip` already uses `zip -y` (symlink-preserving) and macOS `unzip` preserves them on extract. Since iOS artifacts are only ever pulled on macOS hosts, cross-platform symlink portability isn't a concern. We deliberately picked the simpler path of dropping the duplicate entirely rather than maintaining a fork patch; this comment is here so we can revisit if needed. ## CI impact Drops 3 of the 9 macOS shards (~50 min of CPU per engine build saved) and ships an `ios-release/artifacts.zip` without the duplicate `extension_safe/Flutter.xcframework` directory (~half the iOS Flutter binary's bytes per release zip). The shorebird artifact_proxy needs no change: it operates on whole-zip URLs, not on paths inside the zip, so the absence of `extension_safe/` in the unzipped tree never reaches it. Coordinated with shorebirdtech/_build_engine matrix change (drops the three -ext entries from the build-macos strategy matrix). Refs: - flutter/engine#165346 (Make iOS Flutter framework extension-safe) - flutter/engine#168955 (Add --no-extension-safe-frameworks flag) - flutter/engine#142531 (original "nesting bundles disallowed" issue) --- shorebird/ci/compose.json | 4 ++-- shorebird/ci/shards/macos.json | 41 ---------------------------------- 2 files changed, 2 insertions(+), 43 deletions(-) diff --git a/shorebird/ci/compose.json b/shorebird/ci/compose.json index 5cda861198e33..494ad46cc0f23 100644 --- a/shorebird/ci/compose.json +++ b/shorebird/ci/compose.json @@ -1,8 +1,8 @@ { "ios-framework": { - "requires": ["ios-release", "ios-release-ext", "ios-sim-x64", "ios-sim-x64-ext", "ios-sim-arm64", "ios-sim-arm64-ext"], + "requires": ["ios-release", "ios-sim-x64", "ios-sim-arm64"], "script": "flutter/sky/tools/create_ios_framework.py", - "flags": ["--dsym", "--strip"], + "flags": ["--dsym", "--strip", "--no-extension-safe-frameworks"], "path_args": { "--arm64-out-dir": "ios_release", "--simulator-x64-out-dir": "ios_debug_sim", diff --git a/shorebird/ci/shards/macos.json b/shorebird/ci/shards/macos.json index a6b7cf37016ab..0d1336c2be69f 100644 --- a/shorebird/ci/shards/macos.json +++ b/shorebird/ci/shards/macos.json @@ -48,21 +48,6 @@ ], "compose_input": "ios-framework" }, - "ios-release-ext": { - "steps": [ - { - "type": "rust", - "targets": ["aarch64-apple-ios"] - }, - { - "type": "gn_ninja", - "gn_args": ["--ios", "--runtime-mode=release", "--darwin-extension-safe", "--xcode-symlinks", "--gn-arg=shorebird_runtime=true"], - "ninja_targets": ["flutter/shell/platform/darwin/ios:flutter_framework", "flutter/lib/snapshot:generate_snapshot_bins"], - "out_dir": "ios_release_extension_safe" - } - ], - "compose_input": "ios-framework" - }, "ios-sim-x64": { "steps": [ { @@ -78,21 +63,6 @@ ], "compose_input": "ios-framework" }, - "ios-sim-x64-ext": { - "steps": [ - { - "type": "rust", - "targets": ["x86_64-apple-ios"] - }, - { - "type": "gn_ninja", - "gn_args": ["--ios", "--runtime-mode=debug", "--darwin-extension-safe", "--simulator"], - "ninja_targets": ["flutter/shell/platform/darwin/ios:flutter_framework", "flutter/lib/snapshot:generate_snapshot_bins"], - "out_dir": "ios_debug_sim_extension_safe" - } - ], - "compose_input": "ios-framework" - }, "ios-sim-arm64": { "steps": [ { @@ -104,17 +74,6 @@ ], "compose_input": "ios-framework" }, - "ios-sim-arm64-ext": { - "steps": [ - { - "type": "gn_ninja", - "gn_args": ["--ios", "--runtime-mode=debug", "--darwin-extension-safe", "--simulator", "--simulator-cpu=arm64"], - "ninja_targets": ["flutter/shell/platform/darwin/ios:flutter_framework", "flutter/lib/snapshot:generate_snapshot_bins"], - "out_dir": "ios_debug_sim_arm64_extension_safe" - } - ], - "compose_input": "ios-framework" - }, "mac-arm64": { "steps": [ { From 01e85a4ce39400356e8acf49fd0a74b67eaf6f46 Mon Sep 17 00:00:00 2001 From: Brandon DeRosier Date: Wed, 13 May 2026 19:30:54 -0700 Subject: [PATCH 64/95] feat: add 2-pass release build for Dynamic Dispatch table (#119) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Add 2-pass release build for Dynamic Dispatch table When building for arm64 Apple platforms with the linker enabled, run gen_snapshot twice: first in ELF mode to produce a temporary snapshot for analyze_snapshot to compute the DD table manifest, caller links, and slot mapping, then in assembly mode with --dd_slot_mapping to produce the final snapshot with indirect calls wired up. The DD table files (App.dd.link, App.dd_callers.link) are copied into the shorebird supplement directory alongside the existing link files so they can be bundled with releases and used during patch builds. * Add DD analysis gen_snapshot command to test expectations The 2-pass DD table build runs gen_snapshot in ELF mode before the main assembly pass. Update tests to expect this additional command. * Fix macOS universal binary test command order for concurrent DD build The arm64 DD analysis pass delays the arm64 assembly, so x86_64 (which skips DD) completes its build first when both run concurrently via Future.wait. * Skip DD table computation when analyze_snapshot is absent Move the analyze_snapshot existence check before the gen_snapshot ELF pass so the entire DD computation is a no-op on standard Flutter SDKs that don't ship analyze_snapshot. This fixes iOS smoke test failures where gen_snapshot was being invoked unnecessarily in ELF mode. Also reverts test changes that are no longer needed since DD commands won't appear when analyze_snapshot doesn't exist in the test filesystem. * Fix macOS universal binary test command order for DD table async The arm64 build's async _computeDDTable() check (even a no-op when analyze_snapshot is absent) introduces an await that lets x86_64 reach gen_snapshot first in Future.wait. Reorder test expectations to match the actual interleaving: x86_64 before arm64 at each step. * fix: pass DD function identity file in base build pipeline The DD slot mapping now uses kernel_offset-based function matching (instead of function names). The base build must export an identity side file during gen_snapshot pass 1 and pass it to analyze_snapshot --compute_dd_slot_mapping. Without this, the DDSlotMapping has empty kernel_offset_to_slot and FinalizeIndirectStaticCallTable can't assign any DD slots, resulting in an empty DD table in the base snapshot. * feat: make DD table max bytes configurable via environment variable Read SHOREBIRD_DD_MAX_BYTES from the environment to allow overriding the cascade limiter threshold. Defaults to 10000 if not set. An environment variable is used (rather than a command-line flag) so that older Flutter builds without DD table support silently ignore it. * chore: bump dart_sdk_revision to cascade-limiter Updates dart_sdk_revision to include SIMARM64 simulator fixes (DoRedirectedCall, ClobberVolatileRegisters, Execute reason param) needed for ios_debug engine builds. * chore: checkpoint current DD table base build changes Snapshot of work in progress on the base build's DD pipeline: - debug print in AOTSnapshotter.build for usesDDTable diagnosis - BUILD.gn change to place analyze_snapshot in universal/ next to gen_snapshot so flutter_tools can resolve it by path substitution Committing to a clean base before restructuring the pipeline to support the pre-DD optimized pass added in the dart-sdk patch flow. * fix: probe gen_snapshot for DD flag support before running DD pipeline When shorebird_flutter ships with the DD table 2-pass release build enabled but the underlying engine's gen_snapshot binary predates the DD table work (e.g. a user has downgraded their engine cache, or is running against a Shorebird release from before DD landed), gen_snapshot hard-errors on the ELF pass with "Setting VM flags failed: Unrecognized flags: print_dd_function_identity_to" and the entire release build fails. Older engines simply don't know about `--print_dd_function_identity_to`, `--dd_slot_mapping`, or any of the DD table flag family. Add a capability probe in _computeDDTable that runs gen_snapshot once with the DD flag plus a bogus kernel input. Two possible failure modes distinguish support: - Flag recognized โ†’ flag parsing passes, kernel load fails: "Can't load Kernel binary: File size is too small to be a valid kernel file." - Flag not recognized โ†’ VM init fails at flag parsing: "Setting VM flags failed: Unrecognized flags: print_dd_function_identity_to" If stderr contains the "Unrecognized flags" token, skip the entire DD pipeline and fall back to a plain single-pass Shorebird release build. The cascade-limiter linker (in aot_tools) independently handles the no-DD case by falling back to the CT pass's op.link for final pass OP alignment (see dart-sdk commit 139dd8c864d), so the patch side of the pipeline keeps working too. Probe strategies that don't work: - `gen_snapshot --help` doesn't list individual flags. - `gen_snapshot --print_flags` exits early on "At least one input is required" before dumping any flag info. - Passing the flag without a snapshot kind and kernel silently ignores it regardless of support. The flag+bogus-kernel+snapshot-kind combination is the only invocation that reliably distinguishes recognized-but-failed-later from outright-rejected, on both DD-aware and pre-DD engines. Result is cached per gen_snapshot path so multi-arch release builds don't pay the probe cost more than once. * feat: DD 2-pass release build for cascade limiter When SHOREBIRD_DD_MAX_BYTES is set, AOTSnapshotter now performs a 2-pass build: 1. Pass 1: gen_snapshot produces an ELF for analysis + DD identity file 2. analyze_snapshot computes DD table + caller links + slot mapping 3. Pass 2: gen_snapshot rebuilds with --dd_slot_mapping for DD-enabled code This produces DD-aware release snapshots where high-fanout cascade functions are routed through the indirect static call table, enabling the cascade limiter's link percentage benefit. Also adds DD supplement files (App.dd.link, App.dd_callers.link, App.dd_identity.link, App.dd_slots.link) to the LinkSupplement copy list so they're propagated to the Shorebird CLI's supplement directory. * fix: address flutter_tools test failures - build_test.dart: GenSnapshot now requires a `fileSystem` parameter (added by this branch for the analyze_snapshot probe path); pass `MemoryFileSystem.test()` to the test constructor. - macos_test.dart: the DD pass is gated on SHOREBIRD_DD_MAX_BYTES, which isn't set in the test environment, so neither arm64 nor x86_64 has an extra async hop before gen_snapshot. With concurrent Future.wait the archs now reach gen_snapshot in iteration order (arm64 first, x86_64 second). Update the expected command sequence and the explanatory comment to match. * review: address build.dart feedback (dead field, fallback, helper extraction) Three changes from Eric's review: 1. Drop the unused `reportTimings` field on AOTSnapshotter โ€” added in 81304a99b27 but never read anywhere. 2. Replace the analyze_snapshot fallback iteration with explicit arch-based naming, mirroring how GenSnapshot.run picks its binary. 3. Extract the inlined DD 2-pass logic out of build() into _runDdAnalysisPass and _readDdMaxBytes, and collapse the five identical path joins into a local linkPath() helper. * revert: drop universal/ analyze_snapshot move; probe parent dir instead Reverts the snapshot BUILD.gn change that moved the lipo'd analyze_snapshot binary into ${root_out_dir}/universal/. That move broke create_ios_framework.py:96, which still reads analyze_snapshot from the build-dir root, and was only needed to make the flutter_tools getAnalyzeSnapshotPath helper find the binary in local-engine mode (where gen_snapshot resolves via .../universal/...). Instead, getAnalyzeSnapshotPath now probes both dirname(genSnapshot) and dirname(genSnapshot)/.., which finds: * cached SDK layout: analyze_snapshot alongside gen_snapshot (probe 1) * local-engine layout: analyze_snapshot one level up at the build-dir root (probe 2) Keeps engine packaging untouched and avoids coupling this PR to the Shorebird CLI's view of the published artifact layout. * chore: bump dart_sdk_revision to 59680e070c3 (cascade-limiter HEAD) * chore: bump dart_sdk_revision to f9f552a6e77 (DD VERIFY all-slots fix) * chore: bump dart_sdk_revision to 109ff541113 (DD sentinel-fill) * chore: bump dart_sdk_revision to afa77ceb273 (DD sentinel-fill amend) * fix: harden DD analysis pass โ€” version-match analyze_snapshot, fail loudly Two related hardenings to prevent silent DD-disable regressions like the one we just chased on Wonderous (link percentage cratered from 90% to 22.6% because a stale universal/analyze_snapshot binary from a prior BUILD.gn revision was selected, analyze_snapshot rejected gen_snapshot's output with "Wrong full snapshot version", and the DD pass swallowed the failure): 1. getAnalyzeSnapshotPath now verifies the candidate analyze_snapshot's --sdk_version output matches gen_snapshot's --version output. The strings include the build timestamp, so two binaries from the same build agree exactly. A stale binary left behind by a previous build configuration prints a different version line and is skipped. If the version probe itself fails (e.g. binary won't run), fall back to the exists-only check rather than fail-closed. 2. _runDdAnalysisPass now checks exit codes from both analyze_snapshot --compute_dd_table and --compute_dd_slot_mapping invocations, and from the post-conditions on the produced files, surfacing each failure with a concrete printError + non-zero return. Previously the calls used await without an exit-code check, so any failure (snapshot version skew, missing input, OOM, etc.) silently left App.dd.link unwritten and the build proceeded to gen_snapshot pass 2 with no --dd_slot_mapping. The release would publish without DD activation; subsequent patches would fall back to on-the-fly DD computation against a no-DD base and produce structurally divergent patches with devastating link percentages. Also prefer aborting early when getAnalyzeSnapshotPath returns null (versus silently shipping no DD), with a message that explains the patching consequence. * chore: bump dart_sdk_revision to 49005fce564 (May 7 review fixes) * chore: bump dart_sdk_revision to 93ded8b64ac (review #5/#7/#9/#10/#11) * chore: bump dart_sdk_revision to c1f23751d53 (post-rebase) * feat(flutter_tools): pass --print_dd_resolution_to and copy into supplement Wire gen_snapshot's new --print_dd_resolution_to flag (dart-sdk companion commit) through AOTSnapshotter._runDdAnalysisPass: the 2-pass DD release build now writes App.dd_resolution.tsv next to the other DD link files, and LinkSupplement.create copies it into build//shorebird/ so it gets bundled into the release supplement upload (and downstream into the patch debug zip). The file is a TSV diagnostic dump of per-slot DD table resolution outcomes โ€” useful for figuring out which specific slots ended up dropped during DD resolution when investigating patch link percentage regressions. Inert when SHOREBIRD_DD_MAX_BYTES is unset (2-pass DD build doesn't run, no file produced). * chore: dart format build.dart (CI format check) * chore: bump dart_sdk_revision to f3656bfc678 (post-merge of #785) --- .../flutter_tools/lib/src/base/build.dart | 407 +++++++++++++++--- .../lib/src/build_system/targets/common.dart | 16 +- .../test/general.shard/base/build_test.dart | 1 + .../build_system/targets/macos_test.dart | 9 + 4 files changed, 363 insertions(+), 70 deletions(-) diff --git a/packages/flutter_tools/lib/src/base/build.dart b/packages/flutter_tools/lib/src/base/build.dart index 8c355499634c7..8dfb32e070d10 100644 --- a/packages/flutter_tools/lib/src/base/build.dart +++ b/packages/flutter_tools/lib/src/base/build.dart @@ -2,6 +2,8 @@ // Use of this source code is governed by a BSD-style license that can be // found in the LICENSE file. +import 'dart:io' show Platform; + import 'package:process/process.dart'; import '../artifacts.dart'; @@ -29,7 +31,9 @@ class GenSnapshot { required Artifacts artifacts, required ProcessManager processManager, required Logger logger, + required FileSystem fileSystem, }) : _artifacts = artifacts, + _fileSystem = fileSystem, _processUtils = ProcessUtils(logger: logger, processManager: processManager); final Artifacts _artifacts; @@ -54,6 +58,75 @@ class GenSnapshot { ' See dartbug.com/30524 for more information.', }; + /// Returns the path to an analyze_snapshot binary that matches gen_snapshot's + /// SDK version, or null if no matching binary can be found. + /// + /// gen_snapshot and analyze_snapshot share a snapshot-format hash baked into + /// each binary; if the hashes disagree, analyze_snapshot rejects gen_snapshot's + /// output with "Wrong full snapshot version" at parse time. Pre-existing binary + /// layouts (e.g. an older `universal/analyze_snapshot_` left behind from + /// a previous BUILD.gn revision) can shadow the freshly built one. Resolve by + /// probing every candidate location and returning only the one whose + /// `--sdk_version` matches gen_snapshot's `--version` output. + String? getAnalyzeSnapshotPath(SnapshotType snapshotType, DarwinArch? darwinArch) { + final Artifact genSnapshotArtifact; + final String analyzeName; + if (snapshotType.platform == TargetPlatform.ios || + snapshotType.platform == TargetPlatform.darwin) { + if (darwinArch == DarwinArch.arm64) { + genSnapshotArtifact = Artifact.genSnapshotArm64; + analyzeName = 'analyze_snapshot_arm64'; + } else { + genSnapshotArtifact = Artifact.genSnapshotX64; + analyzeName = 'analyze_snapshot_x64'; + } + } else { + genSnapshotArtifact = Artifact.genSnapshot; + analyzeName = 'analyze_snapshot'; + } + final String genSnapshotPath = getSnapshotterPath(snapshotType, genSnapshotArtifact); + final String? genVersion = _probeSdkVersion(genSnapshotPath, '--version'); + final String dir = _fileSystem.path.dirname(genSnapshotPath); + // Cached SDK layout puts analyze_snapshot alongside gen_snapshot. Local + // engine layout resolves gen_snapshot via .../universal/gen_snapshot_arm64 + // while analyze_snapshot lives one level up at the build-dir root. Probe + // both. If we successfully read gen_snapshot's version, accept only a + // candidate whose --sdk_version matches; otherwise fall back to the first + // existing candidate (better than failing closed when the version probe + // itself misbehaves). + for (final candidate in [ + _fileSystem.path.join(dir, analyzeName), + _fileSystem.path.join(dir, '..', analyzeName), + ]) { + if (!_fileSystem.file(candidate).existsSync()) { + continue; + } + if (genVersion == null) { + return candidate; + } + final String? candidateVersion = _probeSdkVersion(candidate, '--sdk_version'); + if (candidateVersion == genVersion) { + return candidate; + } + } + return null; + } + + /// Runs [binary] with [versionFlag] and returns the trimmed stderr output, + /// or null if the binary couldn't be invoked or printed nothing. + /// gen_snapshot and analyze_snapshot both write their version line to stderr. + String? _probeSdkVersion(String binary, String versionFlag) { + try { + final RunResult result = _processUtils.runSync([binary, versionFlag]); + final String stderr = result.stderr.trim(); + return stderr.isEmpty ? null : stderr; + } on Exception { + return null; + } + } + + final FileSystem _fileSystem; + Future run({ required SnapshotType snapshotType, DarwinArch? darwinArch, @@ -95,15 +168,18 @@ class AOTSnapshotter { }) : _logger = logger, _fileSystem = fileSystem, _xcode = xcode, + _processUtils = ProcessUtils(logger: logger, processManager: processManager), _genSnapshot = GenSnapshot( artifacts: artifacts, processManager: processManager, logger: logger, + fileSystem: fileSystem, ); final Logger _logger; final FileSystem _fileSystem; final Xcode _xcode; + final ProcessUtils _processUtils; final GenSnapshot _genSnapshot; /// Builds an architecture-specific ahead-of-time compiled snapshot of the specified script. @@ -159,13 +235,6 @@ class AOTSnapshotter { buildMode == BuildMode.profile || buildMode == BuildMode.release; _logger.printTrace('extractAppleDebugSymbols = $extractAppleDebugSymbols'); - final bool targetingAndroidPlatform = - platform == TargetPlatform.android || - platform == TargetPlatform.android_arm || - platform == TargetPlatform.android_arm64 || - platform == TargetPlatform.android_x64; - _logger.printTrace('targetingAndroidPlatform = $targetingAndroidPlatform'); - // We strip snapshot by default, but allow to suppress this behavior // by supplying --no-strip in extraGenSnapshotOptions. var shouldStrip = true; @@ -180,37 +249,11 @@ class AOTSnapshotter { } } - String aotSharedLibrary = _fileSystem.path.join(outputDir.path, 'app.so'); - String? frameworkPath; + final String assembly = _fileSystem.path.join(outputDir.path, 'snapshot_assembly.S'); if (targetingApplePlatform) { - // On iOS and macOS, we use Xcode to compile the snapshot into a dynamic - // library that the end-developer can link into their app. - const frameworkName = 'App.framework'; - if (!quiet) { - final String targetArch = darwinArch!.name; - _logger.printStatus('Building $frameworkName for $targetArch...'); - } - frameworkPath = _fileSystem.path.join(outputPath, frameworkName); - _fileSystem.directory(frameworkPath).createSync(recursive: true); - - const frameworkSnapshotName = 'App'; - aotSharedLibrary = _fileSystem.path.join(frameworkPath, frameworkSnapshotName); - final String relocatableObject = _fileSystem.path.join(outputPath, 'app.o'); - // When the minimum version is updated, remember to update - // template MinimumOSVersion. - // https://github.com/flutter/flutter/pull/62902 - final minOSVersion = platform == TargetPlatform.ios - ? FlutterDarwinPlatform.ios.deploymentTarget().toString() - : FlutterDarwinPlatform.macos.deploymentTarget().toString(); - genSnapshotArgs.addAll([ - '--snapshot_kind=app-aot-macho-dylib', - '--macho=$aotSharedLibrary', - '--macho-object=$relocatableObject', - '--macho-min-os-version=$minOSVersion', - '--macho-rpath=@executable_path/Frameworks,@loader_path/Frameworks', - '--macho-install-name=@rpath/$frameworkName/$frameworkSnapshotName', - ]); + genSnapshotArgs.addAll(['--snapshot_kind=app-aot-assembly', '--assembly=$assembly']); } else { + final String aotSharedLibrary = _fileSystem.path.join(outputDir.path, 'app.so'); genSnapshotArgs.addAll(['--snapshot_kind=app-aot-elf', '--elf=$aotSharedLibrary']); } @@ -222,9 +265,6 @@ class AOTSnapshotter { if (stripAfterBuild) { _logger.printTrace('Will strip AOT snapshot manually after build and dSYM generation.'); } - } else if (targetingAndroidPlatform) { - stripAfterBuild = false; - // When building for Android, we let AGP handle stripping of debug symbols. } else { stripAfterBuild = false; if (shouldStrip) { @@ -265,6 +305,22 @@ class AOTSnapshotter { genSnapshotArgs.add(mainPath); final snapshotType = SnapshotType(platform, buildMode); + + final int ddMaxBytes = _readDdMaxBytes(); + if (ddMaxBytes > 0 && usesLinker) { + final int pass1Exit = await _runDdAnalysisPass( + snapshotType: snapshotType, + darwinArch: darwinArch, + outputDir: outputDir, + baseGenSnapshotArgs: genSnapshotArgs, + mainPath: mainPath, + ddMaxBytes: ddMaxBytes, + ); + if (pass1Exit != 0) { + return pass1Exit; + } + } + final int genSnapshotExitCode = await _genSnapshot.run( snapshotType: snapshotType, additionalArgs: genSnapshotArgs, @@ -275,38 +331,256 @@ class AOTSnapshotter { return genSnapshotExitCode; } + // On iOS and macOS, we use Xcode to compile the snapshot into a dynamic library that the + // end-developer can link into their app. if (targetingApplePlatform) { - if (extractAppleDebugSymbols) { - final RunResult dsymResult = await _xcode.dsymutil([ - '-o', - '$frameworkPath.dSYM', - aotSharedLibrary, - ]); - if (dsymResult.exitCode != 0) { + return _buildFramework( + appleArch: darwinArch!, + isIOS: platform == TargetPlatform.ios, + sdkRoot: sdkRoot, + assemblyPath: assembly, + outputPath: outputDir.path, + quiet: quiet, + stripAfterBuild: stripAfterBuild, + extractAppleDebugSymbols: extractAppleDebugSymbols, + ); + } else { + return 0; + } + } + + /// Reads the SHOREBIRD_DD_MAX_BYTES env var (preferring `dart-define` over + /// the process environment). Returns 0 if unset, malformed, or non-positive, + /// which signals "no DD pass." + int _readDdMaxBytes() { + const fromDefine = String.fromEnvironment('SHOREBIRD_DD_MAX_BYTES'); + final String? raw = fromDefine.isNotEmpty + ? fromDefine + : Platform.environment['SHOREBIRD_DD_MAX_BYTES']; + return int.tryParse(raw ?? '') ?? 0; + } + + /// Runs the DD analysis pass: gen_snapshot โ†’ ELF + DD identity, then + /// analyze_snapshot to compute the DD table, caller links, and slot mapping. + /// On success, mutates [baseGenSnapshotArgs] to add `--dd_slot_mapping=...` + /// before [mainPath] so the subsequent gen_snapshot run picks it up. + /// Returns the gen_snapshot pass-1 exit code (0 on success). + Future _runDdAnalysisPass({ + required SnapshotType snapshotType, + required DarwinArch? darwinArch, + required Directory outputDir, + required List baseGenSnapshotArgs, + required String mainPath, + required int ddMaxBytes, + }) async { + _logger.printTrace('DD 2-pass build: dd_max_bytes=$ddMaxBytes'); + + String linkPath(String name) => _fileSystem.path.join(outputDir.parent.path, name); + final String elfForAnalysis = linkPath('App_dd_analysis.so'); + final String ddIdentityPath = linkPath('App.dd_identity.link'); + final String ddTablePath = linkPath('App.dd.link'); + final String ddCallerLinksPath = linkPath('App.dd_callers.link'); + final String ddSlotMappingPath = linkPath('App.dd_slots.link'); + final String ddResolutionPath = linkPath('App.dd_resolution.tsv'); + + // Pass 1: build ELF for analysis + DD identity. Strip the existing snapshot + // kind/output args from the base set; mainPath must remain at the end. + final elfArgs = [ + ...baseGenSnapshotArgs.where( + (String a) => + a != mainPath && + !a.startsWith('--snapshot_kind=') && + !a.startsWith('--assembly=') && + !a.startsWith('--elf='), + ), + '--snapshot_kind=app-aot-elf', + '--elf=$elfForAnalysis', + '--print_dd_function_identity_to=$ddIdentityPath', + mainPath, + ]; + final int pass1Exit = await _genSnapshot.run( + snapshotType: snapshotType, + additionalArgs: elfArgs, + darwinArch: darwinArch, + ); + if (pass1Exit != 0) { + _logger.printError('DD pass 1 (ELF for analysis) failed with exit code $pass1Exit'); + return pass1Exit; + } + + final String? analyzeSnapshotPath = _genSnapshot.getAnalyzeSnapshotPath( + snapshotType, + darwinArch, + ); + if (analyzeSnapshotPath == null) { + _logger.printError( + 'DD pass: could not find an analyze_snapshot binary whose --sdk_version ' + 'matches gen_snapshot. The release will ship without DD activation; ' + 'patches against it will fall back to on-the-fly DD computation and ' + 'produce a structurally divergent snapshot (devastating link percentage). ' + 'Aborting the build instead.', + ); + _fileSystem.file(elfForAnalysis).deleteSync(); + return 1; + } + + final int ddTableExit = await _processUtils.stream([ + analyzeSnapshotPath, + '--compute_dd_table=$ddTablePath', + '--dd_caller_links=$ddCallerLinksPath', + '--dd_max_bytes=$ddMaxBytes', + elfForAnalysis, + ]); + if (ddTableExit != 0) { + _logger.printError( + 'DD pass: analyze_snapshot --compute_dd_table failed with exit code ' + '$ddTableExit. App.dd.link will not be produced and the release would ' + 'ship without DD activation.', + ); + _fileSystem.file(elfForAnalysis).deleteSync(); + return ddTableExit; + } + + final int ddSlotMappingExit = await _processUtils.stream([ + analyzeSnapshotPath, + '--compute_dd_slot_mapping=$ddSlotMappingPath', + '--dd_table_data=$ddTablePath', + '--dd_caller_links=$ddCallerLinksPath', + '--dd_function_identity=$ddIdentityPath', + elfForAnalysis, + ]); + if (ddSlotMappingExit != 0) { + _logger.printError( + 'DD pass: analyze_snapshot --compute_dd_slot_mapping failed with exit ' + 'code $ddSlotMappingExit. The DD slot mapping will not be available and ' + 'gen_snapshot pass 2 would emit a no-DD snapshot.', + ); + _fileSystem.file(elfForAnalysis).deleteSync(); + return ddSlotMappingExit; + } + + if (!_fileSystem.file(ddSlotMappingPath).existsSync()) { + _logger.printError( + 'DD pass: analyze_snapshot --compute_dd_slot_mapping reported success ' + 'but $ddSlotMappingPath was not produced.', + ); + _fileSystem.file(elfForAnalysis).deleteSync(); + return 1; + } + final int mainPathIndex = baseGenSnapshotArgs.indexOf(mainPath); + baseGenSnapshotArgs.insertAll(mainPathIndex, [ + '--dd_slot_mapping=$ddSlotMappingPath', + // Per-slot resolution outcome dump (TSV: slot, outcome, rewritten, + // name). Diagnostic for analyzing which slots got dropped during + // the resolver's run; ends up alongside the other supplement files + // and gets carried into the patch debug bundle. + '--print_dd_resolution_to=$ddResolutionPath', + ]); + _logger.printTrace('DD 2-pass build: added --dd_slot_mapping and --print_dd_resolution_to'); + + _fileSystem.file(elfForAnalysis).deleteSync(); + return 0; + } + + /// Builds an iOS or macOS framework at [outputPath]/App.framework from the assembly + /// source at [assemblyPath]. + Future _buildFramework({ + required DarwinArch appleArch, + required bool isIOS, + String? sdkRoot, + required String assemblyPath, + required String outputPath, + required bool quiet, + required bool stripAfterBuild, + required bool extractAppleDebugSymbols, + }) async { + final String targetArch = appleArch.name; + if (!quiet) { + _logger.printStatus('Building App.framework for $targetArch...'); + } + + final commonBuildOptions = [ + '-arch', + targetArch, + if (isIOS) + // When the minimum version is updated, remember to update + // template MinimumOSVersion. + // https://github.com/flutter/flutter/pull/62902 + '-miphoneos-version-min=${FlutterDarwinPlatform.ios.deploymentTarget()}', + if (sdkRoot != null) ...['-isysroot', sdkRoot], + ]; + + final String assemblyO = _fileSystem.path.join(outputPath, 'snapshot_assembly.o'); + + final RunResult compileResult = await _xcode.cc([ + ...commonBuildOptions, + '-c', + assemblyPath, + '-o', + assemblyO, + ]); + if (compileResult.exitCode != 0) { + _logger.printError( + 'Failed to compile AOT snapshot. Compiler terminated with exit code ${compileResult.exitCode}', + ); + return compileResult.exitCode; + } + + final String frameworkDir = _fileSystem.path.join(outputPath, 'App.framework'); + _fileSystem.directory(frameworkDir).createSync(recursive: true); + final String appLib = _fileSystem.path.join(frameworkDir, 'App'); + final linkArgs = [ + ...commonBuildOptions, + '-dynamiclib', + '-Xlinker', + '-rpath', + '-Xlinker', + '@executable_path/Frameworks', + '-Xlinker', + '-rpath', + '-Xlinker', + '@loader_path/Frameworks', + '-fapplication-extension', + '-install_name', + '@rpath/App.framework/App', + '-o', + appLib, + assemblyO, + ]; + + final RunResult linkResult = await _xcode.clang(linkArgs); + if (linkResult.exitCode != 0) { + _logger.printError( + 'Failed to link AOT snapshot. Linker terminated with exit code ${linkResult.exitCode}', + ); + return linkResult.exitCode; + } + + if (extractAppleDebugSymbols) { + final RunResult dsymResult = await _xcode.dsymutil([ + '-o', + '$frameworkDir.dSYM', + appLib, + ]); + if (dsymResult.exitCode != 0) { + _logger.printError( + 'Failed to generate dSYM - dsymutil terminated with exit code ${dsymResult.exitCode}', + ); + return dsymResult.exitCode; + } + + if (stripAfterBuild) { + // See https://www.unix.com/man-page/osx/1/strip/ for arguments + final RunResult stripResult = await _xcode.strip(['-x', appLib, '-o', appLib]); + if (stripResult.exitCode != 0) { _logger.printError( - 'Failed to generate dSYM - dsymutil terminated with exit code ${dsymResult.exitCode}', + 'Failed to strip debugging symbols from the generated AOT snapshot - strip terminated with exit code ${stripResult.exitCode}', ); - return dsymResult.exitCode; - } - - if (stripAfterBuild) { - // See https://www.unix.com/man-page/osx/1/strip/ for arguments - final RunResult stripResult = await _xcode.strip([ - '-x', - aotSharedLibrary, - '-o', - aotSharedLibrary, - ]); - if (stripResult.exitCode != 0) { - _logger.printError( - 'Failed to strip debugging symbols from the generated AOT snapshot - strip terminated with exit code ${stripResult.exitCode}', - ); - return stripResult.exitCode; - } + return stripResult.exitCode; } - } else { - assert(!stripAfterBuild); } + } else { + assert(!stripAfterBuild); } return 0; @@ -324,7 +598,6 @@ class AOTSnapshotter { TargetPlatform.darwin, TargetPlatform.linux_x64, TargetPlatform.linux_arm64, - TargetPlatform.linux_riscv64, TargetPlatform.windows_x64, TargetPlatform.windows_arm64, ].contains(platform); diff --git a/packages/flutter_tools/lib/src/build_system/targets/common.dart b/packages/flutter_tools/lib/src/build_system/targets/common.dart index 3c26b8548d684..d845655439948 100644 --- a/packages/flutter_tools/lib/src/build_system/targets/common.dart +++ b/packages/flutter_tools/lib/src/build_system/targets/common.dart @@ -547,11 +547,13 @@ abstract final class LinkSupplement { } void maybeCopy(String name) { - final File file = environment.fileSystem.file( - environment.fileSystem.path.join(inputBuildDir, name), - ); + final path = environment.fileSystem.path.join(inputBuildDir, name); + final File file = environment.fileSystem.file(path); if (file.existsSync()) { file.copySync(environment.fileSystem.path.join(shorebirdDir.path, name)); + environment.logger.printTrace('LinkSupplement: copied $name'); + } else { + environment.logger.printTrace('LinkSupplement: missing $name at $path'); } } @@ -563,5 +565,13 @@ abstract final class LinkSupplement { maybeCopy('App.dispatch_table.json'); maybeCopy('App.ft.link'); maybeCopy('App.field_table.json'); + // DD table files for cascade limiter (produced by 2-pass release build + // when SHOREBIRD_DD_MAX_BYTES is set). + maybeCopy('App.dd.link'); + maybeCopy('App.dd_callers.link'); + maybeCopy('App.dd_identity.link'); + maybeCopy('App.dd_slots.link'); + // Per-slot DD resolution outcome diagnostic (TSV). + maybeCopy('App.dd_resolution.tsv'); } } diff --git a/packages/flutter_tools/test/general.shard/base/build_test.dart b/packages/flutter_tools/test/general.shard/base/build_test.dart index eb7df9999b7e6..4781e4cbfac7c 100644 --- a/packages/flutter_tools/test/general.shard/base/build_test.dart +++ b/packages/flutter_tools/test/general.shard/base/build_test.dart @@ -70,6 +70,7 @@ void main() { artifacts: artifacts, logger: logger, processManager: processManager, + fileSystem: MemoryFileSystem.test(), ); }); diff --git a/packages/flutter_tools/test/general.shard/build_system/targets/macos_test.dart b/packages/flutter_tools/test/general.shard/build_system/targets/macos_test.dart index b25a21d873f5d..21d68f11cbb6d 100644 --- a/packages/flutter_tools/test/general.shard/build_system/targets/macos_test.dart +++ b/packages/flutter_tools/test/general.shard/build_system/targets/macos_test.dart @@ -890,7 +890,14 @@ flavors: .createSync(recursive: true); final build = environment.buildDir.path; + // Both archs go through the same code path now (the DD-pass is gated on + // SHOREBIRD_DD_MAX_BYTES which isn't set in this test), so neither + // architecture has an extra async hop. With concurrent Future.wait, the + // archs reach gen_snapshot in iteration order: arm64 first + // (`kDarwinArchs = 'arm64 x86_64'`), x86_64 second. They then + // interleave at each subsequent await point in the same order. processManager.addCommands([ + // arm64 gen_snapshot runs first (iteration order). FakeCommand( command: [ 'Artifact.genSnapshotArm64.TargetPlatform.darwin.release', @@ -901,6 +908,7 @@ flavors: environment.buildDir.childFile('app.dill').path, ], ), + // x86_64 gen_snapshot runs next. FakeCommand( command: [ 'Artifact.genSnapshotX64.TargetPlatform.darwin.release', @@ -911,6 +919,7 @@ flavors: environment.buildDir.childFile('app.dill').path, ], ), + // From here on the two builds interleave: arm64 then x86_64 at each step. FakeCommand( command: [ 'xcrun', From a6ad18a4605108045e06320c7c8ee28092fcd039 Mon Sep 17 00:00:00 2001 From: Brandon DeRosier Date: Thu, 14 May 2026 12:02:17 -0700 Subject: [PATCH 65/95] chore: bump updater_rev to fe37333 for inflate diagnostic counters Picks up shorebirdtech/updater#358 (fix(inflate): preserve real upstream cause + add diagnostic byte counters). The on-device Update failed log will now carry patch_file / base_size / decompressed_into_pipe / decompressed_out_of_pipe / base_read / base_last_seek_pos / base_seeks / output_written counters in its anyhow context chain, instead of the misleading "Decompression of patch failed" message that was masking the real upstream cause. --- DEPS | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/DEPS b/DEPS index f4405cf06e071..d66be3e29bc1e 100644 --- a/DEPS +++ b/DEPS @@ -20,7 +20,7 @@ vars = { "dart_sdk_revision": "b65ce89c8057d6880e00693a7b0ecd7b9e5f61ca", "dart_sdk_git": "git@github.com:shorebirdtech/dart-sdk.git", "updater_git": "https://github.com/shorebirdtech/updater.git", - "updater_rev": "90c349287d170991d1b527b7dac7be4fac508768", + "updater_rev": "fe3733374c222bf7d8ff1913f5ab213421a7d94f", # WARNING: DO NOT EDIT canvaskit_cipd_instance MANUALLY # See `lib/web_ui/README.md` for how to roll CanvasKit to a new version. From c36f88efba35ee1639a19a93e1280b5a61475859 Mon Sep 17 00:00:00 2001 From: Brandon DeRosier Date: Thu, 14 May 2026 13:39:52 -0700 Subject: [PATCH 66/95] debug: log per-blob sizes from SnapshotsDataHandle::createForSnapshots Investigating an in-the-wild iOS standalone patch-apply failure where the on-device synthesized base stream is ~903 KB shorter than the host-side `aot_tools dump_blobs` extraction. The patch is generated against the host bytes; bipatch then reads past EOF on the device and errors out (currently visible via the diagnostic counters in shorebirdtech/updater#358 -- base_size=1864048b on device vs 2767106b host-side). This logs each of the 4 mappings' sizes separately when `createForSnapshots` runs, so the next failure report directly names which `Dart_SnapshotDataSize` / `Dart_SnapshotInstrSize` call returned the wrong value. Same `FML_LOG(INFO)` level the rest of the shorebird-side engine logs use; one extra line per app launch. To be reverted after the root cause is identified. --- .../common/shorebird/snapshots_data_handle.cc | 28 ++++++++++++++++--- 1 file changed, 24 insertions(+), 4 deletions(-) diff --git a/engine/src/flutter/shell/common/shorebird/snapshots_data_handle.cc b/engine/src/flutter/shell/common/shorebird/snapshots_data_handle.cc index 0c6c5a45450a1..1fd6688299a11 100644 --- a/engine/src/flutter/shell/common/shorebird/snapshots_data_handle.cc +++ b/engine/src/flutter/shell/common/shorebird/snapshots_data_handle.cc @@ -86,11 +86,31 @@ std::unique_ptr SnapshotsDataHandle::createForSnapshots( const DartSnapshot& isolate_snapshot) { // This needs to match the order in which the blobs are written out in // analyze_snapshot --dump_blobs + auto vm_data = DataMapping(vm_snapshot); + auto iso_data = DataMapping(isolate_snapshot); + auto vm_insns = InstructionsMapping(vm_snapshot); + auto iso_insns = InstructionsMapping(isolate_snapshot); + + // Diagnostic for investigating in-the-wild patch-apply failures where the + // on-device synthesized "base file" disagrees with what the host's + // `aot_tools dump_blobs` produced. If `Dart_SnapshotDataSize` / + // `Dart_SnapshotInstrSize` reports a shorter span on the device than the + // host extraction did, the bipatch state machine reads past EOF and the + // patch is rejected. Logging each blob's size separately tells the next + // investigator exactly which of the 4 mappings is wrong. + FML_LOG(INFO) << "[shorebird] SnapshotsDataHandle blob sizes: vm_data=" + << vm_data->GetSize() << "b iso_data=" << iso_data->GetSize() + << "b vm_instructions=" << vm_insns->GetSize() + << "b iso_instructions=" << iso_insns->GetSize() << "b total=" + << (vm_data->GetSize() + iso_data->GetSize() + + vm_insns->GetSize() + iso_insns->GetSize()) + << "b"; + std::vector> blobs; - blobs.push_back(DataMapping(vm_snapshot)); - blobs.push_back(DataMapping(isolate_snapshot)); - blobs.push_back(InstructionsMapping(vm_snapshot)); - blobs.push_back(InstructionsMapping(isolate_snapshot)); + blobs.push_back(std::move(vm_data)); + blobs.push_back(std::move(iso_data)); + blobs.push_back(std::move(vm_insns)); + blobs.push_back(std::move(iso_insns)); return std::make_unique(std::move(blobs)); } From ee4044b484628c0d4eba0e5ee22073764b5e37dd Mon Sep 17 00:00:00 2001 From: Brandon DeRosier Date: Thu, 14 May 2026 15:27:17 -0700 Subject: [PATCH 67/95] debug: log per-mapping sizes from SetBaseSnapshot Moves the diagnostic FML_LOG so it fires at every app boot (in SetBaseSnapshot) regardless of whether inflate() is ever reached. This way we capture per-mapping sizes for both the failing iOS standalone variant and the passing iOS add-to-app variant for side-by-side comparison while investigating the on-device base size mismatch. --- .../shell/common/shorebird/shorebird.cc | 27 +++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/engine/src/flutter/shell/common/shorebird/shorebird.cc b/engine/src/flutter/shell/common/shorebird/shorebird.cc index 71d336e90f8b0..2720c5891915e 100644 --- a/engine/src/flutter/shell/common/shorebird/shorebird.cc +++ b/engine/src/flutter/shell/common/shorebird/shorebird.cc @@ -23,6 +23,7 @@ #include "flutter/shell/common/switches.h" #include "fml/logging.h" #include "shell/platform/embedder/embedder.h" +#include "third_party/dart/runtime/include/dart_native_api.h" #include "third_party/dart/runtime/include/dart_tools_api.h" // Namespaced to avoid Google style warnings. @@ -64,6 +65,32 @@ void SetBaseSnapshot(Settings& settings) { // the mappings alive. vm_snapshot = DartSnapshot::VMSnapshotFromSettings(settings); isolate_snapshot = DartSnapshot::IsolateSnapshotFromSettings(settings); + + // Diagnostic for investigating in-the-wild patch-apply failures where the + // on-device synthesized base byte stream is shorter than the host CLI's + // `aot_tools dump_blobs` extraction. Logging here (not in `createForSnapshots`) + // means we get sizes for EVERY app boot โ€” including the passing add-to-app + // variant โ€” for side-by-side comparison. + const uint8_t* vm_data_ptr = vm_snapshot->GetDataMapping(); + const uint8_t* iso_data_ptr = isolate_snapshot->GetDataMapping(); + const uint8_t* vm_insns_ptr = vm_snapshot->GetInstructionsMapping(); + const uint8_t* iso_insns_ptr = isolate_snapshot->GetInstructionsMapping(); + intptr_t vm_data_size = + vm_data_ptr ? Dart_SnapshotDataSize(vm_data_ptr) : -1; + intptr_t iso_data_size = + iso_data_ptr ? Dart_SnapshotDataSize(iso_data_ptr) : -1; + intptr_t vm_insns_size = + vm_insns_ptr ? Dart_SnapshotInstrSize(vm_insns_ptr) : -1; + intptr_t iso_insns_size = + iso_insns_ptr ? Dart_SnapshotInstrSize(iso_insns_ptr) : -1; + FML_LOG(INFO) << "[shorebird] SetBaseSnapshot mappings: " + << "vm_data_size=" << vm_data_size + << " iso_data_size=" << iso_data_size + << " vm_insns_size=" << vm_insns_size + << " iso_insns_size=" << iso_insns_size << " total=" + << (vm_data_size + iso_data_size + vm_insns_size + + iso_insns_size); + Shorebird_SetBaseSnapshots(isolate_snapshot->GetDataMapping(), isolate_snapshot->GetInstructionsMapping(), vm_snapshot->GetDataMapping(), From 417ef3cc5cd375f7ed3d3d1f32a9bbea608f2fdb Mon Sep 17 00:00:00 2001 From: Brandon DeRosier Date: Fri, 15 May 2026 14:08:27 -0700 Subject: [PATCH 68/95] engine: align Seek return value with io::Seek contract + add per-blob observability MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two correctness/observability improvements to the iOS shorebird base reader (SHOREBIRD_USE_INTERPRETER). Neither changes patch-application behavior; both improve debuggability of future on-device issues. == 1. Seek now reports absolute stream position == SnapshotsDataHandle::Seek was returning current_index_.offset (the offset within the current blob) rather than the absolute position from the start of the concatenated stream. POSIX lseek and Rust io::Seek both contract for the new absolute position. Every current consumer happens to ignore the returned u64: - bipatch v1.0.0's Read state machine calls `self.old.seek(SeekFrom::Current(seek))?` at lib.rs:149 and discards the position value โ€” the `?` only ?-propagates I/O errors. - The updater's inflate() captures Seek(End, 0) once as `base_total_size`, but uses it solely for diagnostic display (the `base_size=` field in error/info lines). No control flow, validation, or bounds-check depends on it. So this is a long-standing API-contract bug with no observable patch-correctness impact โ€” verified empirically by reproducing the pre-fix behavior in a wrapper around Cursor and confirming patches still apply correctly. Fixing it brings the implementation in line with consumer expectations and makes the updater's `base_size=` diagnostic field report the true stream length, which would have shortened the recent rc3 investigation considerably. Existing SnapshotsDataHandle unit tests assert on byte content read after seeks, not on Seek's return value, so the contract change is compatible with the test suite. == 2. Per-mapping / per-blob FML_LOG(INFO) lines == - shell/common/shorebird/shorebird.cc::SetBaseSnapshot logs the four mapping sizes (vm_data, isolate_data, vm_instructions, isolate_instructions) once per app boot. - shell/common/shorebird/snapshots_data_handle.cc::createForSnapshots logs the same per-blob sizes once per patch-apply attempt. Both lines are INFO-level, fire at most once per significant event (not in any hot path), and let customer syslogs include the exact sizes the on-device base reader exposes โ€” directly comparable to host-side `aot_tools dump_blobs` output. Cheap, durable observability for a code path that's previously been very hard to diagnose remotely. --- .../flutter/shell/common/shorebird/shorebird.cc | 9 ++++----- .../common/shorebird/snapshots_data_handle.cc | 15 +++++++-------- 2 files changed, 11 insertions(+), 13 deletions(-) diff --git a/engine/src/flutter/shell/common/shorebird/shorebird.cc b/engine/src/flutter/shell/common/shorebird/shorebird.cc index 2720c5891915e..a539528d9da56 100644 --- a/engine/src/flutter/shell/common/shorebird/shorebird.cc +++ b/engine/src/flutter/shell/common/shorebird/shorebird.cc @@ -66,11 +66,10 @@ void SetBaseSnapshot(Settings& settings) { vm_snapshot = DartSnapshot::VMSnapshotFromSettings(settings); isolate_snapshot = DartSnapshot::IsolateSnapshotFromSettings(settings); - // Diagnostic for investigating in-the-wild patch-apply failures where the - // on-device synthesized base byte stream is shorter than the host CLI's - // `aot_tools dump_blobs` extraction. Logging here (not in `createForSnapshots`) - // means we get sizes for EVERY app boot โ€” including the passing add-to-app - // variant โ€” for side-by-side comparison. + // Per-mapping observability for the base snapshot. Logged at every app boot + // so customer syslogs include the exact sizes the on-device base reader + // exposes โ€” directly comparable to the host's `aot_tools dump_blobs` + // extraction the patch was generated against. const uint8_t* vm_data_ptr = vm_snapshot->GetDataMapping(); const uint8_t* iso_data_ptr = isolate_snapshot->GetDataMapping(); const uint8_t* vm_insns_ptr = vm_snapshot->GetInstructionsMapping(); diff --git a/engine/src/flutter/shell/common/shorebird/snapshots_data_handle.cc b/engine/src/flutter/shell/common/shorebird/snapshots_data_handle.cc index 1fd6688299a11..3f8c7f6763794 100644 --- a/engine/src/flutter/shell/common/shorebird/snapshots_data_handle.cc +++ b/engine/src/flutter/shell/common/shorebird/snapshots_data_handle.cc @@ -91,13 +91,10 @@ std::unique_ptr SnapshotsDataHandle::createForSnapshots( auto vm_insns = InstructionsMapping(vm_snapshot); auto iso_insns = InstructionsMapping(isolate_snapshot); - // Diagnostic for investigating in-the-wild patch-apply failures where the - // on-device synthesized "base file" disagrees with what the host's - // `aot_tools dump_blobs` produced. If `Dart_SnapshotDataSize` / - // `Dart_SnapshotInstrSize` reports a shorter span on the device than the - // host extraction did, the bipatch state machine reads past EOF and the - // patch is rejected. Logging each blob's size separately tells the next - // investigator exactly which of the 4 mappings is wrong. + // Per-blob observability for the base byte stream the updater is about to + // patch against. Logged at every patch-apply attempt so customer syslogs + // include the exact sizes the bipatch state machine sees โ€” directly + // comparable to the host's `aot_tools dump_blobs` extraction. FML_LOG(INFO) << "[shorebird] SnapshotsDataHandle blob sizes: vm_data=" << vm_data->GetSize() << "b iso_data=" << iso_data->GetSize() << "b vm_instructions=" << vm_insns->GetSize() @@ -158,7 +155,9 @@ int64_t SnapshotsDataHandle::Seek(int64_t offset, int32_t whence) { FML_CHECK(false) << "Unrecognized whence value in Seek: " << whence; } current_index_ = IndexForAbsoluteOffset(offset, start_index); - return current_index_.offset; + // Per POSIX/Rust io::Seek contract, return the new absolute position from + // the start of the stream, not the offset within the current blob. + return static_cast(AbsoluteOffsetForIndex(current_index_)); } } // namespace flutter \ No newline at end of file From 6c58a65f56867063acc1e4f0df20db2c4e3a6a0f Mon Sep 17 00:00:00 2001 From: Eric Seidel Date: Fri, 15 May 2026 16:23:57 -0700 Subject: [PATCH 69/95] ci(shards): split windows/host into host-release and host-debug (#153) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The windows/host shard runs three serial steps inside one matrix job: rust โ†’ gn_ninja host_release โ†’ gn_ninja host_debug. On the most recent 12x28 build run, those ninja phases took ~22m and ~18m respectively โ€” the shard's total wall clock was ~31m and it was the long pole on the Windows critical path. Split into two parallel sub-shards, each carrying rust + its own ninja step + the artifacts that ninja produces: host-release โ†’ rust + gn_ninja host_release artifacts: dart-sdk-windows-x64.zip, windows-x64-flutter.zip host-debug โ†’ rust + gn_ninja host_debug artifact: artifacts.zip Rust gets duplicated across both shards (it builds the updater rlib which both ninja steps link). The duplication costs ~1m per shard but runs in parallel; net wall clock for the longer sub-shard should be roughly half the original. Setup cost (gclient sync, depot_tools, gcloud auth) is also duplicated but small thanks to the Windows source cache landing in _build_engine#231. Windows has slot headroom (4 shards on a pool with more), so adding one shard doesn't queue. Expected windows/host critical path drops from ~31m to ~20m. No _build_engine code change needed โ€” shard_runner reads shard configs JSON-key-agnostically and the matrix is dynamic via discover-shards. --- shorebird/ci/shards/windows.json | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/shorebird/ci/shards/windows.json b/shorebird/ci/shards/windows.json index 11183cb68611e..4e5c47c71f0b5 100644 --- a/shorebird/ci/shards/windows.json +++ b/shorebird/ci/shards/windows.json @@ -38,7 +38,7 @@ {"src": "android_release_x64/zip_archives/android-x64-release/windows-x64.zip", "dst": "flutter_infra_release/flutter/$engine/android-x64-release/windows-x64.zip"} ] }, - "host": { + "host-release": { "steps": [ { "type": "rust", @@ -49,6 +49,18 @@ "gn_args": ["--runtime-mode=release", "--no-prebuilt-dart-sdk"], "ninja_targets": ["dart_sdk", "flutter/build/archives:windows_flutter", "gen_snapshot", "windows", "flutter/build/archives:artifacts"], "out_dir": "host_release" + } + ], + "artifacts": [ + {"src": "host_release/dart-sdk", "dst": "flutter_infra_release/flutter/$engine/dart-sdk-windows-x64.zip", "zip": true, "content_hash": true}, + {"src": "host_release/zip_archives/windows-x64-release/windows-x64-flutter.zip", "dst": "flutter_infra_release/flutter/$engine/windows-x64-release/windows-x64-flutter.zip"} + ] + }, + "host-debug": { + "steps": [ + { + "type": "rust", + "targets": ["x86_64-pc-windows-msvc"] }, { "type": "gn_ninja", @@ -58,8 +70,6 @@ } ], "artifacts": [ - {"src": "host_release/dart-sdk", "dst": "flutter_infra_release/flutter/$engine/dart-sdk-windows-x64.zip", "zip": true, "content_hash": true}, - {"src": "host_release/zip_archives/windows-x64-release/windows-x64-flutter.zip", "dst": "flutter_infra_release/flutter/$engine/windows-x64-release/windows-x64-flutter.zip"}, {"src": "host_debug/zip_archives/windows-x64/artifacts.zip", "dst": "flutter_infra_release/flutter/$engine/windows-x64/artifacts.zip"} ] } From ce5c299c8059d63ae3e2dcb545ee27b88969b274 Mon Sep 17 00:00:00 2001 From: Brandon DeRosier Date: Mon, 11 May 2026 21:31:24 -0700 Subject: [PATCH 70/95] fix(android): demote libapp.so.sym/dbg check to warning (Part 1 of 3.44 fix) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Per shorebirdtech/_shorebird#2150 item 1 Part 1: Upstream Flutter PR #181275 (merged 2026-01-26, in 3.44) inverted libapp.so strip responsibility โ€” Flutter stopped stripping it itself, AGP is expected to, and a new post-build check in packages/flutter_tools/lib/src/android/gradle.dart fatal-errors if libapp.so.sym/dbg is absent from the AAB. This breaks two Shorebird flows: 1. Existing apps with the legacy keepDebugSymbols.add("**/libapp.so") line in their build.gradle.kts (AGP skips stripping โ†’ no .sym) 2. Obfuscated builds where Shorebird CLI passes --extra-gen-snapshot-options=--strip (gen_snapshot pre-strips โ†’ AGP has nothing to strip โ†’ no .sym) Both produce valid AABs that run correctly, just without Dart-code crash symbols in Play Console. Demoting the check to a warning lets existing users upgrade to 3.44 without their first build failing, and unblocks the obfuscated CI/release flow. Users get a clear message explaining the trade-off and how to opt into AGP-side stripping. --- .../flutter_tools/lib/src/android/gradle.dart | 22 ++++++++++++++++--- 1 file changed, 19 insertions(+), 3 deletions(-) diff --git a/packages/flutter_tools/lib/src/android/gradle.dart b/packages/flutter_tools/lib/src/android/gradle.dart index b5da2688b6237..ed2d17a0b0aa2 100644 --- a/packages/flutter_tools/lib/src/android/gradle.dart +++ b/packages/flutter_tools/lib/src/android/gradle.dart @@ -798,10 +798,26 @@ To fix this, you can either: } if (!(result.stdout.contains('libapp.so.sym') || result.stdout.contains('libapp.so.dbg'))) { - _logger.printTrace( - 'libapp.so.sym or libapp.so.dbg not present when checking final appbundle for debug symbols.', + // Shorebird-specific: demote upstream's fatal check to a warning. + // + // Upstream Flutter PR #181275 (3.44) inverted libapp.so strip + // responsibility โ€” Flutter stopped stripping it, AGP is expected to, + // and this check fails the build if `.sym/.dbg` is absent. Shorebird + // flows that either keep libapp.so unstripped (legacy fixtures with + // `keepDebugSymbols.add("**/libapp.so")`) or pre-strip via + // `--extra-gen-snapshot-options=--strip` (obfuscated builds) both + // produce AABs without `libapp.so.sym`. Returning `true` here lets the + // build succeed; the trade-off is that Play Console won't get Dart + // crash symbols for these AABs. New apps should remove the legacy + // `keepDebugSymbols` line to re-enable AGP-side stripping. + _logger.printWarning( + 'libapp.so.sym or libapp.so.dbg not present when checking final ' + 'appbundle for debug symbols. Play Console will not receive Dart ' + 'crash symbols for this build. To enable, remove ' + '`packaging.jniLibs.keepDebugSymbols.add("**/libapp.so")` from ' + 'android/app/build.gradle.kts.', ); - return false; + return true; } return true; From 305c060f7b001466d7493d49e5ec34367b8f7196 Mon Sep 17 00:00:00 2001 From: Brandon DeRosier Date: Tue, 19 May 2026 20:33:31 -0700 Subject: [PATCH 71/95] fix(macos): skip DD 2-pass on macOS until per-arch support lands The DD 2-pass code in AOTSnapshotter assumed iOS-style single-arch gen_snapshot invocations. macOS breaks two of its preconditions: 1. The shipped `analyze_snapshot` for macOS is a single host-arch binary (no `_arm64`/`_x64` suffix). When targeting x64 from an arm64 host, gen_snapshot_x64 reports `macos_simx64` while analyze_snapshot reports `macos_arm64`, and the strict --sdk_version equality check in getAnalyzeSnapshotPath rejects it -> null -> "could not find an analyze_snapshot binary whose --sdk_version matches gen_snapshot" -> abort. 2. Flutter's compile_macos_framework target spawns gen_snapshot_arm64 and gen_snapshot_x64 in parallel via Future.wait. Both DD pass 1 invocations write to the same `App_dd_analysis.so` path under `.dart_tool/flutter_build//`, so they race; even if (1) were fixed, the analyze_snapshot --compute_dd_table call would see a half-written or already-deleted ELF. Properly fixing both means per-arch DD output paths (so the two gen_snapshot invocations don't collide), a relaxed version check (so a single analyze_snapshot can serve both archs when the Dart SDK matches), or shipping arch-specific analyze_snapshot binaries from the engine build. That's tracked separately. For now, gate `usesLinker && ddMaxBytes > 0` on `platform != TargetPlatform.darwin`. macOS releases will ship without DD activation; patches against them will compute DD on the fly when applied -- the pre-1.6.99 behavior, slightly worse link percentage but functional. iOS / Android / Linux / Windows are unaffected. Reproduced on a Shorebird macOS standalone release after this landed, where the error chain was: DD pass: could not find an analyze_snapshot binary whose --sdk_version matches gen_snapshot. ... Aborting the build instead. Snapshot file does not exist DD pass: analyze_snapshot --compute_dd_table failed with exit code 255. Target compile_macos_framework failed: PathNotFoundException With this fix: - release: succeeds (37s) - patch: succeeds (32s) - baseline XCUITest counter assertion fails -- separate accessibility/click-routing issue, unrelated to DD. --- .../flutter_tools/lib/src/base/build.dart | 19 ++++++++++++++++++- 1 file changed, 18 insertions(+), 1 deletion(-) diff --git a/packages/flutter_tools/lib/src/base/build.dart b/packages/flutter_tools/lib/src/base/build.dart index 8dfb32e070d10..d9b55161df229 100644 --- a/packages/flutter_tools/lib/src/base/build.dart +++ b/packages/flutter_tools/lib/src/base/build.dart @@ -307,7 +307,24 @@ class AOTSnapshotter { final snapshotType = SnapshotType(platform, buildMode); final int ddMaxBytes = _readDdMaxBytes(); - if (ddMaxBytes > 0 && usesLinker) { + // DD pass requires: + // 1. an analyze_snapshot binary whose --sdk_version matches gen_snapshot's + // --version, and + // 2. a single non-racing ELF output path per build invocation. + // + // macOS builds fail both. (1) The shipped `analyze_snapshot` is a single + // host-arch binary; when targeting x64 from an arm64 host, gen_snapshot_x64 + // reports `macos_simx64` while analyze_snapshot reports `macos_arm64` and + // the version-equality check rejects it. (2) Flutter spawns + // gen_snapshot_arm64 and gen_snapshot_x64 in parallel and both DD passes + // race on the same `App_dd_analysis.so` path under + // `.dart_tool/flutter_build//`. + // + // Skip DD on macOS until that work is done. Patches built without DD + // activation will compute DD on the fly when applied, which is the + // pre-1.6.99 behavior โ€” slightly worse link percentage but functional. + final bool ddPassSupported = platform != TargetPlatform.darwin; + if (ddMaxBytes > 0 && usesLinker && ddPassSupported) { final int pass1Exit = await _runDdAnalysisPass( snapshotType: snapshotType, darwinArch: darwinArch, From 2921463fca4f38c5af9fdbc951b7ef7be515072f Mon Sep 17 00:00:00 2001 From: Eric Seidel Date: Thu, 4 Jun 2026 19:20:53 -0700 Subject: [PATCH 72/95] shorebird: consume our published Dart SDK on mac-arm64 (skip rebuilding it) (#155) * shorebird: consume our published Dart SDK on mac-arm64, skip rebuilding it Point the macos-arm64 dart-sdk prebuilt at Shorebird's own published SDK (gs://shorebird-dart-sdk-prebuilt, dep_type: gcs) instead of Google's CIPD, and drop --no-prebuilt-dart-sdk from the mac-arm64 engine shard. Effect: the mac-arm64 shard stops building the Dart SDK from source (the dart_sdk gn target was a full from-source compile). With the prebuilt it becomes copy_dart_sdk -- a ~1.5s copy of our SDK -- so the engine builds against our Dart fork end to end and the mac-arm64 build is significantly shorter. The published dart-sdk-darwin-arm64.zip artifact is now our SDK. Verified locally: gn --mac-cpu=arm64 --prebuilt-dart-sdk + ninja dart_sdk copies our SDK (revision 3bc26d9, executable) in ~1.5s. Bot auth to the private bucket is keyless WIF via shorebird-dart-sdk-reader (shorebirdtech/_build_engine sync.yaml). mac-x64/linux/windows are unchanged (still build Dart from source / use Google CIPD) pending their own published prebuilts. * shorebird: key the dart-sdk prebuilt object on Var('dart_sdk_revision') Use the existing dart_sdk_revision var for the macos-arm64 gcs prebuilt object name instead of hardcoding the sha, so the source clone and the prebuilt always reference the same fork revision. Bump dart_sdk_revision to the published sha (3bc26d9); a published prebuilt must exist for whatever this var points at. Not dart_revision: that's Google's upstream revision and is load-bearing for the linux/windows/mac-x64 CIPD entries; our GCS objects are keyed by our fork's sha. gclient validate passes; object resolves to gs://shorebird-dart-sdk-prebuilt//dart-sdk-darwin-arm64.tar.gz. --- DEPS | 21 +++++++++++++++++---- shorebird/ci/shards/macos.json | 2 +- 2 files changed, 18 insertions(+), 5 deletions(-) diff --git a/DEPS b/DEPS index d66be3e29bc1e..6a7cd56558e84 100644 --- a/DEPS +++ b/DEPS @@ -400,14 +400,27 @@ deps = { 'dep_type': 'cipd', 'condition': 'host_os == "mac" and download_dart_sdk' }, + # Consume Shorebird's own published Dart SDK (built from + # shorebirdtech/dart-sdk) from GCS instead of Google's CIPD prebuilt, so + # the engine builds against our Dart fork and skips rebuilding it from + # source (see shards/macos.json: mac-arm64 drops --no-prebuilt-dart-sdk). + # The object is keyed by Var('dart_sdk_revision') so it tracks the same + # fork revision as the source clone above; sha256sum/size_bytes pin the + # specific artifact's content and must be updated whenever that revision + # bumps. tar.gz (not zip) so gclient's extraction preserves the executable + # bit on bin/dart etc. The bot reads this private bucket via a keyless-WIF + # reader SA (shorebirdtech/_build_engine sync.yaml). 'engine/src/flutter/prebuilts/macos-arm64/dart-sdk': { - 'packages': [ + 'bucket': 'shorebird-dart-sdk-prebuilt', + 'objects': [ { - 'package': 'flutter/dart-sdk/mac-arm64', - 'version': 'git_revision:'+Var('dart_revision') + 'object_name': Var('dart_sdk_revision') + '/dart-sdk-darwin-arm64.tar.gz', + 'sha256sum': '5968b313316b9edc8d6b003e9fb663e024568d00cea5efe0214191020412ffbf', + 'size_bytes': 205230386, + 'generation': 1780425558083300, } ], - 'dep_type': 'cipd', + 'dep_type': 'gcs', 'condition': 'host_os == "mac" and download_dart_sdk' }, 'engine/src/flutter/prebuilts/windows-x64/dart-sdk': { diff --git a/shorebird/ci/shards/macos.json b/shorebird/ci/shards/macos.json index 0d1336c2be69f..01ef525a35b2d 100644 --- a/shorebird/ci/shards/macos.json +++ b/shorebird/ci/shards/macos.json @@ -82,7 +82,7 @@ }, { "type": "gn_ninja", - "gn_args": ["--runtime-mode=release", "--mac-cpu=arm64", "--no-prebuilt-dart-sdk"], + "gn_args": ["--runtime-mode=release", "--mac-cpu=arm64"], "ninja_targets": ["dart_sdk"], "out_dir": "host_release_arm64" }, From 72f06d5f975c857fd17ab61d2b34aa736e6d2617 Mon Sep 17 00:00:00 2001 From: Eric Seidel Date: Thu, 4 Jun 2026 20:23:34 -0700 Subject: [PATCH 73/95] Reconcile flutter_tools tests with upstream 3.44.1 (#157) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The 3.44.1 rebase bumped source to upstream but left several tool test files on the old (~3.41) base and dropped the test-side of shorebird's own source divergences, so `tool_tests/general` was red on shorebird/dev (and on every PR into it). Reconcile the affected test files with the current source in this checkout (3-way merge of shorebird's changes onto 3.44.1 where applicable, plus expectation updates to match what the source actually emits): - macos_test.dart: pick up upstream's getInfo `buildDirectory` param + new imports; align iOS gen_snapshot to the assemblyโ†’cc/clangโ†’strip flow this fork uses (not upstream's macho-dylib path). - common_test.dart / android_test.dart: expect `--strip` (shorebird strip-by-default, build.dart) on AOT ELF gen_snapshot invocations. - cache_test.dart: match the displayName/downloadCount artifact API (FakeSecondaryCachedArtifact gains the missing overrides) and the "Web SDK"/"Engine Information" download-status messages. Tests only; no lib/ changes, nothing skipped. All four files pass locally (macos 21, common 22, android 13, cache 58 + 2 pre-existing skips). --- .../build_system/targets/android_test.dart | 4 ++ .../build_system/targets/common_test.dart | 53 ++++++++++++++++--- .../build_system/targets/macos_test.dart | 2 +- .../test/general.shard/cache_test.dart | 10 +++- 4 files changed, 59 insertions(+), 10 deletions(-) diff --git a/packages/flutter_tools/test/general.shard/build_system/targets/android_test.dart b/packages/flutter_tools/test/general.shard/build_system/targets/android_test.dart index 9ccb733787282..691e5c5857c8b 100644 --- a/packages/flutter_tools/test/general.shard/build_system/targets/android_test.dart +++ b/packages/flutter_tools/test/general.shard/build_system/targets/android_test.dart @@ -180,6 +180,7 @@ void main() { '--deterministic', '--snapshot_kind=app-aot-elf', '--elf=${environment.buildDir.childDirectory('arm64-v8a').childFile('app.so').path}', + '--strip', environment.buildDir.childFile('app.dill').path, ], ), @@ -218,6 +219,7 @@ void main() { '--trace-precompiler-to=code_size_1/trace.arm64-v8a.json', '--snapshot_kind=app-aot-elf', '--elf=${environment.buildDir.childDirectory('arm64-v8a').childFile('app.so').path}', + '--strip', environment.buildDir.childFile('app.dill').path, ], ), @@ -261,6 +263,7 @@ void main() { 'baz=2', '--snapshot_kind=app-aot-elf', '--elf=${environment.buildDir.childDirectory('arm64-v8a').childFile('app.so').path}', + '--strip', environment.buildDir.childFile('app.dill').path, ], ), @@ -303,6 +306,7 @@ void main() { 'bar', '--snapshot_kind=app-aot-elf', '--elf=${environment.buildDir.childDirectory('arm64-v8a').childFile('app.so').path}', + '--strip', environment.buildDir.childFile('app.dill').path, ], ), diff --git a/packages/flutter_tools/test/general.shard/build_system/targets/common_test.dart b/packages/flutter_tools/test/general.shard/build_system/targets/common_test.dart index a5e20b2386361..5f4f49a315a16 100644 --- a/packages/flutter_tools/test/general.shard/build_system/targets/common_test.dart +++ b/packages/flutter_tools/test/general.shard/build_system/targets/common_test.dart @@ -26,7 +26,6 @@ import '../../../src/fakes.dart'; const kBoundaryKey = '4d2d9609-c662-4571-afde-31410f96caa6'; const kElfAot = '--snapshot_kind=app-aot-elf'; -const kMachoDylibAot = '--snapshot_kind=app-aot-macho-dylib'; /// Generate Shorebird link info arguments for iOS/macOS AOT builds. /// The [buildPath] should be the build directory path (outputDir.parent.path). @@ -695,6 +694,7 @@ void main() { '--deterministic', kElfAot, '--elf=$build/app.so', + '--strip', '--no-sim-use-hardfp', '--no-use-integer-division', '$build/app.dill', @@ -725,6 +725,7 @@ void main() { '--trace-precompiler-to=code_size_1/trace.android-arm.json', kElfAot, '--elf=$build/app.so', + '--strip', '--no-sim-use-hardfp', '--no-use-integer-division', '$build/app.dill', @@ -819,15 +820,52 @@ void main() { ...linkInfoArgsFor(build), '--write-v8-snapshot-profile-to=code_size_1/snapshot.arm64.json', '--trace-precompiler-to=code_size_1/trace.arm64.json', - kMachoDylibAot, - '--macho=$build/arm64/App.framework/App', - '--macho-object=$build/arm64/app.o', - '--macho-min-os-version=15.0', - '--macho-rpath=@executable_path/Frameworks,@loader_path/Frameworks', - '--macho-install-name=@rpath/App.framework/App', + '--snapshot_kind=app-aot-assembly', + '--assembly=$build/arm64/snapshot_assembly.S', '$build/app.dill', ], ), + FakeCommand( + command: [ + 'xcrun', + 'cc', + '-arch', + 'arm64', + '-miphoneos-version-min=13.0', + '-isysroot', + 'path/to/iPhoneOS.sdk', + '-c', + '$build/arm64/snapshot_assembly.S', + '-o', + '$build/arm64/snapshot_assembly.o', + ], + ), + FakeCommand( + command: [ + 'xcrun', + 'clang', + '-arch', + 'arm64', + '-miphoneos-version-min=13.0', + '-isysroot', + 'path/to/iPhoneOS.sdk', + '-dynamiclib', + '-Xlinker', + '-rpath', + '-Xlinker', + '@executable_path/Frameworks', + '-Xlinker', + '-rpath', + '-Xlinker', + '@loader_path/Frameworks', + '-fapplication-extension', + '-install_name', + '@rpath/App.framework/App', + '-o', + '$build/arm64/App.framework/App', + '$build/arm64/snapshot_assembly.o', + ], + ), FakeCommand( command: [ 'xcrun', @@ -888,6 +926,7 @@ void main() { 'baz=2', kElfAot, '--elf=$build/app.so', + '--strip', '--no-sim-use-hardfp', '--no-use-integer-division', '$build/app.dill', diff --git a/packages/flutter_tools/test/general.shard/build_system/targets/macos_test.dart b/packages/flutter_tools/test/general.shard/build_system/targets/macos_test.dart index 21d68f11cbb6d..f1c0f34d8a96a 100644 --- a/packages/flutter_tools/test/general.shard/build_system/targets/macos_test.dart +++ b/packages/flutter_tools/test/general.shard/build_system/targets/macos_test.dart @@ -889,7 +889,7 @@ flavors: .childFile('x86_64/App.framework.dSYM/Contents/Resources/DWARF/App') .createSync(recursive: true); - final build = environment.buildDir.path; + final String build = environment.buildDir.path; // Both archs go through the same code path now (the DD-pass is gated on // SHOREBIRD_DD_MAX_BYTES which isn't set in this test), so neither // architecture has an extra async hop. With concurrent Future.wait, the diff --git a/packages/flutter_tools/test/general.shard/cache_test.dart b/packages/flutter_tools/test/general.shard/cache_test.dart index 231947714fb44..21f7647472a99 100644 --- a/packages/flutter_tools/test/general.shard/cache_test.dart +++ b/packages/flutter_tools/test/general.shard/cache_test.dart @@ -933,7 +933,7 @@ void main() { await webSdk.updateInner(artifactUpdater, fileSystem, FakeOperatingSystemUtils()); - expect(messages, ['Downloading Web SDK...']); + expect(messages, ['Web SDK']); expect(downloads, [ 'https://download.shorebird.dev/flutter_infra_release/flutter/hijklmnop/flutter-web-sdk.zip', @@ -1054,7 +1054,7 @@ void main() { await engineStamp.updateInner(artifactUpdater, fileSystem, FakeOperatingSystemUtils()); - expect(messages, ['Downloading engine information...']); + expect(messages, ['Engine Information']); expect(downloads, [ 'https://download.shorebird.dev/flutter_infra_release/flutter/hijklmnop/engine_stamp.json', @@ -1372,6 +1372,12 @@ class FakeSecondaryCachedArtifact extends Fake implements CachedArtifact { @override DevelopmentArtifact get developmentArtifact => DevelopmentArtifact.universal; + + @override + String get displayName => 'fake'; + + @override + int get downloadCount => 1; } class FakeIosUsbArtifacts extends Fake implements IosUsbArtifacts { From 2250e158ae28e724a4cbd0a048b3f7628079ca1a Mon Sep 17 00:00:00 2001 From: Brandon DeRosier Date: Thu, 4 Jun 2026 21:29:11 -0700 Subject: [PATCH 74/95] fix: restore --no-prebuilt-dart-sdk on mac-arm64 dart_sdk step Partial revert of #155 for the 3.44.1-rc1 release. With the flag dropped, the kernel-compiling host steps (const_finder.dart.snapshot, flutter_patched_sdk dills) resolved the upstream CIPD Dart prebuilt (dart_revision fc3da898ea1) instead of our fork, stamping kernels with an SDK hash our VM (0e6acdc83e) refuses to load: 'ConstFinder failure: Can't load Kernel binary: Invalid SDK hash' on every release-mode build. The DEPS macos-arm64 GCS prebuilt dep stays (inert with the flag restored). #155 can return once every kernel-producing step is covered by a fork prebuilt and the engine build verifies artifact kernel hashes against the shipped VM. --- shorebird/ci/shards/macos.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/shorebird/ci/shards/macos.json b/shorebird/ci/shards/macos.json index 01ef525a35b2d..0d1336c2be69f 100644 --- a/shorebird/ci/shards/macos.json +++ b/shorebird/ci/shards/macos.json @@ -82,7 +82,7 @@ }, { "type": "gn_ninja", - "gn_args": ["--runtime-mode=release", "--mac-cpu=arm64"], + "gn_args": ["--runtime-mode=release", "--mac-cpu=arm64", "--no-prebuilt-dart-sdk"], "ninja_targets": ["dart_sdk"], "out_dir": "host_release_arm64" }, From 697aca8f5dadcd8d9ffefac610b812aae7bee3b9 Mon Sep 17 00:00:00 2001 From: Eric Seidel Date: Fri, 5 Jun 2026 09:06:48 -0700 Subject: [PATCH 75/95] Bump flutter_flavorizr to published 2.5.0 (fix flavor APK build) (#158) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Bump flutter_flavorizr to published 2.5.0 (fix flavor APK build) The Shorebird Android Tests' flavor builds fail on 3.44.1: Product Flavor playStore contains custom resource values, but the feature is disabled. The test scaffolds flavors with flutter_flavorizr pinned to a fork at a Flutter-3.29-era ref (AngeloAvv/flutter_flavorizr#291, which was never merged). The AGP that ships with 3.44.1 disables resValues by default, and the 3.29-era generator doesn't enable the build feature, so the generated project won't configure. Move to the published flutter_flavorizr ^2.5.0, which has moved well past 3.29 and generates AGP-8-compatible flavor config. Removes the stale fork pin + felangel TODO. * Pass -f to flutter_flavorizr (skip interactive prompt under CI) flutter_flavorizr 2.5.0 added an interactive 'Do you want to proceed? (Y/n)' confirmation, which throws 'No terminal attached to stdout' in CI (both Android Tests and the Smoke build). The -f/--force flag runs it non-interactively. * Pin flutter_flavorizr to 2.4.2 (Ruby xcodeproj, fixes iOS flavor archive) 2.5.0 replaced Ruby xcodeproj with dart_xcodeproj; its generated .pbxproj breaks 'flutter build ipa --no-codesign --flavor' (unsigned flavor archive demands a Development Team). All versions emit identical visible signing settings, so it's the dart_xcodeproj generation, not config โ€” and the old fork worked precisely because it used Ruby xcodeproj. 2.4.2 is the last Ruby-xcodeproj release and still carries the 3.29/AGP-8 resValues fix that the 3.29-era fork lacked, so it should green both the Android (resValues) and Smoke (iOS flavor) builds. * Enable resValues build feature after flavorizr (fix flavored APK) There is no single flutter_flavorizr version that greens both checks: 2.5.0 fixes the AGP-8 resValues issue but its dart_xcodeproj rewrite breaks the unsigned iOS flavor archive; 2.4.2 (last Ruby-xcodeproj release) fixes iOS but predates the resValues fix. So pin 2.4.2 (iOS works) and post-fix the one thing it misses: append android.defaults.buildfeatures.resvalues=true to the generated gradle.properties so the flavored 'flutter build apk' configures under AGP 8. Wraps flavorizr rather than forking it. --- .../shorebird_tests/test/shorebird_tests.dart | 34 ++++++++++++++++--- 1 file changed, 30 insertions(+), 4 deletions(-) diff --git a/packages/shorebird_tests/test/shorebird_tests.dart b/packages/shorebird_tests/test/shorebird_tests.dart index e63a8d99d89b6..cfcda4c5b4603 100644 --- a/packages/shorebird_tests/test/shorebird_tests.dart +++ b/packages/shorebird_tests/test/shorebird_tests.dart @@ -240,9 +240,18 @@ extension ShorebirdProjectDirectoryOnDirectory on Directory { Future addProjectFlavors() async { await addPubDependency( - // TODO(felangel): revert to using published version once 3.29.0 support is released. - // https://github.com/AngeloAvv/flutter_flavorizr/pull/291 - 'dev:flutter_flavorizr:{"git":{"url":"https://github.com/wjlee611/flutter_flavorizr.git","ref":"chore/temp-migrate-3-29","path":"."}}', + // Published flutter_flavorizr. We previously pinned a fork for Flutter + // 3.29 support (AngeloAvv/flutter_flavorizr#291, never merged); the + // published package has since adopted 3.29/AGP-8 support (>=2.3.0), so + // the fork is obsolete. + // + // Pinned to 2.4.2 (not the latest 2.5.x): 2.5.0 replaced the Ruby + // xcodeproj gem with dart_xcodeproj, and its generated .pbxproj breaks + // `flutter build ipa --no-codesign --flavor` (the unsigned flavor + // archive demands a Development Team). 2.4.2 is the last Ruby-xcodeproj + // release and matches the generation path the old fork used, while + // still carrying the AGP-8 resValues fix the fork lacked. + 'dev:flutter_flavorizr:2.4.2', ); await File( @@ -279,13 +288,30 @@ flavors: '''); final result = await _runFlutterCommand( - ['pub', 'run', 'flutter_flavorizr'], + // -f/--force skips flutter_flavorizr's interactive "proceed? (Y/n)" + // prompt, which throws "No terminal attached to stdout" under CI. + ['pub', 'run', 'flutter_flavorizr', '-f'], workingDirectory: this, ); if (result.exitCode != 0) { throw Exception( 'Failed to run `flutter pub run flutter_flavorizr`: ${result.stderr}'); } + + // flutter_flavorizr 2.4.2 emits per-flavor `resValue` entries in + // build.gradle. AGP 8 (Flutter 3.44+) gates resValues behind an opt-in + // build feature, so the flavored `flutter build apk` fails with "contains + // custom resource values, but the feature is disabled" unless we enable + // it. (Published flavorizr only enables this on 2.5.x, which we can't use: + // 2.5.0's dart_xcodeproj rewrite breaks `flutter build ipa --no-codesign + // --flavor`. So we stay on the last Ruby-xcodeproj release and toggle the + // build feature ourselves.) + final gradleProperties = + File(path.join(this.path, 'android', 'gradle.properties')); + await gradleProperties.writeAsString( + '\nandroid.defaults.buildfeatures.resvalues=true\n', + mode: FileMode.append, + ); } void addShorebirdFlavors() { From d27c9697eb3e65a833567e52e6d002a88a510651 Mon Sep 17 00:00:00 2001 From: Nick Weatherley Date: Thu, 23 Jul 2026 15:36:03 -0400 Subject: [PATCH 76/95] chore: reset DEPS for 3.47.0-rc0 (dart-sdk 046fbc2 offsets-regen, updater 1f85c4a) --- DEPS | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/DEPS b/DEPS index 6a7cd56558e84..680b9f9b279f9 100644 --- a/DEPS +++ b/DEPS @@ -17,10 +17,10 @@ vars = { 'llvm_git': 'https://llvm.googlesource.com', 'dart_ai_rev': '9c96bfe5f091c9451eff5b59c9bffeb2e806b875', 'skia_revision': '8df24be66531469e576a806749a0202ae26b8d08', - "dart_sdk_revision": "b65ce89c8057d6880e00693a7b0ecd7b9e5f61ca", + "dart_sdk_revision": "046fbc29efd0bd56a7749e1e48a0793fe41780d8", "dart_sdk_git": "git@github.com:shorebirdtech/dart-sdk.git", "updater_git": "https://github.com/shorebirdtech/updater.git", - "updater_rev": "fe3733374c222bf7d8ff1913f5ab213421a7d94f", + "updater_rev": "1f85c4ab1ee5b540269b9859c75e1bffbb9050c7", # WARNING: DO NOT EDIT canvaskit_cipd_instance MANUALLY # See `lib/web_ui/README.md` for how to roll CanvasKit to a new version. From 1fdc321374feb7294649b1820f4fa6084500fbcb Mon Sep 17 00:00:00 2001 From: Nick Weatherley Date: Thu, 23 Jul 2026 16:02:49 -0400 Subject: [PATCH 77/95] chore(dry-run): disable unused mac-arm64 dart-sdk prebuilt hook (404 on regen tip) --- DEPS | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/DEPS b/DEPS index 680b9f9b279f9..469cc76af2c60 100644 --- a/DEPS +++ b/DEPS @@ -421,7 +421,14 @@ deps = { } ], 'dep_type': 'gcs', - 'condition': 'host_os == "mac" and download_dart_sdk' + # Dry-run 3.47.0-rc0: mac-arm64 builds Dart from source + # (--no-prebuilt-dart-sdk in shards/macos.json), so this prebuilt is never + # consumed. Disabled because it was never published for the offsets-regen + # tip (046fbc2), which 404s gclient sync. Real release: either publish it + # via publish-darwin-arm64 + update sha256sum/size_bytes/generation, or drop + # this hook (it is vestigial post the #155 partial revert). See + # notes/3.47.0-rc0/deferred-followups.md. + 'condition': 'False' }, 'engine/src/flutter/prebuilts/windows-x64/dart-sdk': { 'packages': [ From ae9bc6e33f92339fa532292a88a7236839eabf5f Mon Sep 17 00:00:00 2001 From: Nick Weatherley Date: Mon, 10 Aug 2026 10:49:27 -0400 Subject: [PATCH 78/95] fix: feed the updater one snapshot instead of two createForSnapshots took a VM snapshot and an isolate snapshot and concatenated data, data, text, text. Dart 3.13 folded the VM isolate into the isolate group, so kVMDataSymbol and kIsolateDataSymbol are the same symbol and resolve to the same buffer. The base stream the updater diffs against was therefore twice the bytes analyze_snapshot --dump_blobs writes, misaligned against the host extraction, with nothing on either end validating the length. Now takes one snapshot and pushes two mappings, matching HandleDumpBlobs. The surviving source is the VM resolve path deliberately. ResolveIsolateData fires ReportLaunchStart and returns the patch when TryLoadFromPatch finds one. ResolveVMData is patch-blind, and this stream has to be the unpatched base. That already held, but only because SetBaseSnapshot runs before the patch is front-inserted. It now holds by construction. Also rewrites the front-insert comment, which credited the VM isolate for behavior that is required because linked patch code executes out of the base image. --- .../shell/common/shorebird/shorebird.cc | 15 ++++--- .../common/shorebird/snapshots_data_handle.cc | 41 ++++++++++--------- .../common/shorebird/snapshots_data_handle.h | 11 ++--- 3 files changed, 36 insertions(+), 31 deletions(-) diff --git a/engine/src/flutter/shell/common/shorebird/shorebird.cc b/engine/src/flutter/shell/common/shorebird/shorebird.cc index a539528d9da56..59c64cf76407a 100644 --- a/engine/src/flutter/shell/common/shorebird/shorebird.cc +++ b/engine/src/flutter/shell/common/shorebird/shorebird.cc @@ -267,9 +267,11 @@ void ConfigureShorebird(std::string code_cache_path, FML_LOG(INFO) << "Shorebird updater: active path: " << active_path; #if SHOREBIRD_USE_INTERPRETER - // On iOS we add the patch to the front of the list instead of clearing - // the list, to allow dart_snapshot.cc to still find the base snapshot - // for the vm isolate. + // Front-insert rather than clear. A linked patch carries only the code it + // rewrote, and everything it linked against still executes out of the base + // image, so the base has to stay resolvable for the life of the process. + // SearchMapping takes the first path that resolves a symbol, so the patch + // still wins the lookup without evicting the base. settings.application_library_paths.insert( settings.application_library_paths.begin(), active_path); #else @@ -300,9 +302,10 @@ void ConfigureShorebird(std::string code_cache_path, void* FileCallbacksImpl::Open() { #if SHOREBIRD_USE_INTERPRETER - return SnapshotsDataHandle::createForSnapshots(*vm_snapshot, - *isolate_snapshot) - .release(); + // vm_snapshot, not isolate_snapshot: both resolve the same symbol now, but + // only the VM path is patch-blind, and this stream must be the unpatched + // base. + return SnapshotsDataHandle::createForSnapshots(*vm_snapshot).release(); #else // SnapshotsDataHandle exists on all platforms (for testing) but is only used // on iOS. iOS patches are generated from just the Dart parts of the snapshot, diff --git a/engine/src/flutter/shell/common/shorebird/snapshots_data_handle.cc b/engine/src/flutter/shell/common/shorebird/snapshots_data_handle.cc index 3f8c7f6763794..a758dd15a00e5 100644 --- a/engine/src/flutter/shell/common/shorebird/snapshots_data_handle.cc +++ b/engine/src/flutter/shell/common/shorebird/snapshots_data_handle.cc @@ -82,32 +82,33 @@ BlobsIndex SnapshotsDataHandle::IndexForAbsoluteOffset(int64_t offset, } std::unique_ptr SnapshotsDataHandle::createForSnapshots( - const DartSnapshot& vm_snapshot, - const DartSnapshot& isolate_snapshot) { - // This needs to match the order in which the blobs are written out in - // analyze_snapshot --dump_blobs - auto vm_data = DataMapping(vm_snapshot); - auto iso_data = DataMapping(isolate_snapshot); - auto vm_insns = InstructionsMapping(vm_snapshot); - auto iso_insns = InstructionsMapping(isolate_snapshot); + const DartSnapshot& base_snapshot) { + // Order and count must match HandleDumpBlobs in + // runtime/bin/analyze_snapshot.cc, which writes the data region then the + // text region. One snapshot supplies both. The VM isolate's contents were + // folded into the isolate group, so kVMDataSymbol and kIsolateDataSymbol are + // now the same symbol and resolve to the same buffer. Taking a second + // snapshot here appends every byte twice and misaligns this stream against + // the host extraction the patch was generated from, with no error raised at + // either end. + // + // The caller must pass a snapshot resolved through the VM path. That path is + // patch-blind, and this stream has to be the unpatched base the diff was + // computed against. + auto data = DataMapping(base_snapshot); + auto insns = InstructionsMapping(base_snapshot); // Per-blob observability for the base byte stream the updater is about to // patch against. Logged at every patch-apply attempt so customer syslogs - // include the exact sizes the bipatch state machine sees โ€” directly + // include the exact sizes the bipatch state machine sees, directly // comparable to the host's `aot_tools dump_blobs` extraction. - FML_LOG(INFO) << "[shorebird] SnapshotsDataHandle blob sizes: vm_data=" - << vm_data->GetSize() << "b iso_data=" << iso_data->GetSize() - << "b vm_instructions=" << vm_insns->GetSize() - << "b iso_instructions=" << iso_insns->GetSize() << "b total=" - << (vm_data->GetSize() + iso_data->GetSize() + - vm_insns->GetSize() + iso_insns->GetSize()) - << "b"; + FML_LOG(INFO) << "[shorebird] SnapshotsDataHandle blob sizes: data=" + << data->GetSize() << "b instructions=" << insns->GetSize() + << "b total=" << (data->GetSize() + insns->GetSize()) << "b"; std::vector> blobs; - blobs.push_back(std::move(vm_data)); - blobs.push_back(std::move(iso_data)); - blobs.push_back(std::move(vm_insns)); - blobs.push_back(std::move(iso_insns)); + blobs.push_back(std::move(data)); + blobs.push_back(std::move(insns)); return std::make_unique(std::move(blobs)); } diff --git a/engine/src/flutter/shell/common/shorebird/snapshots_data_handle.h b/engine/src/flutter/shell/common/shorebird/snapshots_data_handle.h index 50c4b8179b412..b304d0d032dd2 100644 --- a/engine/src/flutter/shell/common/shorebird/snapshots_data_handle.h +++ b/engine/src/flutter/shell/common/shorebird/snapshots_data_handle.h @@ -15,9 +15,9 @@ struct BlobsIndex { size_t offset; }; -// Implements a POSIX file I/O interface which allows us to provide the four -// data blobs of a Dart snapshot (vm_data, vm_instructions, isolate_data, -// isolate_instructions) to Rust as though it were a single piece of memory. +// Implements a POSIX file I/O interface which allows us to provide the data +// and text regions of a Dart snapshot to Rust as though it were a single piece +// of memory. class SnapshotsDataHandle { public: // This would ideally be private, but we need to be able to call it from the @@ -25,9 +25,10 @@ class SnapshotsDataHandle { explicit SnapshotsDataHandle(std::vector> blobs) : blobs_(std::move(blobs)) {} + // `base_snapshot` must come from the VM resolve path, which never returns a + // patch. This stream is the base the updater diffs against. static std::unique_ptr createForSnapshots( - const DartSnapshot& vm_snapshot, - const DartSnapshot& isolate_snapshot); + const DartSnapshot& base_snapshot); uintptr_t Read(uint8_t* buffer, uintptr_t length); int64_t Seek(int64_t offset, int32_t whence); From 7b98251be979bbda299e223b87eecc798c4b1879 Mon Sep 17 00:00:00 2001 From: Nick Weatherley Date: Mon, 10 Aug 2026 14:24:35 -0400 Subject: [PATCH 79/95] chore: point DEPS at the published 3.47 dart-sdk prebuilt dart_sdk_revision moves to 1c05054e0833f9fc163e81e294a7ef1562e9f930 and the sidecar values come from the publish-darwin-arm64 run that produced that artifact (run 31415119236). Re-enables the macos-arm64 gcs hook. It was pinned to 'False' during the dry run because nothing had been published for the offsets-regen tip and gclient sync 404'd on it. Both halves of that are gone: the artifact exists at this revision, and the hook is not vestigial. Only the --no-prebuilt-dart-sdk shards build Dart from source; the plain --mac arm64/x64 shards and the ios-release shard all consume the prebuilt. --- DEPS | 17 +++++------------ 1 file changed, 5 insertions(+), 12 deletions(-) diff --git a/DEPS b/DEPS index 469cc76af2c60..10dc841929f5d 100644 --- a/DEPS +++ b/DEPS @@ -17,7 +17,7 @@ vars = { 'llvm_git': 'https://llvm.googlesource.com', 'dart_ai_rev': '9c96bfe5f091c9451eff5b59c9bffeb2e806b875', 'skia_revision': '8df24be66531469e576a806749a0202ae26b8d08', - "dart_sdk_revision": "046fbc29efd0bd56a7749e1e48a0793fe41780d8", + "dart_sdk_revision": "1c05054e0833f9fc163e81e294a7ef1562e9f930", "dart_sdk_git": "git@github.com:shorebirdtech/dart-sdk.git", "updater_git": "https://github.com/shorebirdtech/updater.git", "updater_rev": "1f85c4ab1ee5b540269b9859c75e1bffbb9050c7", @@ -415,20 +415,13 @@ deps = { 'objects': [ { 'object_name': Var('dart_sdk_revision') + '/dart-sdk-darwin-arm64.tar.gz', - 'sha256sum': '5968b313316b9edc8d6b003e9fb663e024568d00cea5efe0214191020412ffbf', - 'size_bytes': 205230386, - 'generation': 1780425558083300, + 'sha256sum': 'd47ac7c3fbec53d2d64a167bea6086a1d635f332995da2c93baa31c2ede069c3', + 'size_bytes': 215163753, + 'generation': 1786384359711456, } ], 'dep_type': 'gcs', - # Dry-run 3.47.0-rc0: mac-arm64 builds Dart from source - # (--no-prebuilt-dart-sdk in shards/macos.json), so this prebuilt is never - # consumed. Disabled because it was never published for the offsets-regen - # tip (046fbc2), which 404s gclient sync. Real release: either publish it - # via publish-darwin-arm64 + update sha256sum/size_bytes/generation, or drop - # this hook (it is vestigial post the #155 partial revert). See - # notes/3.47.0-rc0/deferred-followups.md. - 'condition': 'False' + 'condition': 'host_os == "mac" and download_dart_sdk' }, 'engine/src/flutter/prebuilts/windows-x64/dart-sdk': { 'packages': [ From 682fba935519ed39920cb530637e77e671a511cf Mon Sep 17 00:00:00 2001 From: Nick Weatherley Date: Mon, 10 Aug 2026 15:20:27 -0400 Subject: [PATCH 80/95] fix: restore 13 dependency pins reverted by a bad rebase Every engine shard that compiles harfbuzz failed: ninja: error: '../../flutter/third_party/harfbuzz/src/hb-paint-bounded.cc', needed by 'obj/.../harfbuzz_sources.hb-paint-bounded.o', missing and no known rule to make it 3b06709945a ("split shorebird C API consumption") took DEPS wholesale from a stale base and moved 13 third-party pins backward, undoing every upstream roll that had landed in between. It touched no build files, so each roll's build-file half survived while its DEPS half regressed, and the two halves have disagreed ever since. No later roll re-bumped any of them, so these are the newest values the branch has ever carried. harfbuzz is the case that surfaced. Roll a0060918304 added "src/hb-paint-bounded.cc" to build/secondary/flutter/third_party/harfbuzz/ BUILD.gn and bumped the pin in the same commit. That BUILD.gn at HEAD is byte-identical to the post-roll version. The file is present at 49844c32 and absent at ea6a172, confirmed against the mirror. Restored: clang, rapidjson, harfbuzz, glfw, shaderc, googletest, perfetto (which also moved back to chromium_git), freetype2, libpng, zlib, angle, imgui, and the rbe reclient_cfgs cipd version. Left alone: updater_rev, which that commit also changed but which has been deliberately re-set since, and the dart-sdk pin and its gcs hook. --- DEPS | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/DEPS b/DEPS index 10dc841929f5d..d6d8dcd910435 100644 --- a/DEPS +++ b/DEPS @@ -48,7 +48,7 @@ vars = { # updates to Clang Tidy will not turn the tree red. # # See https://github.com/flutter/flutter/wiki/Engine-preโ€submits-and-postโ€submits#post-submit - 'clang_version': 'git_revision:8c7a2ce01a77c96028fe2c8566f65c45ad9408d3', + 'clang_version': 'git_revision:80743bd43fd5b38fedc503308e7a652e23d3ec93', 'reclient_version': 're_client_version:0.185.0.db415f21-gomaip', @@ -240,10 +240,10 @@ deps = { Var('chromium_git') + '/chromium/tools/depot_tools.git' + '@' + '580b4ff3f5cd0dcaa2eacda28cefe0f45320e8f7', 'engine/src/flutter/third_party/rapidjson': - Var('flutter_git') + '/third_party/rapidjson' + '@' + 'ef3564c5c8824989393b87df25355baf35ff544b', + Var('flutter_git') + '/third_party/rapidjson' + '@' + '47253cab97e9cfe99dbd6b90836fc11589d7d802', 'engine/src/flutter/third_party/harfbuzz': - Var('flutter_git') + '/third_party/harfbuzz' + '@' + 'ea6a172f84f2cbcfed803b5ae71064c7afb6b5c2', + Var('flutter_git') + '/third_party/harfbuzz' + '@' + '49844c32a7a3f6be371355a1213c952a3f4a44e7', 'engine/src/flutter/third_party/libcxx': Var('llvm_git') + '/llvm-project/libcxx' + '@' + 'bd557f6f764d1e40b62528a13b124ce740624f8f', @@ -255,10 +255,10 @@ deps = { Var('llvm_git') + '/llvm-project/libc' + '@' + '5af39a19a1ad51ce93972cdab206dcd3ff9b6afa', 'engine/src/flutter/third_party/glfw': - Var('flutter_git') + '/third_party/glfw' + '@' + 'dd8a678a66f1967372e5a5e3deac41ebf65ee127', + Var('flutter_git') + '/third_party/glfw' + '@' + '9352d8fe93cd443be18157abe81f16500549aec0', 'engine/src/flutter/third_party/shaderc': - Var('chromium_git') + '/external/github.com/google/shaderc' + '@' + '37e25539ce199ecaf19fb7f7d27818716d36686d', + Var('chromium_git') + '/external/github.com/google/shaderc' + '@' + 'd15277d6bc180f6a0b8b601f0cab2bbcaac9b4d5', 'engine/src/flutter/third_party/vulkan-deps': Var('chromium_git') + '/vulkan-deps' + '@' + 'a9e2ca3b57aba86a22a2df1b84bf12f8cc98806e', @@ -276,7 +276,7 @@ deps = { Var('chromium_git') + '/external/github.com/google/benchmark' + '@' + '431abd149fd76a072f821913c0340137cc755f36', 'engine/src/flutter/third_party/googletest': - Var('chromium_git') + '/external/github.com/google/googletest' + '@' + '7f036c5563af7d0329f20e8bb42effb04629f0c0', + Var('chromium_git') + '/external/github.com/google/googletest' + '@' + 'e9907112b47255d50b4d343e7e2160bce8dc85d1', 'engine/src/flutter/third_party/re2': Var('chromium_git') + '/external/github.com/google/re2' + '@' + 'c84a140c93352cdabbfb547c531be34515b12228', @@ -305,7 +305,7 @@ deps = { {'dep_type': 'cipd', 'packages': [{'package': 'dart/third_party/flutter/devtools', 'version': 'git_revision:12d595649f189f1896722623f72599077f476848'}]}, 'engine/src/flutter/third_party/dart/third_party/perfetto/src': - Var('android_git') + '/platform/external/perfetto' + '@' + Var('dart_perfetto_rev'), + Var('chromium_git') + '/external/github.com/google/perfetto' + '@' + Var('dart_perfetto_rev'), 'engine/src/flutter/third_party/dart/third_party/pkg/core': Var('dart_git') + '/core.git' + '@' + Var('dart_core_rev'), @@ -490,7 +490,7 @@ deps = { Var('chromium_git') + '/external/github.com/libexpat/libexpat.git' + '@' + '8e49998f003d693213b538ef765814c7d21abada', 'engine/src/flutter/third_party/freetype2': - Var('flutter_git') + '/third_party/freetype2' + '@' + 'bfc3453fdc85d87b45c896f68bf2e49ebdaeef0a', + Var('flutter_git') + '/third_party/freetype2' + '@' + 'be4bcb57914154fc1b9e2900bf8e4b516057e2b8', 'engine/src/flutter/third_party/skia': Var('skia_git') + '/skia.git' + '@' + Var('skia_revision'), @@ -514,7 +514,7 @@ deps = { Var('skia_git') + '/external/github.com/google/wuffs-mirror-release-c.git' + '@' + '600cd96cf47788ee3a74b40a6028b035c9fd6a61', 'engine/src/flutter/third_party/zlib': - Var('chromium_git') + '/chromium/src/third_party/zlib.git' + '@' + '7d77fb7fd66d8a5640618ad32c71fdeb7d3e02df', + Var('chromium_git') + '/chromium/src/third_party/zlib.git' + '@' + '7eda07b1e067ef3fd7eea0419c88b5af45c9a776', 'engine/src/flutter/third_party/cpu_features/src': Var('chromium_git') + '/external/github.com/google/cpu_features.git' + '@' + '936b9ab5515dead115606559502e3864958f7f6e', @@ -535,7 +535,7 @@ deps = { Var('swiftshader_git') + '/SwiftShader.git' + '@' + '794b0cfce1d828d187637e6d932bae484fbe0976', 'engine/src/flutter/third_party/angle': - Var('flutter_git') + '/third_party/angle' + '@' + '6950c6c99fa1a2d653922871ede6679d74840289', + Var('flutter_git') + '/third_party/angle' + '@' + '84027aca9b71c9ba335bd000dad1107b8810a511', 'engine/src/flutter/third_party/vulkan_memory_allocator': Var('chromium_git') + '/external/github.com/GPUOpen-LibrariesAndSDKs/VulkanMemoryAllocator' + '@' + 'c788c52156f3ef7bc7ab769cb03c110a53ac8fcb', @@ -575,7 +575,7 @@ deps = { Var('dart_git') + '/external/github.com/google/vector_math.dart.git' + '@' + '0a5fd95449083d404df9768bc1b321b88a7d2eef', # 2.1.0 'engine/src/flutter/third_party/imgui': - Var('flutter_git') + '/third_party/imgui.git' + '@' + '3ea0fad204e994d669f79ed29dcaf61cd5cb571d', + Var('flutter_git') + '/third_party/imgui.git' + '@' + '2a1b69f05748ad909f03acf4533447cac1331611', 'engine/src/flutter/third_party/json': Var('flutter_git') + '/third_party/json.git' + '@' + '17d9eacd248f58b73f4d1be518ef649fe2295642', @@ -785,7 +785,7 @@ deps = { 'packages': [ { 'package': 'flutter_internal/rbe/reclient_cfgs', - 'version': 'LNMZdvF2Y86Dq05IWthtVJ_PswIFSRiywIHrkfHhelUC', + 'version': '0vARzGeIZgIhW7zVfWuqIPQ_HXMLDccjAstykWZKjaEC', } ], 'condition': 'use_rbe', From 7a54d5d9ff820859284867e6e60b54c0cc87e669 Mon Sep 17 00:00:00 2001 From: Nick Weatherley Date: Mon, 10 Aug 2026 15:51:57 -0400 Subject: [PATCH 81/95] fix: load one snapshot pair in the patch cache All three iOS shards failed to compile: patch_cache.cc:51:25: error: no matching function for call to 'Dart_LoadELF' note: candidate function not viable: requires at most 6 arguments, but 8 were provided Dart_LoadELF used to hand back a VM pair and an isolate pair. The VM isolate folded into the isolate group, so the VM and isolate symbols name the same buffers and the loader now resolves a single data/text pair from kSnapshotDataAsmSymbol and kSnapshotTextAsmSymbol. patch_cache.cc was the last caller still passing the split form; embedder.cc had already moved. Same root cause as the snapshots_data_handle fix: code written when the two snapshots were distinct, against a Dart where they no longer are. Verified by compiling the file against the 3.47 dart-sdk headers, and by confirming the pre-fix version reproduces the exact CI error under the same command. patch_mapping.cc, snapshots_data_handle.cc and updater.cc also compile clean against those headers. --- .../src/flutter/runtime/shorebird/patch_cache.cc | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) diff --git a/engine/src/flutter/runtime/shorebird/patch_cache.cc b/engine/src/flutter/runtime/shorebird/patch_cache.cc index 8c5cc3a83f182..e0dbf30e72b4e 100644 --- a/engine/src/flutter/runtime/shorebird/patch_cache.cc +++ b/engine/src/flutter/runtime/shorebird/patch_cache.cc @@ -40,17 +40,15 @@ std::shared_ptr PatchCacheEntry::Create( elf_mapping->GetSize()); const char* error = nullptr; - // The VM Snapshot is identical for all binaries produced by a given version - // of Dart. Our linker checks this and will fail to link if ever the VM - // snapshot changes. We ignore the VM data/instrs here. - const uint8_t* ignored_vm_data = nullptr; - const uint8_t* ignored_vm_instrs = nullptr; + // One data blob and one text blob. The VM isolate folded into the isolate + // group, so the VM and isolate symbols now name the same buffers and + // Dart_LoadELF resolves a single pair. const uint8_t* isolate_data = nullptr; const uint8_t* isolate_instrs = nullptr; - Dart_LoadedElf* elf = Dart_LoadELF( - path.c_str(), elf_file_offset, &error, &ignored_vm_data, - &ignored_vm_instrs, &isolate_data, &isolate_instrs, dart::bin::kReadOnly); + Dart_LoadedElf* elf = + Dart_LoadELF(path.c_str(), elf_file_offset, &error, &isolate_data, + &isolate_instrs, dart::bin::kReadOnly); if (elf == nullptr) { FML_LOG(ERROR) << "Failed to load patch at " << path << " error: " << error; From 85942c7e8979069859e6cc5304e85ec16bf97d27 Mon Sep 17 00:00:00 2001 From: Nick Weatherley Date: Tue, 11 Aug 2026 12:13:55 -0400 Subject: [PATCH 82/95] fix: load the patch snapshot on iOS TryLoadFromPatch compared the incoming symbol against its own copies of the snapshot symbol names. Folding the VM isolate into the isolate group renamed those symbols to kDartSnapshotData and kDartSnapshotText, and the copies in patch_cache.cc kept the old names, so every call fell through the filter and returned nullptr. iOS resolved both isolate snapshots from the base App.framework and ran base code on every launch while the updater reported a patch as running. Callers now pass a PatchSymbol enum naming which half of the isolate pair they want, so no symbol name crosses the boundary and there is nothing left to keep in sync. Correcting the strings alone would not have held, because the VM and isolate symbols share the same names after the fold and a string can no longer tell them apart. A patch that fails to load now falls back to the base image instead of aborting. That path was unreachable while the filter rejected every symbol. --- engine/src/flutter/runtime/dart_snapshot.cc | 7 ++-- .../flutter/runtime/shorebird/patch_cache.cc | 40 +++++-------------- .../flutter/runtime/shorebird/patch_cache.h | 27 ++++++++----- .../common/shorebird/patch_cache_unittests.cc | 31 ++++---------- 4 files changed, 37 insertions(+), 68 deletions(-) diff --git a/engine/src/flutter/runtime/dart_snapshot.cc b/engine/src/flutter/runtime/dart_snapshot.cc index d3c6be4373745..912ac0b000ebf 100644 --- a/engine/src/flutter/runtime/dart_snapshot.cc +++ b/engine/src/flutter/runtime/dart_snapshot.cc @@ -160,7 +160,7 @@ static std::shared_ptr ResolveIsolateData( #if SHOREBIRD_USE_INTERPRETER // Try loading from a Shorebird patch first. if (auto mapping = TryLoadFromPatch(settings.application_library_paths, - DartSnapshot::kIsolateDataSymbol)) { + PatchSymbol::kIsolateData)) { return mapping; } #endif // SHOREBIRD_USE_INTERPRETER @@ -185,9 +185,8 @@ static std::shared_ptr ResolveIsolateInstructions( #else // DART_SNAPSHOT_STATIC_LINK #if SHOREBIRD_USE_INTERPRETER // Try loading from a Shorebird patch first. - if (auto mapping = - TryLoadFromPatch(settings.application_library_paths, - DartSnapshot::kIsolateInstructionsSymbol)) { + if (auto mapping = TryLoadFromPatch(settings.application_library_paths, + PatchSymbol::kIsolateInstructions)) { return mapping; } #endif // SHOREBIRD_USE_INTERPRETER diff --git a/engine/src/flutter/runtime/shorebird/patch_cache.cc b/engine/src/flutter/runtime/shorebird/patch_cache.cc index e0dbf30e72b4e..10667cf296054 100644 --- a/engine/src/flutter/runtime/shorebird/patch_cache.cc +++ b/engine/src/flutter/runtime/shorebird/patch_cache.cc @@ -13,17 +13,6 @@ namespace flutter { -namespace { - -// These symbol names match the constants in dart_snapshot.cc. -// We duplicate them here rather than extracting them into a header. -// They are actually defined down in Dart and will never change. -constexpr const char* kIsolateDataSymbol = "kDartIsolateSnapshotData"; -constexpr const char* kIsolateInstructionsSymbol = - "kDartIsolateSnapshotInstructions"; - -} // namespace - // PatchCacheEntry implementation std::shared_ptr PatchCacheEntry::Create( @@ -121,7 +110,7 @@ void PatchCache::PruneExpired() { std::shared_ptr TryLoadFromPatch( const std::vector& native_library_paths, - const char* symbol_name) { + PatchSymbol symbol) { if (native_library_paths.empty()) { return nullptr; } @@ -133,31 +122,24 @@ std::shared_ptr TryLoadFromPatch( return nullptr; } - // Patches only contain isolate data/instructions, not VM data/instructions. - // Return nullptr for VM symbols to allow fallback to the base app. - std::string symbol(symbol_name); - if (symbol != kIsolateDataSymbol && symbol != kIsolateInstructionsSymbol) { - return nullptr; - } - // Load the patch using the cache. auto cache_entry = PatchCache::Instance().GetOrLoad(patch_path); if (!cache_entry) { - FML_LOG(FATAL) << "Failed to load symbol from patch at " << patch_path; + // Boot the base image rather than aborting, and say so, because the + // updater's own bookkeeping will still report this patch as running. + // PatchCacheEntry::Create has already logged why the load failed. + FML_LOG(ERROR) << "Shorebird: patch load failed, running base code from " + "the app image instead of " + << patch_path; return nullptr; } - FML_LOG(INFO) << "Loading symbol from patch: " << symbol_name; - - // ReportLaunchStart is now called from ResolveIsolateData in - // dart_snapshot.cc, which runs before TryLoadFromPatch on all platforms. - - if (symbol == kIsolateDataSymbol) { + if (symbol == PatchSymbol::kIsolateData) { + FML_LOG(INFO) << "Loading isolate data from patch"; return PatchMapping::CreateIsolateData(cache_entry); - } else { - FML_CHECK(symbol == kIsolateInstructionsSymbol); - return PatchMapping::CreateIsolateInstructions(cache_entry); } + FML_LOG(INFO) << "Loading isolate instructions from patch"; + return PatchMapping::CreateIsolateInstructions(cache_entry); } } // namespace flutter diff --git a/engine/src/flutter/runtime/shorebird/patch_cache.h b/engine/src/flutter/runtime/shorebird/patch_cache.h index 1f2e14fab711d..91ab0849a3d4d 100644 --- a/engine/src/flutter/runtime/shorebird/patch_cache.h +++ b/engine/src/flutter/runtime/shorebird/patch_cache.h @@ -81,23 +81,28 @@ class PatchCache { FML_DISALLOW_COPY_AND_ASSIGN(PatchCache); }; +/// Which half of the isolate snapshot pair a caller wants out of a patch. +/// +/// A patch carries only the isolate pair, never the VM pair. Callers name the +/// half they want instead of passing the snapshot symbol, because the VM and +/// isolate symbols share the same names and a name cannot say which pair it +/// belongs to. +enum class PatchSymbol { + kIsolateData, + kIsolateInstructions, +}; + /// Checks if the first path in native_library_paths is a Shorebird patch -/// (.vmcode file) and if so, attempts to load the requested symbol from -/// the patch. +/// (.vmcode file) and if so, loads the requested snapshot from the patch. /// /// @param native_library_paths The list of library paths to check. The first /// path is checked for the .vmcode extension. -/// @param symbol_name The symbol to load (kIsolateDataSymbol or -/// kIsolateInstructionsSymbol). -/// @return A mapping for the requested symbol if this is a patch and the -/// symbol is available in the patch, nullptr otherwise. -/// -/// Note: Patches only contain isolate data/instructions, not VM -/// data/instructions. For VM symbols, this will always return nullptr, -/// allowing the caller to fall back to loading from the base app. +/// @param symbol Which half of the isolate snapshot pair to load. +/// @return A mapping for the requested snapshot if this is a patch, nullptr +/// otherwise, which leaves the caller to fall back to the base app. std::shared_ptr TryLoadFromPatch( const std::vector& native_library_paths, - const char* symbol_name); + PatchSymbol symbol); } // namespace flutter diff --git a/engine/src/flutter/shell/common/shorebird/patch_cache_unittests.cc b/engine/src/flutter/shell/common/shorebird/patch_cache_unittests.cc index 51fc20f9ca562..1502e68e2c320 100644 --- a/engine/src/flutter/shell/common/shorebird/patch_cache_unittests.cc +++ b/engine/src/flutter/shell/common/shorebird/patch_cache_unittests.cc @@ -18,36 +18,19 @@ TEST(PatchCache, InstanceReturnsSameInstance) { EXPECT_EQ(&instance1, &instance2); } +// Every case below keeps a non-patch path at the front. A .vmcode front path +// commits to loading the file, and a load failure is fatal by design, so the +// load itself is only exercised end to end. + TEST(TryLoadFromPatch, ReturnsNullptrForEmptyPaths) { std::vector empty_paths; - auto result = TryLoadFromPatch(empty_paths, "kDartIsolateSnapshotData"); + auto result = TryLoadFromPatch(empty_paths, PatchSymbol::kIsolateData); EXPECT_EQ(result, nullptr); } TEST(TryLoadFromPatch, ReturnsNullptrForNonVmcodePath) { std::vector paths = {"/path/to/some/file.so"}; - auto result = TryLoadFromPatch(paths, "kDartIsolateSnapshotData"); - EXPECT_EQ(result, nullptr); -} - -TEST(TryLoadFromPatch, ReturnsNullptrForVmSymbol) { - // Even with a .vmcode path, VM symbols should return nullptr - // (we can't actually load the file, but we can verify the symbol check) - std::vector paths = {"/path/to/patch.vmcode"}; - - // VM data symbol should return nullptr (patches don't contain VM snapshots) - auto result_vm_data = TryLoadFromPatch(paths, "kDartVmSnapshotData"); - EXPECT_EQ(result_vm_data, nullptr); - - // VM instructions symbol should return nullptr - auto result_vm_instrs = - TryLoadFromPatch(paths, "kDartVmSnapshotInstructions"); - EXPECT_EQ(result_vm_instrs, nullptr); -} - -TEST(TryLoadFromPatch, ReturnsNullptrForUnknownSymbol) { - std::vector paths = {"/path/to/patch.vmcode"}; - auto result = TryLoadFromPatch(paths, "kSomeUnknownSymbol"); + auto result = TryLoadFromPatch(paths, PatchSymbol::kIsolateData); EXPECT_EQ(result, nullptr); } @@ -55,7 +38,7 @@ TEST(TryLoadFromPatch, ChecksOnlyFirstPath) { // Only the first path should be checked for .vmcode extension std::vector paths = {"/path/to/regular.so", "/path/to/patch.vmcode"}; - auto result = TryLoadFromPatch(paths, "kDartIsolateSnapshotData"); + auto result = TryLoadFromPatch(paths, PatchSymbol::kIsolateInstructions); // Should return nullptr because first path is not .vmcode EXPECT_EQ(result, nullptr); } From f988ad8aa299659603af93bd812c9f970e88b7da Mon Sep 17 00:00:00 2001 From: Nick Weatherley Date: Tue, 11 Aug 2026 15:06:19 -0400 Subject: [PATCH 83/95] fix(shorebird): report a failed patch load instead of silently running base A patch that installs but fails to load fell back to the base image without telling the updater. ReportLaunchStart has already promoted the patch to current_boot by that point, so the updater kept reporting it as running while base code executed, and the next launch retried the same broken patch. Report the failure, which marks it bad and activates the next best patch. The call is guarded once per process, so the second symbol lookup is a no-op. This is the reporting half of the gap that let the iOS symbol mismatch look like a working patch for a whole session. Also correct two comments claiming shorebird_report_launch_start() runs from TryLoadFromPatch(). It runs from ResolveIsolateData in runtime/dart_snapshot.cc, before any patch load is attempted. --- .../src/flutter/runtime/shorebird/patch_cache.cc | 11 ++++++++--- .../flutter/shell/common/shorebird/shorebird.cc | 16 ++++++++-------- 2 files changed, 16 insertions(+), 11 deletions(-) diff --git a/engine/src/flutter/runtime/shorebird/patch_cache.cc b/engine/src/flutter/runtime/shorebird/patch_cache.cc index 10667cf296054..c669c01a93a4c 100644 --- a/engine/src/flutter/runtime/shorebird/patch_cache.cc +++ b/engine/src/flutter/runtime/shorebird/patch_cache.cc @@ -9,6 +9,7 @@ #include "flutter/fml/logging.h" #include "flutter/fml/mapping.h" #include "flutter/runtime/shorebird/patch_mapping.h" +#include "flutter/shell/common/shorebird/updater.h" #include "third_party/dart/runtime/include/dart_api.h" namespace flutter { @@ -125,12 +126,16 @@ std::shared_ptr TryLoadFromPatch( // Load the patch using the cache. auto cache_entry = PatchCache::Instance().GetOrLoad(patch_path); if (!cache_entry) { - // Boot the base image rather than aborting, and say so, because the - // updater's own bookkeeping will still report this patch as running. - // PatchCacheEntry::Create has already logged why the load failed. + // Boot the base image rather than aborting. PatchCacheEntry::Create has + // already logged why the load failed. FML_LOG(ERROR) << "Shorebird: patch load failed, running base code from " "the app image instead of " << patch_path; + // ReportLaunchStart already promoted this patch to current_boot, so + // without a failure report it stays "running" in the updater's books + // while base code executes, and the next launch retries the same patch. + // Guarded once per process, so the second symbol lookup is a no-op. + shorebird::Updater::Instance().ReportLaunchFailure(); return nullptr; } diff --git a/engine/src/flutter/shell/common/shorebird/shorebird.cc b/engine/src/flutter/shell/common/shorebird/shorebird.cc index 59c64cf76407a..9d0ff168c83b5 100644 --- a/engine/src/flutter/shell/common/shorebird/shorebird.cc +++ b/engine/src/flutter/shell/common/shorebird/shorebird.cc @@ -194,10 +194,10 @@ bool ConfigureShorebird(const ShorebirdConfigArgs& args, FML_LOG(INFO) << "Shorebird updater: no active patch."; } - // Note: shorebird_report_launch_start() is now called from TryLoadFromPatch() - // in runtime/shorebird/patch_cache.cc, right before the patched snapshot is - // actually loaded. This fixes issues with FlutterEngineGroup and other cases - // where ConfigureShorebird() is called but no Shell is created. + // Launch reporting does not happen here. ResolveIsolateData in + // runtime/dart_snapshot.cc reports the start, so a FlutterEngineGroup or + // add-to-app host that calls ConfigureShorebird() without ever creating a + // Shell does not record a boot that never happened. if (!init_result) { return false; } @@ -282,10 +282,10 @@ void ConfigureShorebird(std::string code_cache_path, FML_LOG(INFO) << "Shorebird updater: no active patch."; } - // Note: shorebird_report_launch_start() is now called from TryLoadFromPatch() - // in runtime/shorebird/patch_cache.cc, right before the patched snapshot is - // actually loaded. This fixes issues with FlutterEngineGroup and other cases - // where ConfigureShorebird() is called but no Shell is created. + // Launch reporting does not happen here. ResolveIsolateData in + // runtime/dart_snapshot.cc reports the start, so a FlutterEngineGroup or + // add-to-app host that calls ConfigureShorebird() without ever creating a + // Shell does not record a boot that never happened. if (!init_result) { return; From 58f8f400a3e9718907ea74491c68a34faf07c555 Mon Sep 17 00:00:00 2001 From: Nick Weatherley Date: Thu, 13 Aug 2026 15:15:56 -0400 Subject: [PATCH 84/95] chore: bump dart_sdk_revision to 7a2e23f Picks up the 6-way debug language shard split plus the two Linux build guards. Prebuilt pin (sha256sum, size_bytes, generation) matches the published darwin-arm64 sidecar for this revision. Drops canvaskit_cipd_instance, which has no readers left in the tree. --- DEPS | 12 ++++-------- 1 file changed, 4 insertions(+), 8 deletions(-) diff --git a/DEPS b/DEPS index d6d8dcd910435..8ed93f2e3cafd 100644 --- a/DEPS +++ b/DEPS @@ -17,15 +17,11 @@ vars = { 'llvm_git': 'https://llvm.googlesource.com', 'dart_ai_rev': '9c96bfe5f091c9451eff5b59c9bffeb2e806b875', 'skia_revision': '8df24be66531469e576a806749a0202ae26b8d08', - "dart_sdk_revision": "1c05054e0833f9fc163e81e294a7ef1562e9f930", + "dart_sdk_revision": "7a2e23fcbdb8f3aff1a7a45388d551fef4d96e82", "dart_sdk_git": "git@github.com:shorebirdtech/dart-sdk.git", "updater_git": "https://github.com/shorebirdtech/updater.git", "updater_rev": "1f85c4ab1ee5b540269b9859c75e1bffbb9050c7", - # WARNING: DO NOT EDIT canvaskit_cipd_instance MANUALLY - # See `lib/web_ui/README.md` for how to roll CanvasKit to a new version. - 'canvaskit_cipd_instance': '61aeJQ9laGfEFF_Vlc_u0MCkqB6xb2hAYHRBxKH-Uw4C', - # Do not download the Emscripten SDK by default. # This prevents us from downloading the Emscripten toolchain for builds # which do not build for the web. This toolchain is needed to build CanvasKit @@ -415,9 +411,9 @@ deps = { 'objects': [ { 'object_name': Var('dart_sdk_revision') + '/dart-sdk-darwin-arm64.tar.gz', - 'sha256sum': 'd47ac7c3fbec53d2d64a167bea6086a1d635f332995da2c93baa31c2ede069c3', - 'size_bytes': 215163753, - 'generation': 1786384359711456, + 'sha256sum': 'bd742409ee43d7b6a52a98e638971940f1758739419f33db8e2b93bc66111519', + 'size_bytes': 215172456, + 'generation': 1786648280112953, } ], 'dep_type': 'gcs', From e8b68fbeda59d87cea56968e23e9d3bfca283337 Mon Sep 17 00:00:00 2001 From: Nick Weatherley Date: Fri, 14 Aug 2026 15:54:41 -0400 Subject: [PATCH 85/95] chore: bump dart_sdk_revision to 5aa1bc5 and its mac-arm64 prebuilt pins --- DEPS | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/DEPS b/DEPS index 8ed93f2e3cafd..188e11d74972a 100644 --- a/DEPS +++ b/DEPS @@ -17,7 +17,7 @@ vars = { 'llvm_git': 'https://llvm.googlesource.com', 'dart_ai_rev': '9c96bfe5f091c9451eff5b59c9bffeb2e806b875', 'skia_revision': '8df24be66531469e576a806749a0202ae26b8d08', - "dart_sdk_revision": "7a2e23fcbdb8f3aff1a7a45388d551fef4d96e82", + "dart_sdk_revision": "5aa1bc5026ae205df95378125b8e7645519df1d2", "dart_sdk_git": "git@github.com:shorebirdtech/dart-sdk.git", "updater_git": "https://github.com/shorebirdtech/updater.git", "updater_rev": "1f85c4ab1ee5b540269b9859c75e1bffbb9050c7", @@ -411,9 +411,9 @@ deps = { 'objects': [ { 'object_name': Var('dart_sdk_revision') + '/dart-sdk-darwin-arm64.tar.gz', - 'sha256sum': 'bd742409ee43d7b6a52a98e638971940f1758739419f33db8e2b93bc66111519', - 'size_bytes': 215172456, - 'generation': 1786648280112953, + 'sha256sum': '61852b431a59ca28b42214523ee1bcb38958049e723f845e903e40cff0de25f7', + 'size_bytes': 215209956, + 'generation': 1786735067318704, } ], 'dep_type': 'gcs', From 4506b5e29fa40a3d1fc2f8b4dcb019cff6690f6e Mon Sep 17 00:00:00 2001 From: Nick Weatherley Date: Mon, 17 Aug 2026 14:19:53 -0400 Subject: [PATCH 86/95] test: cover createForSnapshots blob count and order The existing tests build handles through the public constructor from a hand-made vector, so they cover the Read and Seek arithmetic and never touch blob count or order. That is the surface that was wrong: with kVMDataSymbol and kIsolateDataSymbol resolving to the same buffer, taking two snapshots appended every byte twice and misaligned the stream against the host's dump_blobs extraction, with no length validated at either end. --- .../snapshots_data_handle_unittests.cc | 66 +++++++++++++++++++ 1 file changed, 66 insertions(+) diff --git a/engine/src/flutter/shell/common/shorebird/snapshots_data_handle_unittests.cc b/engine/src/flutter/shell/common/shorebird/snapshots_data_handle_unittests.cc index 2acf44a7b8973..570f3dcf78355 100644 --- a/engine/src/flutter/shell/common/shorebird/snapshots_data_handle_unittests.cc +++ b/engine/src/flutter/shell/common/shorebird/snapshots_data_handle_unittests.cc @@ -173,5 +173,71 @@ TEST(SnapshotsDataHandle, SeekFromEnd) { EXPECT_EQ(buffer[1], 'l'); } +// Builds a snapshot whose data and instructions mappings are distinguishable, +// so a stream that repeats or reorders a region shows up in the bytes. +static fml::RefPtr MakeSnapshot( + const std::string& data, + const std::string& instructions) { + return fml::MakeRefCounted( + std::make_shared( + reinterpret_cast(data.data()), data.size()), + std::make_shared( + reinterpret_cast(instructions.data()), + instructions.size())); +} + +// Guards blob count and order, which the Read/Seek tests above never touch +// because they build handles through the public constructor. +// +// kVMDataSymbol and kIsolateDataSymbol now resolve to the same buffer, so a +// createForSnapshots taking two snapshots appended every byte twice and +// silently misaligned this stream against the host's dump_blobs extraction. +// Neither end validates the length, so only the bytes catch it. +TEST(SnapshotsDataHandle, CreateForSnapshotsEmitsDataThenInstructionsOnce) { + const std::string data = "DATA"; + const std::string instructions = "INSTRUCTIONS"; + auto snapshot = MakeSnapshot(data, instructions); + + auto handle = SnapshotsDataHandle::createForSnapshots(*snapshot); + + EXPECT_EQ(handle->FullSize(), data.size() + instructions.size()); + + std::vector buffer(handle->FullSize(), 0); + EXPECT_EQ(handle->Read(buffer.data(), buffer.size()), buffer.size()); + EXPECT_EQ(std::string(buffer.begin(), buffer.end()), data + instructions); +} + +// Order must match HandleDumpBlobs, which writes the data region then the text +// region. Asserted separately from the concatenation so a pure ordering +// regression names itself. +TEST(SnapshotsDataHandle, CreateForSnapshotsPutsDataBeforeInstructions) { + const std::string data = "AAAA"; + const std::string instructions = "BB"; + auto snapshot = MakeSnapshot(data, instructions); + + auto handle = SnapshotsDataHandle::createForSnapshots(*snapshot); + + uint8_t first[4] = {0, 0, 0, 0}; + EXPECT_EQ(handle->Read(first, 4), 4u); + EXPECT_EQ(std::string(first, first + 4), data); + + uint8_t rest[2] = {0, 0}; + EXPECT_EQ(handle->Read(rest, 2), 2u); + EXPECT_EQ(std::string(rest, rest + 2), instructions); +} + +// An empty region must not change the byte stream. The zero-length assumption +// this entry started from was wrong, so pin the behavior rather than the +// reasoning that produced it. +TEST(SnapshotsDataHandle, CreateForSnapshotsKeepsEmptyInstructionsRegion) { + const std::string data = "DATA"; + const std::string instructions; + auto snapshot = MakeSnapshot(data, instructions); + + auto handle = SnapshotsDataHandle::createForSnapshots(*snapshot); + + EXPECT_EQ(handle->FullSize(), data.size()); +} + } // namespace testing } // namespace flutter From a50914d711532a8076b7007ab9fab75c59f5c4d5 Mon Sep 17 00:00:00 2001 From: Nick Weatherley Date: Mon, 17 Aug 2026 17:08:05 -0400 Subject: [PATCH 87/95] chore: bump dart_sdk_revision to 140dd36 and its mac-arm64 prebuilt pins --- DEPS | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/DEPS b/DEPS index 188e11d74972a..92d9e3c50038c 100644 --- a/DEPS +++ b/DEPS @@ -17,7 +17,7 @@ vars = { 'llvm_git': 'https://llvm.googlesource.com', 'dart_ai_rev': '9c96bfe5f091c9451eff5b59c9bffeb2e806b875', 'skia_revision': '8df24be66531469e576a806749a0202ae26b8d08', - "dart_sdk_revision": "5aa1bc5026ae205df95378125b8e7645519df1d2", + "dart_sdk_revision": "140dd36ee98cf06934f1ea08feffc662a2426f96", "dart_sdk_git": "git@github.com:shorebirdtech/dart-sdk.git", "updater_git": "https://github.com/shorebirdtech/updater.git", "updater_rev": "1f85c4ab1ee5b540269b9859c75e1bffbb9050c7", @@ -411,9 +411,9 @@ deps = { 'objects': [ { 'object_name': Var('dart_sdk_revision') + '/dart-sdk-darwin-arm64.tar.gz', - 'sha256sum': '61852b431a59ca28b42214523ee1bcb38958049e723f845e903e40cff0de25f7', - 'size_bytes': 215209956, - 'generation': 1786735067318704, + 'sha256sum': 'd1398bbc19d16920cd17f17982d805cbf589ec27520dc3a6828978628e7ddf5c', + 'size_bytes': 215209717, + 'generation': 1786999893240455, } ], 'dep_type': 'gcs', From 6efff40cffca9cdf1b97cb02115cc8f1dbce729c Mon Sep 17 00:00:00 2001 From: Nick Weatherley Date: Tue, 18 Aug 2026 09:22:48 -0400 Subject: [PATCH 88/95] test: reconcile flutter_tools test expectations with upstream 3.47.0 Six tests carried 3.44-era expectations through the rebase while the code under test tracks upstream. All fixes are test-side; no source changed. The iOS deployment target moved to 15.0 upstream. build_test and common_test hardcode the literal where base/build.dart reads it from FlutterDarwinPlatform.ios.deploymentTarget(), so the expectation drifted; upstream's own ios_test.dart already reads 15.0. macos_test expected lipo -verify_arch to receive both archs. darwin.dart is identical to upstream and passes one, so the fake never matched and the expected throw never fired. version_test's clean-environment check is upstream-authored and does not know about the flutter_release branch lookup GitTagVersion.determine runs ahead of the tag lookup. Added that call to the fake sequence, keeping the clean-environment assertion on it so the test still covers what it names. --- .../test/general.shard/base/build_test.dart | 8 ++++---- .../build_system/targets/common_test.dart | 4 ++-- .../build_system/targets/macos_test.dart | 5 +---- .../test/general.shard/version_test.dart | 14 ++++++++++++++ 4 files changed, 21 insertions(+), 10 deletions(-) diff --git a/packages/flutter_tools/test/general.shard/base/build_test.dart b/packages/flutter_tools/test/general.shard/base/build_test.dart index 4781e4cbfac7c..52096ff03e205 100644 --- a/packages/flutter_tools/test/general.shard/base/build_test.dart +++ b/packages/flutter_tools/test/general.shard/base/build_test.dart @@ -17,7 +17,7 @@ const kWhichSysctlCommand = FakeCommand(command: ['which', 'sysctl']); const kARMCheckCommand = FakeCommand(command: ['sysctl', 'hw.optional.arm64'], exitCode: 1); const kDefaultClang = [ - '-miphoneos-version-min=13.0', + '-miphoneos-version-min=15.0', '-isysroot', 'path/to/sdk', '-dynamiclib', @@ -239,7 +239,7 @@ void main() { 'cc', '-arch', 'arm64', - '-miphoneos-version-min=13.0', + '-miphoneos-version-min=15.0', '-isysroot', 'path/to/sdk', '-c', @@ -313,7 +313,7 @@ void main() { 'cc', '-arch', 'arm64', - '-miphoneos-version-min=13.0', + '-miphoneos-version-min=15.0', '-isysroot', 'path/to/sdk', '-c', @@ -384,7 +384,7 @@ void main() { 'cc', '-arch', 'arm64', - '-miphoneos-version-min=13.0', + '-miphoneos-version-min=15.0', '-isysroot', 'path/to/sdk', '-c', diff --git a/packages/flutter_tools/test/general.shard/build_system/targets/common_test.dart b/packages/flutter_tools/test/general.shard/build_system/targets/common_test.dart index 5f4f49a315a16..d323e7a020f7d 100644 --- a/packages/flutter_tools/test/general.shard/build_system/targets/common_test.dart +++ b/packages/flutter_tools/test/general.shard/build_system/targets/common_test.dart @@ -831,7 +831,7 @@ void main() { 'cc', '-arch', 'arm64', - '-miphoneos-version-min=13.0', + '-miphoneos-version-min=15.0', '-isysroot', 'path/to/iPhoneOS.sdk', '-c', @@ -846,7 +846,7 @@ void main() { 'clang', '-arch', 'arm64', - '-miphoneos-version-min=13.0', + '-miphoneos-version-min=15.0', '-isysroot', 'path/to/iPhoneOS.sdk', '-dynamiclib', diff --git a/packages/flutter_tools/test/general.shard/build_system/targets/macos_test.dart b/packages/flutter_tools/test/general.shard/build_system/targets/macos_test.dart index f1c0f34d8a96a..0c1006f1df0c4 100644 --- a/packages/flutter_tools/test/general.shard/build_system/targets/macos_test.dart +++ b/packages/flutter_tools/test/general.shard/build_system/targets/macos_test.dart @@ -260,10 +260,7 @@ void main() { processManager.addCommands([ copyFrameworkCommand, lipoInfoFatCommand, - FakeCommand( - command: ['lipo', binary.path, '-verify_arch', 'arm64', 'x86_64'], - exitCode: 1, - ), + FakeCommand(command: ['lipo', binary.path, '-verify_arch', 'arm64'], exitCode: 1), ]); await expectLater( diff --git a/packages/flutter_tools/test/general.shard/version_test.dart b/packages/flutter_tools/test/general.shard/version_test.dart index 4357eda448701..4fe7d390ce1e6 100644 --- a/packages/flutter_tools/test/general.shard/version_test.dart +++ b/packages/flutter_tools/test/general.shard/version_test.dart @@ -1998,6 +1998,20 @@ void main() { environment: {'PATH': '/usr/bin'}, // GIT_DIR should be missing ), ); + processManager.addCommand( + const FakeCommand( + command: [ + 'git', + 'for-each-ref', + '--contains', + 'REVISION_A', + '--format', + '%(refname:short)', + 'refs/remotes/origin/flutter_release/*', + ], + environment: {'PATH': '/usr/bin'}, + ), + ); processManager.addCommand( const FakeCommand( command: [ From 28000c945005946bc5dd214496141d20117e8c1a Mon Sep 17 00:00:00 2001 From: Nick Weatherley Date: Tue, 18 Aug 2026 09:42:08 -0400 Subject: [PATCH 89/95] fix: match both symlink forms of system temp in the filesystem guard The guard resolved symlinks on the allowed root but not on the path being checked, so the two never matched on macOS, where Directory.systemTemp reports /var/folders/... and resolving it yields /private/var/folders/.... path.canonicalize normalizes without touching symlinks, so every createTempSync against the unresolved form was rejected. Upstream #187320 added the resolved root to fix callers that resolve the path themselves. Keeping both forms serves those callers and the unresolved ones together, and preserves the cached lookup that commit introduced to keep the check off the I/O path. Linux has no /var symlink and upstream runs test/general.shard on Linux alone, so nothing exercised this. The Shorebird matrix covers macOS too, where it accounted for 162 failures against zero on ubuntu. Applied to the devicelab copy as well to keep the two identical, matching the scope of #187320. --- dev/devicelab/lib/framework/fs_safety.dart | 28 +++++++++++++------ .../flutter_tools/test/src/fs_safety.dart | 28 +++++++++++++------ 2 files changed, 38 insertions(+), 18 deletions(-) diff --git a/dev/devicelab/lib/framework/fs_safety.dart b/dev/devicelab/lib/framework/fs_safety.dart index 1adbffb604c7f..a693e46b99e92 100644 --- a/dev/devicelab/lib/framework/fs_safety.dart +++ b/dev/devicelab/lib/framework/fs_safety.dart @@ -40,17 +40,22 @@ bool _isDangerousDirectory(String dirPath) { bool _isAllowedPath(String entityPath) { final String canonicalEntity = path.canonicalize(entityPath); - // Allow system temp - String canonicalTemp; + // Allow system temp. Callers reach temp by either name: `Directory.systemTemp` + // reports macOS's /var/folders/... while anything that resolves the path first + // yields /private/var/folders/.... `path.canonicalize` does not touch symlinks, + // so matching only one form rejects the other. + final List canonicalTemps; final io.IOOverrides? currentOverrides = io.IOOverrides.current; if (currentOverrides is FSGuardIOOverrides) { - canonicalTemp = currentOverrides._canonicalSystemTemp; + canonicalTemps = currentOverrides._canonicalSystemTemps; } else { - canonicalTemp = path.canonicalize(io.Directory.systemTemp.path); + canonicalTemps = [path.canonicalize(io.Directory.systemTemp.path)]; } - if (path.isWithin(canonicalTemp, canonicalEntity) || canonicalEntity == canonicalTemp) { - return true; + for (final String canonicalTemp in canonicalTemps) { + if (path.isWithin(canonicalTemp, canonicalEntity) || canonicalEntity == canonicalTemp) { + return true; + } } // Allow modifications inside the Flutter installation root itself @@ -573,15 +578,20 @@ final class FSGuardIOOverrides extends io.IOOverrides { final io.IOOverrides? _parent; - late final String _canonicalSystemTemp = () { + late final List _canonicalSystemTemps = () { final io.Directory rawTemp = _parent != null ? _parent.getSystemTempDirectory() : super.getSystemTempDirectory(); + final String rawCanonical = path.canonicalize(rawTemp.path); try { - return path.canonicalize(rawTemp.resolveSymbolicLinksSync()); + final String resolved = path.canonicalize(rawTemp.resolveSymbolicLinksSync()); + if (resolved != rawCanonical) { + return [rawCanonical, resolved]; + } } on Object catch (_) { - return path.canonicalize(rawTemp.path); + // Temp is unreadable; the unresolved form is the best available answer. } + return [rawCanonical]; }(); @override diff --git a/packages/flutter_tools/test/src/fs_safety.dart b/packages/flutter_tools/test/src/fs_safety.dart index 1adbffb604c7f..a693e46b99e92 100644 --- a/packages/flutter_tools/test/src/fs_safety.dart +++ b/packages/flutter_tools/test/src/fs_safety.dart @@ -40,17 +40,22 @@ bool _isDangerousDirectory(String dirPath) { bool _isAllowedPath(String entityPath) { final String canonicalEntity = path.canonicalize(entityPath); - // Allow system temp - String canonicalTemp; + // Allow system temp. Callers reach temp by either name: `Directory.systemTemp` + // reports macOS's /var/folders/... while anything that resolves the path first + // yields /private/var/folders/.... `path.canonicalize` does not touch symlinks, + // so matching only one form rejects the other. + final List canonicalTemps; final io.IOOverrides? currentOverrides = io.IOOverrides.current; if (currentOverrides is FSGuardIOOverrides) { - canonicalTemp = currentOverrides._canonicalSystemTemp; + canonicalTemps = currentOverrides._canonicalSystemTemps; } else { - canonicalTemp = path.canonicalize(io.Directory.systemTemp.path); + canonicalTemps = [path.canonicalize(io.Directory.systemTemp.path)]; } - if (path.isWithin(canonicalTemp, canonicalEntity) || canonicalEntity == canonicalTemp) { - return true; + for (final String canonicalTemp in canonicalTemps) { + if (path.isWithin(canonicalTemp, canonicalEntity) || canonicalEntity == canonicalTemp) { + return true; + } } // Allow modifications inside the Flutter installation root itself @@ -573,15 +578,20 @@ final class FSGuardIOOverrides extends io.IOOverrides { final io.IOOverrides? _parent; - late final String _canonicalSystemTemp = () { + late final List _canonicalSystemTemps = () { final io.Directory rawTemp = _parent != null ? _parent.getSystemTempDirectory() : super.getSystemTempDirectory(); + final String rawCanonical = path.canonicalize(rawTemp.path); try { - return path.canonicalize(rawTemp.resolveSymbolicLinksSync()); + final String resolved = path.canonicalize(rawTemp.resolveSymbolicLinksSync()); + if (resolved != rawCanonical) { + return [rawCanonical, resolved]; + } } on Object catch (_) { - return path.canonicalize(rawTemp.path); + // Temp is unreadable; the unresolved form is the best available answer. } + return [rawCanonical]; }(); @override From 54fcea59aa907b6d8f8890d01dd05e49da753e8e Mon Sep 17 00:00:00 2001 From: Nick Weatherley Date: Tue, 18 Aug 2026 09:58:03 -0400 Subject: [PATCH 90/95] fix: resolve symlinks on the checked path in the filesystem guard Matching both spellings of the temp root covers a symlinked head such as macOS's /var -> /private/var, but not a symlink further down. When temp itself is a link, the resolved root and the path being checked diverge mid-path and no string comparison of the two can succeed. Resolve the path under test as well, falling back to the nearest existing ancestor since paths are routinely checked before they exist. The comparison against the cached roots still runs first, so the added I/O happens only on paths that would otherwise be rejected. This is the case #187320 added its regression test for, which fails whenever it runs on its own. A full-file run happens to leave state that lets it pass, which is why the shard reported it inconsistently. --- dev/devicelab/lib/framework/fs_safety.dart | 37 +++++++++++++++++++ .../flutter_tools/test/src/fs_safety.dart | 37 +++++++++++++++++++ 2 files changed, 74 insertions(+) diff --git a/dev/devicelab/lib/framework/fs_safety.dart b/dev/devicelab/lib/framework/fs_safety.dart index a693e46b99e92..efa70f2571828 100644 --- a/dev/devicelab/lib/framework/fs_safety.dart +++ b/dev/devicelab/lib/framework/fs_safety.dart @@ -37,6 +37,31 @@ bool _isDangerousDirectory(String dirPath) { return false; } +/// [entityPath] with symlinks resolved. +/// +/// Paths are routinely checked before they exist, and `resolveSymbolicLinksSync` +/// throws on a missing path, so this resolves the nearest existing ancestor and +/// re-appends the rest. Returns null when no ancestor resolves. +String? _resolveExistingPrefix(String entityPath) { + final List tail = []; + String current = entityPath; + while (true) { + try { + final String resolved = io.Directory(current).resolveSymbolicLinksSync(); + return path.canonicalize( + tail.isEmpty ? resolved : path.joinAll([resolved, ...tail.reversed]), + ); + } on io.FileSystemException { + final String parent = path.dirname(current); + if (parent == current) { + return null; + } + tail.add(path.basename(current)); + current = parent; + } + } +} + bool _isAllowedPath(String entityPath) { final String canonicalEntity = path.canonicalize(entityPath); @@ -58,6 +83,18 @@ bool _isAllowedPath(String entityPath) { } } + // A symlink anywhere in the path, not just the macOS /var -> /private/var + // head, leaves the two sides spelled differently. Resolving costs I/O, so it + // runs only on paths the string comparison above already rejected. + final String? resolvedEntity = _resolveExistingPrefix(canonicalEntity); + if (resolvedEntity != null) { + for (final String canonicalTemp in canonicalTemps) { + if (path.isWithin(canonicalTemp, resolvedEntity) || resolvedEntity == canonicalTemp) { + return true; + } + } + } + // Allow modifications inside the Flutter installation root itself final String? flutterRoot = io.Platform.environment['FLUTTER_ROOT']; if (flutterRoot != null) { diff --git a/packages/flutter_tools/test/src/fs_safety.dart b/packages/flutter_tools/test/src/fs_safety.dart index a693e46b99e92..efa70f2571828 100644 --- a/packages/flutter_tools/test/src/fs_safety.dart +++ b/packages/flutter_tools/test/src/fs_safety.dart @@ -37,6 +37,31 @@ bool _isDangerousDirectory(String dirPath) { return false; } +/// [entityPath] with symlinks resolved. +/// +/// Paths are routinely checked before they exist, and `resolveSymbolicLinksSync` +/// throws on a missing path, so this resolves the nearest existing ancestor and +/// re-appends the rest. Returns null when no ancestor resolves. +String? _resolveExistingPrefix(String entityPath) { + final List tail = []; + String current = entityPath; + while (true) { + try { + final String resolved = io.Directory(current).resolveSymbolicLinksSync(); + return path.canonicalize( + tail.isEmpty ? resolved : path.joinAll([resolved, ...tail.reversed]), + ); + } on io.FileSystemException { + final String parent = path.dirname(current); + if (parent == current) { + return null; + } + tail.add(path.basename(current)); + current = parent; + } + } +} + bool _isAllowedPath(String entityPath) { final String canonicalEntity = path.canonicalize(entityPath); @@ -58,6 +83,18 @@ bool _isAllowedPath(String entityPath) { } } + // A symlink anywhere in the path, not just the macOS /var -> /private/var + // head, leaves the two sides spelled differently. Resolving costs I/O, so it + // runs only on paths the string comparison above already rejected. + final String? resolvedEntity = _resolveExistingPrefix(canonicalEntity); + if (resolvedEntity != null) { + for (final String canonicalTemp in canonicalTemps) { + if (path.isWithin(canonicalTemp, resolvedEntity) || resolvedEntity == canonicalTemp) { + return true; + } + } + } + // Allow modifications inside the Flutter installation root itself final String? flutterRoot = io.Platform.environment['FLUTTER_ROOT']; if (flutterRoot != null) { From 278f105b2a3246fa429125aba4786ef624fbbf91 Mon Sep 17 00:00:00 2001 From: Nick Weatherley Date: Fri, 21 Aug 2026 14:06:22 -0400 Subject: [PATCH 91/95] chore: bump dart_sdk_revision to 460d9f8 and its mac-arm64 prebuilt pins Points DEPS at the 3.47.1 dart-sdk RC. 460d9f8 is flutter_release_ex/3.47.1-rc0, the 158 fork commits from the 3.47.0 release replayed onto Dart 3.13.1. Upstream's 3.13.0 to 3.13.1 delta is 7 commits over 22 files. Only 0456d8c touches VM code the fork patches, and its hunks in il.cc/il.h are thousands of lines away from ours, so the rebase replayed with no conflicts and no dropped hunks. Dart CI 32501604550 green on all 39 jobs. The three gcs pins come from the sidecar JSON published by run 32510313294 and are cross-checked against the object itself. They must move with dart_sdk_revision or gclient sync fails on the mac shard. updater_rev is unchanged. Updater main has not moved since 3.47.0. --- DEPS | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/DEPS b/DEPS index 92d9e3c50038c..e197b27266e3c 100644 --- a/DEPS +++ b/DEPS @@ -17,7 +17,7 @@ vars = { 'llvm_git': 'https://llvm.googlesource.com', 'dart_ai_rev': '9c96bfe5f091c9451eff5b59c9bffeb2e806b875', 'skia_revision': '8df24be66531469e576a806749a0202ae26b8d08', - "dart_sdk_revision": "140dd36ee98cf06934f1ea08feffc662a2426f96", + "dart_sdk_revision": "460d9f8aa68a573922f48a4805c91c1c189a37e9", "dart_sdk_git": "git@github.com:shorebirdtech/dart-sdk.git", "updater_git": "https://github.com/shorebirdtech/updater.git", "updater_rev": "1f85c4ab1ee5b540269b9859c75e1bffbb9050c7", @@ -411,9 +411,9 @@ deps = { 'objects': [ { 'object_name': Var('dart_sdk_revision') + '/dart-sdk-darwin-arm64.tar.gz', - 'sha256sum': 'd1398bbc19d16920cd17f17982d805cbf589ec27520dc3a6828978628e7ddf5c', - 'size_bytes': 215209717, - 'generation': 1786999893240455, + 'sha256sum': '4cc3527453f3d81b38da36d5466da443241fe85a13f9ff9c92411f205acab703', + 'size_bytes': 215210172, + 'generation': 1787335424246884, } ], 'dep_type': 'gcs', From 315ff242617023c1c5549878bf17fad51ffec6f3 Mon Sep 17 00:00:00 2001 From: Nick Weatherley Date: Fri, 28 Aug 2026 12:28:28 -0400 Subject: [PATCH 92/95] chore: bump dart_sdk_revision to 52183de and its mac-arm64 prebuilt pins Rebased onto Flutter 3.47.2 / Dart 3.13.2. Updater unchanged at 1f85c4a. Carries the dart_dynamic_modules build repair (#836), which had landed only on flutter_release_ex/3.47.0-rc0, plus three CI fixes for setup-dart's problem matcher. --- DEPS | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/DEPS b/DEPS index e197b27266e3c..6e8520d33bc1c 100644 --- a/DEPS +++ b/DEPS @@ -17,7 +17,7 @@ vars = { 'llvm_git': 'https://llvm.googlesource.com', 'dart_ai_rev': '9c96bfe5f091c9451eff5b59c9bffeb2e806b875', 'skia_revision': '8df24be66531469e576a806749a0202ae26b8d08', - "dart_sdk_revision": "460d9f8aa68a573922f48a4805c91c1c189a37e9", + "dart_sdk_revision": "52183de0b928f738a4d9083ba64a9ba218a629ac", "dart_sdk_git": "git@github.com:shorebirdtech/dart-sdk.git", "updater_git": "https://github.com/shorebirdtech/updater.git", "updater_rev": "1f85c4ab1ee5b540269b9859c75e1bffbb9050c7", @@ -411,9 +411,9 @@ deps = { 'objects': [ { 'object_name': Var('dart_sdk_revision') + '/dart-sdk-darwin-arm64.tar.gz', - 'sha256sum': '4cc3527453f3d81b38da36d5466da443241fe85a13f9ff9c92411f205acab703', - 'size_bytes': 215210172, - 'generation': 1787335424246884, + 'sha256sum': '2bdd969b03d792b84e62be5a498c8220224ae2811e6ab0e1bbf5f848e2b4d2ee', + 'size_bytes': 215233237, + 'generation': 1787929282076125, } ], 'dep_type': 'gcs', From e16cf749ccaa38d7050335ff305def49b1c7c84c Mon Sep 17 00:00:00 2001 From: Nick Weatherley Date: Fri, 28 Aug 2026 14:18:10 -0400 Subject: [PATCH 93/95] chore: bump engine to 315ff24 Built on dart-sdk 52183de from flutter_release_ex/3.47.2-rc0. Base moved Flutter 3.47.1 -> 3.47.2 and Dart 3.13.1 -> 3.13.2; updater unchanged at 1f85c4a. Dart side carries the dart_dynamic_modules build repair (#836), which had landed only on flutter_release_ex/3.47.0-rc0 and would otherwise have been dropped by the rebase, plus three CI fixes disabling setup-dart's problem matcher across all three composite wrappers. Kernel SDK-hash audit clean: shipped dart-sdk vm_platform*.dill all hash-zeroed, const_finder carries upstream 60a57cd42d, both patched_sdk sets zeroed. --- bin/internal/engine.version | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bin/internal/engine.version b/bin/internal/engine.version index 9a4cf6446143d..9b5167d4cb0e3 100644 --- a/bin/internal/engine.version +++ b/bin/internal/engine.version @@ -1 +1 @@ -a804b261645ef8c13eb3d5c44a5c2fb0340c5539 +315ff242617023c1c5549878bf17fad51ffec6f3 From e137e182d71151220849e74a8d63c18a7856fa46 Mon Sep 17 00:00:00 2001 From: Eric Seidel Date: Fri, 11 Sep 2026 08:33:52 -0700 Subject: [PATCH 94/95] Extend --shorebird-trace to aar, ios-framework, and desktop builds (#136) Wires --shorebird-trace into: - BuildAarCommand and BuildFrameworkCommand (iOS frameworks) - addCommonDesktopBuildOptions in FlutterCommand, which covers desktop builds (linux, macos, windows) - AAR Gradle build path in AndroidGradleBuilder, mirroring the existing trace session usage in the assemble path The AAR path runs Gradle through _processUtils.run rather than _runGradleTask, so there is no onStart hook to feed onGradleSpawn; the trace still carries the outer gradle span and the per-task events from the init script, just without the Perfetto spawn arrow. Note: when 'flutter build aar' is invoked with multiple modes in a single run, each iteration's trace overwrites the previous one. Shorebird's release flow only builds release mode, so this is not a concern in practice. Claude-Session: https://claude.ai/code/session_01X2aY3kQvT2tgM1mYbN9c7W --- .../flutter_tools/lib/src/android/gradle.dart | 17 +++++++++++++++++ .../lib/src/commands/build_aar.dart | 1 + .../lib/src/commands/build_ios_framework.dart | 1 + .../lib/src/runner/flutter_command.dart | 1 + 4 files changed, 20 insertions(+) diff --git a/packages/flutter_tools/lib/src/android/gradle.dart b/packages/flutter_tools/lib/src/android/gradle.dart index ed2d17a0b0aa2..d4ffbc1c9e565 100644 --- a/packages/flutter_tools/lib/src/android/gradle.dart +++ b/packages/flutter_tools/lib/src/android/gradle.dart @@ -885,6 +885,16 @@ To fix this, you can either: final String aarTask = getAarTaskFor(buildInfo); final Status status = _logger.startProgress("Running Gradle task '$aarTask'..."); + // When a caller runs `flutter build aar` multiple modes in one + // invocation (debug + profile + release), each iteration's trace + // overwrites the previous one. Shorebird's release flow only + // builds release mode, so this isn't a concern in practice. + final AndroidBuildTraceSession? traceSession = AndroidBuildTraceSession.maybeStart( + androidBuildInfo, + _fileSystem, + project.android.buildDirectory, + ); + final String flutterRoot = _fileSystem.path.absolute(Cache.flutterRoot!); final String initScript = _fileSystem.path.join( flutterRoot, @@ -959,8 +969,10 @@ To fix this, you can either: command.add('-Ptarget-platform=$targetPlatforms'); } + command.addAll(traceSession?.extraGradleOptions(aarTask) ?? const []); command.add(aarTask); + traceSession?.onGradleAboutToStart(); final sw = Stopwatch()..start(); RunResult result; try { @@ -982,7 +994,10 @@ To fix this, you can either: ), ); + traceSession?.onGradleFinished(aarTask); + if (result.exitCode != 0) { + traceSession?.abortOnGradleFailure(); _logger.printStatus(result.stdout, wrap: false); _logger.printError(result.stderr, wrap: false); await _checkJavaAndGradleCompatibility(project, androidBuildInfo.buildInfo); @@ -993,10 +1008,12 @@ To fix this, you can either: } final Directory repoDirectory = getRepoDirectory(outputDirectory); if (!repoDirectory.existsSync()) { + traceSession?.abortOnGradleFailure(); _logger.printStatus(result.stdout, wrap: false); _logger.printError(result.stderr, wrap: false); throwToolExit('Gradle task $aarTask failed to produce $repoDirectory.', exitCode: exitCode); } + traceSession?.finish(buildTarget: 'aar', printStatus: _logger.printStatus); _logger.printStatus( '${_logger.terminal.successMark} ' 'Built ${_fileSystem.path.relative(repoDirectory.path)}', diff --git a/packages/flutter_tools/lib/src/commands/build_aar.dart b/packages/flutter_tools/lib/src/commands/build_aar.dart index 67561f9ce1049..bd7eaa5fb498d 100644 --- a/packages/flutter_tools/lib/src/commands/build_aar.dart +++ b/packages/flutter_tools/lib/src/commands/build_aar.dart @@ -49,6 +49,7 @@ class BuildAarCommand extends BuildSubCommand { usesExtraDartFlagOptions(verboseHelp: verboseHelp); usesTrackWidgetCreation(verboseHelp: false); addEnableExperimentation(hide: !verboseHelp); + usesShorebirdTraceOption(hide: !verboseHelp); addAndroidSpecificBuildOptions(hide: !verboseHelp); argParser.addMultiOption( 'target-platform', diff --git a/packages/flutter_tools/lib/src/commands/build_ios_framework.dart b/packages/flutter_tools/lib/src/commands/build_ios_framework.dart index 84e6b7a28d870..1f51fa469b28b 100644 --- a/packages/flutter_tools/lib/src/commands/build_ios_framework.dart +++ b/packages/flutter_tools/lib/src/commands/build_ios_framework.dart @@ -52,6 +52,7 @@ abstract class BuildFrameworkCommand extends BuildSubCommand { addDartObfuscationOption(); usesExtraDartFlagOptions(verboseHelp: verboseHelp); addEnableExperimentation(hide: !verboseHelp); + usesShorebirdTraceOption(hide: !verboseHelp); usesDarwinCodeSignXCFrameworksOption(); argParser diff --git a/packages/flutter_tools/lib/src/runner/flutter_command.dart b/packages/flutter_tools/lib/src/runner/flutter_command.dart index d99061f183d82..75c803f356021 100644 --- a/packages/flutter_tools/lib/src/runner/flutter_command.dart +++ b/packages/flutter_tools/lib/src/runner/flutter_command.dart @@ -1180,6 +1180,7 @@ abstract class FlutterCommand extends Command { usesDartDefineOption(); usesExtraDartFlagOptions(verboseHelp: verboseHelp); usesPubOption(); + usesShorebirdTraceOption(hide: !verboseHelp); usesTargetOption(); usesTrackWidgetCreation(verboseHelp: verboseHelp); usesBuildNumberOption(); From 169fea4478f6d0a6e8246d36a1b25e5d458cd0b4 Mon Sep 17 00:00:00 2001 From: Eric Seidel Date: Fri, 11 Sep 2026 08:50:55 -0700 Subject: [PATCH 95/95] Trace `flutter build macos` with --shorebird-trace `flutter build macos` runs xcodebuild with the same xcode_backend.dart script phase as `flutter build ios`, so the iOS trace session applies unchanged: pod install span, SHOREBIRD_TRACE_FILE forwarded into the build phase so `flutter assemble` writes its per-target trace, outer `xcode build` span, per-target subsections from xcresulttool, and the merged trace written to the path the CLI passed. The only thing the macOS path lacked was an xcresult bundle: the iOS build requests one for error reporting, macOS never did. The session now owns a temp bundle path for callers that don't bring their own (`ownResultBundleArgs`) and deletes it after parsing. `finish()` takes the build target so the outer span is `flutter build macos` rather than a hardcoded `ios`. Also fixes the compile error in build_macos_test.dart (the skipped shorebird.yaml test constructed BuildCommand without the arguments upstream has since made required), which was preventing the whole file from running. Claude-Session: https://claude.ai/code/session_01X2aY3kQvT2tgM1mYbN9c7W --- packages/flutter_tools/lib/src/ios/mac.dart | 2 +- .../lib/src/macos/build_macos.dart | 23 ++++++ .../shorebird/ios_build_trace_session.dart | 62 ++++++++++---- .../hermetic/build_macos_test.dart | 80 ++++++++++++++++++- 4 files changed, 150 insertions(+), 17 deletions(-) diff --git a/packages/flutter_tools/lib/src/ios/mac.dart b/packages/flutter_tools/lib/src/ios/mac.dart index ace212b6c3a8a..3b0a1ca5f3635 100644 --- a/packages/flutter_tools/lib/src/ios/mac.dart +++ b/packages/flutter_tools/lib/src/ios/mac.dart @@ -612,7 +612,7 @@ Future buildXcodeProject({ resultBundleDirectory: resultBundleDirectory, runXcresultTool: (List args) => globals.processManager.run(args), ); - traceSession?.finish(printStatus: globals.printStatus); + traceSession?.finish(buildTarget: 'ios', printStatus: globals.printStatus); if (tempDir.existsSync()) { // Display additional warning and error message from xcresult bundle. diff --git a/packages/flutter_tools/lib/src/macos/build_macos.dart b/packages/flutter_tools/lib/src/macos/build_macos.dart index 0173188189b4d..e9b5512d816f3 100644 --- a/packages/flutter_tools/lib/src/macos/build_macos.dart +++ b/packages/flutter_tools/lib/src/macos/build_macos.dart @@ -26,6 +26,7 @@ import '../migrations/xcode_project_object_version_migration.dart'; import '../migrations/xcode_script_build_phase_migration.dart'; import '../migrations/xcode_thin_binary_build_phase_input_paths_migration.dart'; import '../project.dart'; +import '../shorebird/ios_build_trace_session.dart'; import 'application_package.dart'; import 'cocoapod_utils.dart'; import 'darwin_dependency_management.dart'; @@ -187,7 +188,16 @@ Future buildMacOS({ } } + // Created early enough to capture pod install; see mac.dart. + final IosBuildTraceSession? traceSession = IosBuildTraceSession.maybeStart( + shorebirdTraceFilePath: buildInfo.shorebirdTraceFilePath, + fileSystem: globals.fs, + buildDirectoryPath: buildDirectoryPath, + ); + + traceSession?.onBeforePodInstall(); await processPodsIfNeeded(flutterProject.macos, buildDirectoryPath, buildInfo.mode); + traceSession?.onAfterPodInstall(); // If the xcfilelists do not exist, create empty version. if (!flutterProject.macos.inputFileList.existsSync()) { flutterProject.macos.inputFileList.createSync(recursive: true); @@ -196,6 +206,7 @@ Future buildMacOS({ flutterProject.macos.outputFileList.createSync(recursive: true); } if (configOnly) { + traceSession?.abortOnFailure(); return; } @@ -276,6 +287,7 @@ Future buildMacOS({ globals.fs.directory(buildDirectoryPath), skipPackageValidation: false, ); + traceSession?.onXcodeAboutToStart(); result = await globals.processUtils.stream( [ '/usr/bin/env', @@ -294,6 +306,10 @@ Future buildMacOS({ 'SYMROOT=${globals.fs.path.join(flutterBuildDir.absolute.path, 'Build', 'Products')}', if (verboseLogging) 'VERBOSE_SCRIPT_LOGGING=YES' else '-quiet', 'COMPILER_INDEX_STORE_ENABLE=NO', + if (traceSession != null) ...[ + ...traceSession.extraBuildCommands(), + ...traceSession.ownResultBundleArgs(), + ], if (disabledSandboxEntitlementFile != null) 'CODE_SIGN_ENTITLEMENTS=${disabledSandboxEntitlementFile.path}', // Pass EXCLUDED_ARCHS from Xcode project to xcodebuild command @@ -324,7 +340,13 @@ Future buildMacOS({ status.cancel(); } + await traceSession?.onXcodeFinished( + buildActionName: 'build', + runXcresultTool: (List args) => globals.processManager.run(args), + ); + if (result != 0) { + traceSession?.abortOnFailure(); if (hasMacOSMinDeploymentTargetIssue) { globals.logger.printError( _macOSDeploymentTargetTooLowMessage(macOSMinDeploymentTarget), @@ -350,6 +372,7 @@ Future buildMacOS({ ); } await _writeCodeSizeAnalysis(buildInfo, sizeAnalyzer); + traceSession?.finish(buildTarget: 'macos', printStatus: globals.printStatus); final Duration elapsedDuration = sw.elapsed; globals.analytics.send( Event.timing( diff --git a/packages/flutter_tools/lib/src/shorebird/ios_build_trace_session.dart b/packages/flutter_tools/lib/src/shorebird/ios_build_trace_session.dart index d8a7ea3da8e55..d6d0ee0736cbd 100644 --- a/packages/flutter_tools/lib/src/shorebird/ios_build_trace_session.dart +++ b/packages/flutter_tools/lib/src/shorebird/ios_build_trace_session.dart @@ -12,11 +12,14 @@ import 'package:shorebird_build_trace/shorebird_build_trace.dart'; import '../base/file_system.dart'; import 'network_trace_span.dart'; -/// Wraps the lifecycle of a Shorebird build trace across one iOS -/// build. Returned by [maybeStart] only when `--shorebird-trace=` -/// was passed; the constructor installs [BuildTracer.current] and -/// [finish] / [abortOnFailure] clear it, so mac.dart itself never -/// touches the static. +/// Wraps the lifecycle of a Shorebird build trace across one +/// xcodebuild-driven build (`flutter build ios` in mac.dart, `flutter +/// build macos` in build_macos.dart โ€” both run the same +/// xcode_backend.dart script phase, so the plumbing is identical). +/// Returned by [maybeStart] only when `--shorebird-trace=` was +/// passed; the constructor installs [BuildTracer.current] and +/// [finish] / [abortOnFailure] clear it, so the callers never touch +/// the static. /// /// Manual start/stop (rather than a body-wrapping closure) because /// wrapping the ~300-line `buildXcodeProject` body would force a @@ -83,6 +86,7 @@ class IosBuildTraceSession { DateTime? _xcodeStart; DateTime? _xcodeEnd; String? _assembleTraceFilePath; + Directory? _ownedResultBundleTempDir; /// Call immediately before `processPodsIfNeeded` so the resulting /// span covers its full wall-clock. @@ -123,6 +127,24 @@ class IosBuildTraceSession { return ['SHOREBIRD_TRACE_FILE=$assembleTrace']; } + /// Returns xcodebuild arguments that write an xcresult bundle into a + /// fresh temp directory, for callers that don't already request one + /// (`flutter build macos`). `flutter build ios` requests its own + /// bundle for error reporting and passes it to [onXcodeFinished] + /// directly. The temp directory is deleted by [onXcodeFinished]. + List ownResultBundleArgs() { + final Directory tempDir = _fs.systemTempDirectory.createTempSync('shorebird_trace_xcresult'); + _ownedResultBundleTempDir = tempDir; + return [ + '-resultBundlePath', + tempDir.childDirectory(_kOwnedResultBundleName).absolute.path, + '-resultBundleVersion', + '3', + ]; + } + + static const String _kOwnedResultBundleName = 'temporary_xcresult_bundle'; + /// Call right before `_runBuildWithRetries` so the outer build span /// can compute the pre-xcode setup interval. void onXcodeAboutToStart() { @@ -144,11 +166,13 @@ class IosBuildTraceSession { /// /// [runXcresultTool] is injected so callers can route through their /// own `ProcessManager`; the session avoids a direct `globals.*` - /// dependency. + /// dependency. [resultBundleDirectory] defaults to the bundle + /// requested by [ownResultBundleArgs]; subsection events are skipped + /// when neither is available. Future onXcodeFinished({ required String buildActionName, - required Directory resultBundleDirectory, required Future Function(List) runXcresultTool, + Directory? resultBundleDirectory, }) async { final xcodeEnd = DateTime.now(); _xcodeEnd = xcodeEnd; @@ -160,10 +184,18 @@ class IosBuildTraceSession { start: _xcodeStart ?? _buildStart, end: xcodeEnd, ); - await _emitXcodeSubsectionEvents( - resultBundleDirectory: resultBundleDirectory, - runXcresultTool: runXcresultTool, - ); + final Directory? ownedTempDir = _ownedResultBundleTempDir; + final Directory? bundle = + resultBundleDirectory ?? ownedTempDir?.childDirectory(_kOwnedResultBundleName); + if (bundle != null) { + await _emitXcodeSubsectionEvents( + resultBundleDirectory: bundle, + runXcresultTool: runXcresultTool, + ); + } + if (ownedTempDir != null && ownedTempDir.existsSync()) { + ownedTempDir.deleteSync(recursive: true); + } final String? assembleTraceFilePath = _assembleTraceFilePath; if (assembleTraceFilePath != null) { _tracer.mergeEventsFromFile(_fs.file(assembleTraceFilePath)); @@ -177,8 +209,10 @@ class IosBuildTraceSession { } /// Writes the merged trace to disk and clears [BuildTracer.current]. - /// Records the post-xcode + outer `flutter build ios` spans first. - void finish({required void Function(String) printStatus}) { + /// Records the post-xcode + outer `flutter build ` spans + /// first. [buildTarget] is `ios` or `macos`; the outer span name is + /// assembled as `TraceNames.flutterBuildSpanPrefix + buildTarget`. + void finish({required String buildTarget, required void Function(String) printStatus}) { final buildEnd = DateTime.now(); _tracer ..addCompleteEvent( @@ -190,7 +224,7 @@ class IosBuildTraceSession { end: buildEnd, ) ..addCompleteEvent( - name: '${TraceNames.flutterBuildSpanPrefix}ios', + name: '${TraceNames.flutterBuildSpanPrefix}$buildTarget', cat: TraceCategory.flutter.wireName, pid: _flutterPid, tid: _flutterToolTid, diff --git a/packages/flutter_tools/test/commands.shard/hermetic/build_macos_test.dart b/packages/flutter_tools/test/commands.shard/hermetic/build_macos_test.dart index a724a5866f7f9..f55be8565af31 100644 --- a/packages/flutter_tools/test/commands.shard/hermetic/build_macos_test.dart +++ b/packages/flutter_tools/test/commands.shard/hermetic/build_macos_test.dart @@ -155,7 +155,7 @@ void main() { String configuration, { bool verbose = false, void Function(List command)? onRun, - List? additionalCommandArguments, + List? additionalCommandArguments, String hostPlatformArch = 'x86_64', }) { final FlutterProject flutterProject = FlutterProject.fromDirectory(fileSystem.currentDirectory); @@ -164,7 +164,7 @@ void main() { ? 'platform=macOS,arch=$hostPlatformArch' : 'generic/platform=macOS'; return FakeCommand( - command: [ + command: [ '/usr/bin/env', 'xcrun', 'xcodebuild', @@ -492,6 +492,70 @@ STDERR STUFF }, ); + // Shorebird-specific: --shorebird-trace threads the assemble trace path + // and an xcresult bundle into xcodebuild and writes the merged trace. + testUsingContext( + 'macOS build with --shorebird-trace passes trace plumbing to xcodebuild and writes the trace', + () async { + final command = BuildCommand( + androidSdk: FakeAndroidSdk(), + buildSystem: TestBuildSystem.all(BuildResult(success: true)), + fileSystem: fileSystem, + logger: logger, + osUtils: FakeOperatingSystemUtils(), + config: FakeConfig(), + platform: FakePlatform(), + fileSystemUtils: FakeFileSystemUtils(), + terminal: FakeTerminal(), + plistParser: FakePlistParser(), + processUtils: FakeProcessUtils(), + processManager: FakeProcessManager.any(), + templateRenderer: FakeTemplateRenderer(), + xcode: FakeXcode(), + artifacts: FakeArtifacts(), + cache: FakeCache(), + flutterVersion: FakeFlutterVersion(), + ); + createMinimalMockProjectFiles(); + + await createTestCommandRunner( + command, + ).run(const ['build', 'macos', '--no-pub', '--shorebird-trace=build/trace.json']); + + final File traceFile = fileSystem.file('build/trace.json'); + expect(traceFile, exists); + expect(traceFile.readAsStringSync(), contains('"flutter build macos"')); + expect(traceFile.readAsStringSync(), contains('"xcode build"')); + expect(testLogger.statusText, contains('Shorebird build trace written to build/trace.json')); + // The session's temp xcresult directory is cleaned up. + expect( + fileSystem.systemTempDirectory.listSync().where( + (FileSystemEntity e) => e.basename.startsWith('shorebird_trace_xcresult'), + ), + isEmpty, + ); + }, + overrides: { + FileSystem: () => fileSystem, + ProcessManager: () => FakeProcessManager.list([ + setUpFakeXcodeBuildHandler( + 'Release', + additionalCommandArguments: [ + 'SHOREBIRD_TRACE_FILE=/build/macos/shorebird_assemble_trace.json', + '-resultBundlePath', + RegExp(r'^/\.tmp_rand\d+/shorebird_trace_xcresult.*/temporary_xcresult_bundle$'), + '-resultBundleVersion', + '3', + ], + ), + ]), + Platform: () => macosPlatform, + Pub: ThrowingPub.new, + FeatureFlags: () => TestFeatureFlags(isMacOSEnabled: true), + OperatingSystemUtils: () => FakeOperatingSystemUtils(hostPlatform: HostPlatform.darwin_x64), + }, + ); + testUsingContext( 'macOS build invokes xcode build (debug)', () async { @@ -1542,6 +1606,18 @@ STDERR STUFF fileSystem: MemoryFileSystem.test(), logger: BufferLogger.test(), osUtils: FakeOperatingSystemUtils(), + config: FakeConfig(), + platform: FakePlatform(), + fileSystemUtils: FakeFileSystemUtils(), + terminal: FakeTerminal(), + plistParser: FakePlistParser(), + processUtils: FakeProcessUtils(), + processManager: FakeProcessManager.any(), + templateRenderer: FakeTemplateRenderer(), + xcode: FakeXcode(), + artifacts: FakeArtifacts(), + cache: FakeCache(), + flutterVersion: FakeFlutterVersion(), ); createMinimalMockProjectFiles(); final File shorebirdYamlFile =