diff --git a/Configuration.props b/Configuration.props index f3f4619ff0e..3c21ea53d9a 100644 --- a/Configuration.props +++ b/Configuration.props @@ -99,7 +99,6 @@ $(MSBuildThisFileDirectory)external\sqlite $(MSBuildThisFileDirectory)external\libunwind $(BootstrapOutputDirectory)\libunwind - $(MSBuildThisFileDirectory)external\lz4 $(MSBuildThisFileDirectory) $(MSBuildThisFileDirectory)src-ThirdParty\ armeabi-v7a;x86 @@ -171,7 +170,6 @@ $([System.IO.Path]::GetFullPath ('$(SqliteSourceDirectory)')) $([System.IO.Path]::GetFullPath ('$(LibUnwindSourceDirectory)')) $([System.IO.Path]::GetFullPath ('$(LibUnwindGeneratedHeadersDirectory)')) - $([System.IO.Path]::GetFullPath ('$(LZ4SourceDirectory)')) net10.0 diff --git a/src/Xamarin.Android.Build.Tasks/Tasks/CompressAssemblies.cs b/src/Xamarin.Android.Build.Tasks/Tasks/CompressAssemblies.cs index 039cfc0626c..9089a66c821 100644 --- a/src/Xamarin.Android.Build.Tasks/Tasks/CompressAssemblies.cs +++ b/src/Xamarin.Android.Build.Tasks/Tasks/CompressAssemblies.cs @@ -7,9 +7,9 @@ namespace Xamarin.Android.Tasks; /// -/// Compresses assemblies using LZ4 compression before placing them in the APK. +/// Compresses assemblies using Zstandard compression before placing them in the APK. /// Note this is independent of whether they are stored compressed with ZIP in the APK. -/// Our runtime bits will LZ4 decompress them at assembly load time. +/// Our runtime bits will Zstd decompress them at assembly load time. /// public class CompressAssemblies : AndroidTask { diff --git a/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.ProjectTools/Resources/Base/BuildReleaseArm64SimpleDotNet.CoreCLR.apkdesc b/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.ProjectTools/Resources/Base/BuildReleaseArm64SimpleDotNet.CoreCLR.apkdesc index 0d3d9720ca1..9cd4d7cbfcb 100644 --- a/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.ProjectTools/Resources/Base/BuildReleaseArm64SimpleDotNet.CoreCLR.apkdesc +++ b/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.ProjectTools/Resources/Base/BuildReleaseArm64SimpleDotNet.CoreCLR.apkdesc @@ -8,16 +8,16 @@ "Size": 402352 }, "lib/arm64-v8a/libassembly-store.so": { - "Size": 3461344 + "Size": 2903016 }, "lib/arm64-v8a/libclrjit.so": { - "Size": 2804464 + "Size": 2824392 }, "lib/arm64-v8a/libcoreclr.so": { - "Size": 4872088 + "Size": 4890432 }, "lib/arm64-v8a/libmonodroid.so": { - "Size": 1325808 + "Size": 1265504 }, "lib/arm64-v8a/libSystem.Globalization.Native.so": { "Size": 72112 @@ -32,7 +32,7 @@ "Size": 168080 }, "lib/arm64-v8a/libxamarin-app.so": { - "Size": 20776 + "Size": 20984 }, "res/drawable-hdpi-v4/icon.png": { "Size": 2178 @@ -59,5 +59,5 @@ "Size": 1904 } }, - "PackageSize": 7497147 + "PackageSize": 7452091 } \ No newline at end of file diff --git a/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.ProjectTools/Resources/Base/BuildReleaseArm64XFormsDotNet.CoreCLR.apkdesc b/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.ProjectTools/Resources/Base/BuildReleaseArm64XFormsDotNet.CoreCLR.apkdesc index f156471b0da..042ee6a45db 100644 --- a/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.ProjectTools/Resources/Base/BuildReleaseArm64XFormsDotNet.CoreCLR.apkdesc +++ b/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.ProjectTools/Resources/Base/BuildReleaseArm64XFormsDotNet.CoreCLR.apkdesc @@ -32,16 +32,16 @@ "Size": 2396 }, "lib/arm64-v8a/libassembly-store.so": { - "Size": 14137920 + "Size": 11658648 }, "lib/arm64-v8a/libclrjit.so": { - "Size": 2804464 + "Size": 2824392 }, "lib/arm64-v8a/libcoreclr.so": { - "Size": 4872088 + "Size": 4890432 }, "lib/arm64-v8a/libmonodroid.so": { - "Size": 1325808 + "Size": 1265504 }, "lib/arm64-v8a/libSystem.Globalization.Native.so": { "Size": 72112 @@ -56,7 +56,7 @@ "Size": 168080 }, "lib/arm64-v8a/libxamarin-app.so": { - "Size": 147616 + "Size": 147824 }, "META-INF/androidx.activity_activity.version": { "Size": 6 @@ -2234,5 +2234,5 @@ "Size": 794696 } }, - "PackageSize": 20778573 + "PackageSize": 20786765 } \ No newline at end of file diff --git a/src/Xamarin.Android.Build.Tasks/Utilities/AssemblyCompression.cs b/src/Xamarin.Android.Build.Tasks/Utilities/AssemblyCompression.cs index ae852f42feb..da0b4704ef7 100644 --- a/src/Xamarin.Android.Build.Tasks/Utilities/AssemblyCompression.cs +++ b/src/Xamarin.Android.Build.Tasks/Utilities/AssemblyCompression.cs @@ -4,7 +4,6 @@ using System.Buffers; using System.IO; -using K4os.Compression.LZ4; using Microsoft.Android.Build.Tasks; using Microsoft.Build.Framework; using Microsoft.Build.Utilities; @@ -45,7 +44,7 @@ public void SetData (string sourcePath, uint descriptorIndex) } } - const uint CompressedDataMagic = 0x5A4C4158; // 'XALZ', little-endian + const uint CompressedDataMagic = 0x535A4158; // 'XAZS', little-endian static readonly ArrayPool bytePool = ArrayPool.Shared; @@ -75,11 +74,25 @@ static CompressionResult Compress (AssemblyData data, string outputFilePath) int fileSize = checked((int)fi.Length); sourceBytes = bytePool.Rent (fileSize); using (var fs = File.Open (data.SourcePath, FileMode.Open, FileAccess.Read, FileShare.Read)) { - bytesRead = fs.Read (sourceBytes, 0, fileSize); + bytesRead = 0; + while (bytesRead < fileSize) { + int read = fs.Read (sourceBytes, bytesRead, fileSize - bytesRead); + if (read == 0) + break; + + bytesRead += read; + } } - destBytes = bytePool.Rent (LZ4Codec.MaximumOutputSize (bytesRead)); - int encodedLength = LZ4Codec.Encode (sourceBytes, 0, bytesRead, destBytes, 0, destBytes.Length, LZ4Level.L12_MAX); + if (bytesRead != fileSize) + return CompressionResult.EncodingFailed; + + int maxOutputSize = Zstd.MaximumOutputSize (bytesRead); + if (maxOutputSize <= 0) + return CompressionResult.EncodingFailed; + + destBytes = bytePool.Rent (maxOutputSize); + int encodedLength = Zstd.Compress (sourceBytes, bytesRead, destBytes); if (encodedLength < 0) return CompressionResult.EncodingFailed; @@ -154,7 +167,7 @@ public static bool TryGetDescriptorIndex (TaskLoggingHelper log, ITaskItem assem public static string GetCompressedAssemblyOutputPath (ITaskItem assembly, string compressedOutputDir) { var assemblyOutputDir = GetCompressedAssemblyOutputDirectory (assembly, compressedOutputDir); - return Path.Combine (assemblyOutputDir, $"{Path.GetFileName (assembly.ItemSpec)}.lz4"); + return Path.Combine (assemblyOutputDir, $"{Path.GetFileName (assembly.ItemSpec)}.zst"); } static string GetCompressedAssemblyOutputDirectory (ITaskItem assembly, string compressedOutputDir) diff --git a/src/Xamarin.Android.Build.Tasks/Utilities/AssemblyStoreAssemblyInfo.cs b/src/Xamarin.Android.Build.Tasks/Utilities/AssemblyStoreAssemblyInfo.cs index 590ca553c7d..a8c38d99a29 100644 --- a/src/Xamarin.Android.Build.Tasks/Utilities/AssemblyStoreAssemblyInfo.cs +++ b/src/Xamarin.Android.Build.Tasks/Utilities/AssemblyStoreAssemblyInfo.cs @@ -34,7 +34,7 @@ public AssemblyStoreAssemblyInfo (string sourceFilePath, ITaskItem assembly, boo throw new InvalidOperationException ("Internal error: info without assembly name"); } - if (name.EndsWith (".lz4", StringComparison.OrdinalIgnoreCase)) { + if (name.EndsWith (".zst", StringComparison.OrdinalIgnoreCase)) { name = Path.GetFileNameWithoutExtension (name); } diff --git a/src/Xamarin.Android.Build.Tasks/Utilities/NativeRuntimeComponents.cs b/src/Xamarin.Android.Build.Tasks/Utilities/NativeRuntimeComponents.cs index 4773d0cd1c1..141bab21947 100644 --- a/src/Xamarin.Android.Build.Tasks/Utilities/NativeRuntimeComponents.cs +++ b/src/Xamarin.Android.Build.Tasks/Utilities/NativeRuntimeComponents.cs @@ -125,7 +125,6 @@ public NativeRuntimeComponents (ITaskItem[]? monoComponents) new AndroidArchive ("libruntime-base-common-release.a"), new AndroidArchive ("libruntime-base-release.a"), new AndroidArchive ("libxa-java-interop-release.a"), - new AndroidArchive ("libxa-lz4-release.a"), new AndroidArchive ("libxa-shared-bits-release.a"), new AndroidArchive ("libxamarin-startup-release.a"), diff --git a/src/Xamarin.Android.Build.Tasks/Utilities/Zstd.cs b/src/Xamarin.Android.Build.Tasks/Utilities/Zstd.cs new file mode 100644 index 00000000000..775c66fff6e --- /dev/null +++ b/src/Xamarin.Android.Build.Tasks/Utilities/Zstd.cs @@ -0,0 +1,118 @@ +#nullable enable +using System; +using System.Runtime.InteropServices; + +namespace Xamarin.Android.Tasks +{ + /// + /// Minimal managed wrapper around the Zstandard compression functions exported by + /// libSystem.IO.Compression.Native, which ships in the .NET runtime pack. + /// We use it to compress assemblies that are placed in the assembly store; the native + /// runtime decompresses them at load time using the same library. + /// + static class Zstd + { + // libSystem.IO.Compression.Native exports the raw zstd entry points (no prefix). + const string ZstdLibrary = "System.IO.Compression.Native"; + + // ZSTD_cParameter value for ZSTD_c_compressionLevel (see zstd.h). + const int ZSTD_c_compressionLevel = 100; + const int ZstdCompressionLevel = 3; + + [DllImport (ZstdLibrary, CallingConvention = CallingConvention.Cdecl)] + static extern UIntPtr ZSTD_compressBound (UIntPtr srcSize); + + [DllImport (ZstdLibrary, CallingConvention = CallingConvention.Cdecl)] + static extern IntPtr ZSTD_createCCtx (); + + [DllImport (ZstdLibrary, CallingConvention = CallingConvention.Cdecl)] + static extern UIntPtr ZSTD_freeCCtx (IntPtr cctx); + + [DllImport (ZstdLibrary, CallingConvention = CallingConvention.Cdecl)] + static extern UIntPtr ZSTD_CCtx_setParameter (IntPtr cctx, int param, int value); + + [DllImport (ZstdLibrary, CallingConvention = CallingConvention.Cdecl)] + static extern UIntPtr ZSTD_compress2 (IntPtr cctx, byte[] dst, UIntPtr dstCapacity, byte[] src, UIntPtr srcSize); + + [DllImport (ZstdLibrary, CallingConvention = CallingConvention.Cdecl)] + static extern uint ZSTD_isError (UIntPtr code); + + /// + /// Returns the maximum size that compressed data of bytes can occupy. + /// + public static int MaximumOutputSize (int inputSize) + { + if (inputSize < 0) + return -1; + + try { + UIntPtr result = ZSTD_compressBound ((UIntPtr) (uint) inputSize); + if (ZSTD_isError (result) != 0) + return -1; + + ulong maxOutputSize = (ulong) result; + if (maxOutputSize > int.MaxValue) + return -1; + + return (int) maxOutputSize; + } catch (DllNotFoundException) { + return -1; + } catch (EntryPointNotFoundException) { + return -1; + } + } + + /// + /// Compresses bytes from into + /// using zstd's default compression level. Returns the number of + /// bytes written to , or -1 if compression failed. + /// + public static int Compress (byte[] input, int inputLength, byte[] output) + { + if (input == null) + throw new ArgumentNullException (nameof (input)); + if (output == null) + throw new ArgumentNullException (nameof (output)); + if (inputLength < 0 || inputLength > input.Length) + throw new ArgumentOutOfRangeException (nameof (inputLength)); + + IntPtr cctx; + try { + cctx = ZSTD_createCCtx (); + } catch (DllNotFoundException) { + return -1; + } catch (EntryPointNotFoundException) { + return -1; + } + + if (cctx == IntPtr.Zero) + return -1; + + try { + UIntPtr setParameterResult = ZSTD_CCtx_setParameter (cctx, ZSTD_c_compressionLevel, ZstdCompressionLevel); + if (ZSTD_isError (setParameterResult) != 0) + return -1; + + UIntPtr result = ZSTD_compress2 (cctx, output, (UIntPtr) (uint) output.Length, input, (UIntPtr) (uint) inputLength); + if (ZSTD_isError (result) != 0) + return -1; + + ulong encodedLength = (ulong) result; + if (encodedLength > int.MaxValue) + return -1; + + return (int) encodedLength; + } catch (DllNotFoundException) { + return -1; + } catch (EntryPointNotFoundException) { + return -1; + } finally { + try { + ZSTD_freeCCtx (cctx); + } catch (DllNotFoundException) { + } catch (EntryPointNotFoundException) { + } + } + } + } +} diff --git a/src/Xamarin.Android.Build.Tasks/Xamarin.Android.Common.targets b/src/Xamarin.Android.Build.Tasks/Xamarin.Android.Common.targets index ea3cf49c141..47d1a76d490 100644 --- a/src/Xamarin.Android.Build.Tasks/Xamarin.Android.Common.targets +++ b/src/Xamarin.Android.Build.Tasks/Xamarin.Android.Common.targets @@ -2150,7 +2150,7 @@ because xbuild doesn't support framework reference assemblies. This dependency chain should be addressed in a separate PR. --> -#if defined (HAVE_LZ4) -#include -#endif - #include #include #include #include #include #include +#include using namespace xamarin::android; @@ -26,7 +23,7 @@ auto AssemblyStore::get_assembly_data (AssemblyStoreSingleAssemblyRuntimeData co uint8_t *assembly_data = nullptr; uint32_t assembly_data_size = 0; -#if defined (HAVE_LZ4) && defined (RELEASE) +#if defined (RELEASE) auto header = reinterpret_cast(e.image_data); if (header->magic == COMPRESSED_DATA_MAGIC) { log_debug (LOG_ASSEMBLY, "Decompressing assembly '{}' from the assembly store"sv, name); @@ -114,20 +111,20 @@ auto AssemblyStore::get_assembly_data (AssemblyStoreSingleAssemblyRuntimeData co } const char *data_start = pointer_add(e.image_data, sizeof(CompressedAssemblyHeader)); - int ret = LZ4_decompress_safe (data_start, reinterpret_cast(data_buffer), static_cast(assembly_data_size), static_cast(cad.uncompressed_file_size)); + size_t ret = ZSTD_decompress (data_buffer, cad.uncompressed_file_size, data_start, assembly_data_size); - if (ret < 0) { + if (ZSTD_isError (ret)) { Helpers::abort_application ( LOG_ASSEMBLY, std::format ( - "Decompression of assembly {} failed with code {}"sv, + "Decompression of assembly {} failed: {}"sv, name, - ret + ZSTD_getErrorName (ret) ) ); } - if (static_cast(ret) != cad.uncompressed_file_size) { + if (ret != cad.uncompressed_file_size) { Helpers::abort_application ( LOG_ASSEMBLY, std::format ( @@ -147,7 +144,7 @@ auto AssemblyStore::get_assembly_data (AssemblyStoreSingleAssemblyRuntimeData co set_assembly_data_and_size (data_buffer, cad.uncompressed_file_size, assembly_data, assembly_data_size); } else -#endif // def HAVE_LZ4 && def RELEASE +#endif // def RELEASE { log_debug (LOG_ASSEMBLY, "Assembly '{}' is not compressed in the assembly store"sv, name); diff --git a/src/native/clr/include/xamarin-app.hh b/src/native/clr/include/xamarin-app.hh index c2140e2c882..39fbb97f822 100644 --- a/src/native/clr/include/xamarin-app.hh +++ b/src/native/clr/include/xamarin-app.hh @@ -9,7 +9,7 @@ #include static constexpr uint64_t FORMAT_TAG = 0x00045E6972616D58; // 'Xmari^XY' where XY is the format version -static constexpr uint32_t COMPRESSED_DATA_MAGIC = 0x5A4C4158; // 'XALZ', little-endian +static constexpr uint32_t COMPRESSED_DATA_MAGIC = 0x535A4158; // 'XAZS', little-endian static constexpr uint32_t ASSEMBLY_STORE_MAGIC = 0x41424158; // 'XABA', little-endian // The highest bit of assembly store version is a 64-bit ABI flag diff --git a/src/native/common/include/runtime-base/timing-internal.hh b/src/native/common/include/runtime-base/timing-internal.hh index 31099b86322..5bd95f11af4 100644 --- a/src/native/common/include/runtime-base/timing-internal.hh +++ b/src/native/common/include/runtime-base/timing-internal.hh @@ -430,7 +430,7 @@ namespace xamarin::android { switch (kind) { case TimingEventKind::AssemblyDecompression: - append_desc ("LZ4 decompression time for "sv); + append_desc ("Zstd decompression time for "sv); return; case TimingEventKind::AssemblyLoad: diff --git a/src/native/common/include/runtime-base/zstd.hh b/src/native/common/include/runtime-base/zstd.hh new file mode 100644 index 00000000000..bbab4c91471 --- /dev/null +++ b/src/native/common/include/runtime-base/zstd.hh @@ -0,0 +1,18 @@ +// Dear Emacs, this is a -*- C++ -*- header +#pragma once + +#include + +// +// Minimal declarations for the Zstandard decompression functions that are exported by +// `libSystem.IO.Compression.Native`, which ships in the .NET runtime pack and is linked +// into our runtime. The assembly store compresses assemblies with Zstd at build time and +// we decompress them here at load time. +// +// We declare only the few entry points we need instead of pulling in `zstd.h`. +// +extern "C" { + size_t ZSTD_decompress (void *dst, size_t dst_capacity, const void *src, size_t compressed_size) noexcept; + unsigned ZSTD_isError (size_t code) noexcept; + const char* ZSTD_getErrorName (size_t code) noexcept; +} diff --git a/src/native/common/lz4/CMakeLists.txt b/src/native/common/lz4/CMakeLists.txt deleted file mode 100644 index 57c8a717d01..00000000000 --- a/src/native/common/lz4/CMakeLists.txt +++ /dev/null @@ -1,42 +0,0 @@ -set(LIB_NAME xa-lz4) -set(LIB_ALIAS xa::lz4) - -set(LZ4_SRC_DIR "${EXTERNAL_DIR}/lz4/lib") -set(LZ4_INCLUDE_DIR ${LZ4_SRC_DIR}) - -set(LZ4_SOURCES - ${LZ4_SRC_DIR}/lz4.c -) - -add_library( - ${LIB_NAME} - STATIC - ${LZ4_SOURCES} -) -set_static_library_suffix(${LIB_NAME}) - -add_library(${LIB_ALIAS} ALIAS ${LIB_NAME}) - -target_compile_definitions( - ${LIB_NAME} - PRIVATE - # Ugly, but this is the only way to change LZ4 symbols visibility without modifying lz4.h - "LZ4LIB_VISIBILITY=__attribute__ ((visibility (\"hidden\")))" - XXH_NAMESPACE=LZ4_ -) - -target_include_directories( - ${LIB_NAME} - PUBLIC - "$" -) - -if(DEBUG_BUILD) - set_target_properties( - ${LIB_NAME} - PROPERTIES - ARCHIVE_OUTPUT_DIRECTORY "${CMAKE_CURRENT_BINARY_DIR}" - ) -endif() - -xa_add_compile_definitions(${LIB_NAME}) diff --git a/src/native/mono/monodroid/CMakeLists.txt b/src/native/mono/monodroid/CMakeLists.txt index 888b87d3d98..43da300d837 100644 --- a/src/native/mono/monodroid/CMakeLists.txt +++ b/src/native/mono/monodroid/CMakeLists.txt @@ -152,7 +152,6 @@ macro(lib_target_options TARGET_NAME) ${TARGET_NAME} PRIVATE HAVE_CONFIG_H - HAVE_LZ4 JI_DLL_EXPORT JI_NO_VISIBILITY MONO_DLL_EXPORT @@ -231,7 +230,7 @@ macro(lib_target_options TARGET_NAME) xa::runtime-base xa::java-interop xa::pinvoke-override-precompiled - xa::lz4 + -lSystem.IO.Compression.Native -lmonosgen-2.0 -landroid -llog diff --git a/src/native/mono/monodroid/embedded-assemblies.cc b/src/native/mono/monodroid/embedded-assemblies.cc index c3aa3488aa8..08099f65324 100644 --- a/src/native/mono/monodroid/embedded-assemblies.cc +++ b/src/native/mono/monodroid/embedded-assemblies.cc @@ -15,10 +15,6 @@ #include #include -#if defined (HAVE_LZ4) -#include -#endif - #include #include #include @@ -37,6 +33,7 @@ #include "startup-aware-lock.hh" #include #include +#include using namespace xamarin::android; using namespace xamarin::android::internal; @@ -82,7 +79,7 @@ EmbeddedAssemblies::set_assembly_data_and_size (uint8_t* source_assembly_data, u [[gnu::always_inline]] void EmbeddedAssemblies::get_assembly_data (uint8_t *data, uint32_t data_size, [[maybe_unused]] const char *name, uint8_t*& assembly_data, uint32_t& assembly_data_size) noexcept { -#if defined (HAVE_LZ4) && defined (RELEASE) +#if defined (RELEASE) auto header = reinterpret_cast(data); if (header->magic == COMPRESSED_DATA_MAGIC) { if (compressed_assembly_count == 0) [[unlikely]] { @@ -156,20 +153,20 @@ EmbeddedAssemblies::get_assembly_data (uint8_t *data, uint32_t data_size, [[mayb } const char *data_start = pointer_add(data, sizeof(CompressedAssemblyHeader)); - int ret = LZ4_decompress_safe (data_start, reinterpret_cast(data_buffer), static_cast(assembly_data_size), static_cast(cad.uncompressed_file_size)); + size_t ret = ZSTD_decompress (data_buffer, cad.uncompressed_file_size, data_start, assembly_data_size); - if (ret < 0) { + if (ZSTD_isError (ret)) { Helpers::abort_application ( LOG_ASSEMBLY, std::format ( - "Decompression of assembly {} failed with code {}", + "Decompression of assembly {} failed: {}", optional_string (name), - ret + ZSTD_getErrorName (ret) ) ); } - if (static_cast(ret) != cad.uncompressed_file_size) { + if (ret != cad.uncompressed_file_size) { Helpers::abort_application ( LOG_ASSEMBLY, std::format ( @@ -185,7 +182,7 @@ EmbeddedAssemblies::get_assembly_data (uint8_t *data, uint32_t data_size, [[mayb set_assembly_data_and_size (data_buffer, cad.uncompressed_file_size, assembly_data, assembly_data_size); } else -#endif // def HAVE_LZ4 && def RELEASE +#endif // def RELEASE { set_assembly_data_and_size (data, data_size, assembly_data, assembly_data_size); } diff --git a/src/native/mono/xamarin-app-stub/xamarin-app.hh b/src/native/mono/xamarin-app-stub/xamarin-app.hh index c0e3d3012d6..2a21c1e827b 100644 --- a/src/native/mono/xamarin-app-stub/xamarin-app.hh +++ b/src/native/mono/xamarin-app-stub/xamarin-app.hh @@ -12,7 +12,7 @@ #include static constexpr uint64_t FORMAT_TAG = 0x00035E6972616D58; // 'Xmari^XY' where XY is the format version -static constexpr uint32_t COMPRESSED_DATA_MAGIC = 0x5A4C4158; // 'XALZ', little-endian +static constexpr uint32_t COMPRESSED_DATA_MAGIC = 0x535A4158; // 'XAZS', little-endian static constexpr uint32_t ASSEMBLY_STORE_MAGIC = 0x41424158; // 'XABA', little-endian // The highest bit of assembly store version is a 64-bit ABI flag diff --git a/src/native/native.targets b/src/native/native.targets index 01a2522e38c..80c87e42131 100644 --- a/src/native/native.targets +++ b/src/native/native.targets @@ -36,7 +36,6 @@ <_ConfigureRuntimesInputs Include="CMakeLists.txt" /> <_ConfigureRuntimesInputs Include="common\java-interop\CMakeLists.txt" /> <_ConfigureRuntimesInputs Include="common\libunwind\CMakeLists.txt" /> - <_ConfigureRuntimesInputs Include="common\lz4\CMakeLists.txt" /> <_ConfigureRuntimesInputs Include="common\runtime-base\CMakeLists.txt" /> <_ConfigureRuntimesOutputs Include="@(AndroidSupportedTargetJitAbi->'$(FlavorIntermediateOutputPath)\%(AndroidRID)-Debug\CMakeCache.txt')" /> @@ -217,7 +216,6 @@ <_RuntimeSources Include="common\archive-dso-stub\*.cc" /> <_RuntimeSources Include="common\include\**\*.hh" /> <_RuntimeSources Include="common\runtime-base\*.cc" /> - <_RuntimeSources Include="$(LZ4SourceFullPath)\lib\lz4.c;$(LZ4SourceFullPath)\lib\lz4.h" /> diff --git a/tools/assembly-store-reader-mk2/assembly-store-reader.csproj b/tools/assembly-store-reader-mk2/assembly-store-reader.csproj index ec4c90fecb1..1a760876dfc 100644 --- a/tools/assembly-store-reader-mk2/assembly-store-reader.csproj +++ b/tools/assembly-store-reader-mk2/assembly-store-reader.csproj @@ -17,7 +17,6 @@ - diff --git a/tools/assembly-store-reader/assembly-store-reader.csproj b/tools/assembly-store-reader/assembly-store-reader.csproj index 9b0e604b1da..c2b4c58ff21 100644 --- a/tools/assembly-store-reader/assembly-store-reader.csproj +++ b/tools/assembly-store-reader/assembly-store-reader.csproj @@ -17,7 +17,6 @@ - diff --git a/tools/decompress-assemblies/decompress-assemblies.csproj b/tools/decompress-assemblies/decompress-assemblies.csproj index 1f789958973..cf59128cbc2 100644 --- a/tools/decompress-assemblies/decompress-assemblies.csproj +++ b/tools/decompress-assemblies/decompress-assemblies.csproj @@ -4,7 +4,7 @@ Microsoft Corporation 2021 Microsoft Corporation 0.0.1 - $(DotNetStableTargetFramework) + $(DotNetTargetFramework) false Xamarin.Android.Tools.DecompressAssemblies decompress-assemblies @@ -22,7 +22,6 @@ - diff --git a/tools/decompress-assemblies/main.cs b/tools/decompress-assemblies/main.cs index ba7d9ad4b2c..5efe0837a7e 100644 --- a/tools/decompress-assemblies/main.cs +++ b/tools/decompress-assemblies/main.cs @@ -2,15 +2,15 @@ using System.Buffers; using System.IO; -using K4os.Compression.LZ4; using Xamarin.Tools.Zip; using Xamarin.Android.AssemblyStore; +using ZstandardDecoder = System.IO.Compression.ZstandardDecoder; namespace Xamarin.Android.Tools.DecompressAssemblies { class App { - const uint CompressedDataMagic = 0x5A4C4158; // 'XALZ', little-endian + const uint CompressedDataMagic = 0x535A4158; // 'XAZS', little-endian static readonly ArrayPool bytePool = ArrayPool.Shared; @@ -30,8 +30,8 @@ static bool UncompressDLL (Stream inputStream, string fileName, string filePath, Console.WriteLine ($"Processing {fileName}"); // - // LZ4 compressed assembly header format: - // uint magic; // 0x5A4C4158; 'XALZ', little-endian + // Zstd compressed assembly header format: + // uint magic; // 0x535A4158; 'XAZS', little-endian // uint descriptor_index; // Index into an internal assembly descriptor table // uint uncompressed_length; // Size of assembly, uncompressed // @@ -46,9 +46,12 @@ static bool UncompressDLL (Stream inputStream, string fileName, string filePath, reader.Read (sourceBytes, 0, inputLength); byte[] assemblyBytes = bytePool.Rent ((int)decompressedLength); - int decoded = LZ4Codec.Decode (sourceBytes, 0, inputLength, assemblyBytes, 0, (int)decompressedLength); + int decoded = ZstandardDecoder.TryDecompress ( + sourceBytes.AsSpan (0, inputLength), + assemblyBytes.AsSpan (0, (int)decompressedLength), + out int bytesWritten) ? bytesWritten : -1; if (decoded != (int)decompressedLength) { - Console.Error.WriteLine ($" Failed to decompress LZ4 data of {fileName} (decoded: {decoded})"); + Console.Error.WriteLine ($" Failed to decompress Zstd data of {fileName} (decoded: {decoded})"); retVal = false; } else { string? outputDir = Path.GetDirectoryName (outputFile); diff --git a/tools/tmt/ApkManagedTypeResolver.cs b/tools/tmt/ApkManagedTypeResolver.cs index 5f519e07275..444efcb187d 100644 --- a/tools/tmt/ApkManagedTypeResolver.cs +++ b/tools/tmt/ApkManagedTypeResolver.cs @@ -2,16 +2,16 @@ using System.Collections.Generic; using System.IO; -using K4os.Compression.LZ4; using Mono.Cecil; using Xamarin.Android.AssemblyStore; using Xamarin.Tools.Zip; +using ZstandardDecoder = System.IO.Compression.ZstandardDecoder; namespace tmt { class ApkManagedTypeResolver : ManagedTypeResolver { - const uint CompressedDataMagic = 0x5A4C4158; // 'XALZ', little-endian + const uint CompressedDataMagic = 0x535A4158; // 'XAZS', little-endian readonly Dictionary? individualAssemblies; readonly Dictionary? blobAssemblies; @@ -130,11 +130,12 @@ Stream PrepStream (Stream stream) protected override AssemblyDefinition ReadAssembly (string assemblyPath) { byte[]? assemblyBytes = null; + byte[]? sourceBytes = null; Stream stream = GetAssemblyStream (assemblyPath); // - // LZ4 compressed assembly header format: - // uint magic; // 0x5A4C4158; 'XALZ', little-endian + // Zstd compressed assembly header format: + // uint magic; // 0x535A4158; 'XAZS', little-endian // uint descriptor_index; // Index into an internal assembly descriptor table // uint uncompressed_length; // Size of assembly, uncompressed // @@ -145,18 +146,32 @@ protected override AssemblyDefinition ReadAssembly (string assemblyPath) uint decompressedLength = reader.ReadUInt32 (); int inputLength = (int)(stream.Length - 12); - byte[] sourceBytes = Utilities.BytePool.Rent (inputLength); - reader.Read (sourceBytes, 0, inputLength); - - assemblyBytes = Utilities.BytePool.Rent ((int)decompressedLength); - int decoded = LZ4Codec.Decode (sourceBytes, 0, inputLength, assemblyBytes, 0, (int)decompressedLength); - if (decoded != (int)decompressedLength) { - throw new InvalidOperationException ($"Failed to decompress LZ4 data of {assemblyPath} (decoded: {decoded})"); + try { + sourceBytes = Utilities.BytePool.Rent (inputLength); + ReadFully (reader, sourceBytes, inputLength); + + assemblyBytes = Utilities.BytePool.Rent ((int)decompressedLength); + int decoded = ZstandardDecoder.TryDecompress ( + sourceBytes.AsSpan (0, inputLength), + assemblyBytes.AsSpan (0, (int)decompressedLength), + out int bytesWritten) ? bytesWritten : -1; + if (decoded != (int)decompressedLength) { + throw new InvalidOperationException ($"Failed to decompress Zstd data of {assemblyPath} (decoded: {decoded})"); + } + } catch { + if (assemblyBytes != null) { + Utilities.BytePool.Return (assemblyBytes); + assemblyBytes = null; + } + throw; + } finally { + if (sourceBytes != null) { + Utilities.BytePool.Return (sourceBytes); + sourceBytes = null; + } } - Utilities.BytePool.Return (sourceBytes); } - if (assemblyBytes != null) { stream.Close (); stream.Dispose (); @@ -168,5 +183,17 @@ protected override AssemblyDefinition ReadAssembly (string assemblyPath) return AssemblyDefinition.ReadAssembly (stream); } + + static void ReadFully (BinaryReader reader, byte[] destination, int count) + { + int totalRead = 0; + while (totalRead < count) { + int read = reader.Read (destination, totalRead, count - totalRead); + if (read <= 0) + throw new EndOfStreamException ("Unexpected end of stream while reading compressed assembly."); + + totalRead += read; + } + } } } diff --git a/tools/tmt/tmt.csproj b/tools/tmt/tmt.csproj index bf31f337de7..4c1ba65babd 100644 --- a/tools/tmt/tmt.csproj +++ b/tools/tmt/tmt.csproj @@ -4,7 +4,7 @@ Microsoft Corporation 2020 Microsoft Corporation 0.0.1 - $(DotNetStableTargetFramework) + $(DotNetTargetFramework) false ../../bin/$(Configuration)/bin/typemap-tool Exe @@ -25,7 +25,6 @@ -