From f8d7c46788bef62181fc5dc29d6298dc2f91ed25 Mon Sep 17 00:00:00 2001 From: Simon Rozsival Date: Wed, 19 Aug 2026 18:03:23 +0200 Subject: [PATCH 1/5] [build] Remove LibZipSharp Move BuildArchive to the .NET 12 task assembly and replace remaining LibZipSharp usages with System.IO.Compression. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: 4a826a98-ebc1-4ff9-9fa8-4a71854812f4 --- .../scripts/extract-android-assemblies.cs | 14 +- .../src/AssemblyStore/AssemblyStore.csproj | 1 - .../AssemblyStore/AssemblyStoreExplorer.cs | 12 +- .../src/AssemblyStore/Utils.cs | 34 +- .../AssemblyStore/V1/AssemblyStoreExplorer.cs | 10 +- .github/skills/update-tpn/SKILL.md | 8 - Directory.Build.props | 1 - THIRD-PARTY-NOTICES.TXT | 75 +--- ...amarin.Android.Tools.BootstrapTasks.csproj | 1 - .../CheckApiCompatibility.cs | 10 +- .../UnzipDirectoryChildren.cs | 21 +- .../Zip.cs | 17 +- build-tools/create-packs/SignList.xml | 1 - build-tools/debian-metadata/rules | 2 - .../installers/create-installers.targets | 10 - .../PackagingUtils.cs | 2 +- .../Files.cs | 50 ++- .../HexUtilities.cs | 2 +- .../MSBuildReferences.projitems | 2 - .../Microsoft.Android.Build.BaseTasks.csproj | 1 + .../ZipArchiveExtensions.cs | 247 +++++++++++++ .../ZipArchiveMetadataReader.cs | 157 ++++++++ .../BuildArchive.cs | 347 ++++++++++++++++++ .../Microsoft.Android.Build.Tasks.csproj | 4 +- .../Tasks/GetGdbSymbols.cs | 6 +- ...marin.Android.Build.Debugging.Tasks.csproj | 1 - .../Tasks/Aapt2LinkAssetPack.cs | 21 +- .../Tasks/BuildArchive.cs | 305 --------------- .../Tasks/CollectJarContentFilesForArchive.cs | 8 +- .../Tasks/CreateAar.cs | 38 +- .../Tasks/ExtractJarsFromAar.cs | 8 +- .../Tasks/GenerateResourceCaseMap.cs | 1 - .../Tasks/Javac.cs | 10 +- .../Tasks/ResolveLibraryProjectImports.cs | 20 +- .../Tasks/UnzipToFolder.cs | 8 +- .../AndroidUpdateResourcesTest.cs | 8 +- .../BindingBuildTest.cs | 12 +- .../Xamarin.Android.Build.Tests/BuildTest.cs | 4 +- .../Xamarin.Android.Build.Tests/BuildTest2.cs | 4 +- .../GlobalUsings.cs | 1 + .../ManifestTest.TestCaseSource.cs | 1 - .../ManifestTest.cs | 10 +- .../PackagingTest.cs | 83 +---- .../Tasks/Aapt2Tests.cs | 4 +- .../Tasks/BuildArchiveTests.cs | 94 ++++- .../Tasks/FilterAssembliesTests.cs | 8 +- .../Utilities/ArchiveAssemblyHelper.cs | 16 +- .../Utilities/AssertionExtensions.cs | 19 +- .../Xamarin.Android.Build.Tests/XASdkTests.cs | 4 +- .../Xamarin.Android.Build.Tests.csproj | 4 +- .../ZipArchiveExTests.cs | 219 ----------- .../Common/BuildOutput.cs | 18 +- .../Utilities/ZipHelper.cs | 41 ++- .../Xamarin.ProjectTools.csproj | 1 - .../Utilities/MonoAndroidHelper.cs | 6 +- .../Utilities/UtilityExtensions.cs | 39 -- .../Utilities/ZipArchiveDotNet.cs | 263 ------------- .../Utilities/ZipArchiveEx.cs | 309 ---------------- .../Xamarin.Android.Build.Tasks.csproj | 1 - .../Xamarin.Android.Build.Tasks.targets | 6 - .../Xamarin.Android.Common.targets | 13 +- ...marin.Android.Tools.JavadocImporter.csproj | 2 - .../samples.cs | 66 ++-- src/androidsdk/androidsdk.targets | 8 +- .../MSBuildDeviceIntegration/GlobalUsings.cs | 1 + .../MSBuildDeviceIntegration.csproj | 2 +- .../Tests/BundleToolNoAbiSplitTests.cs | 1 - .../Tests/BundleToolTests.cs | 20 +- .../FilesTests.cs | 139 ++++--- .../ZipArchiveExtensionsTests.cs | 50 +++ tools/tmt/ApkManagedTypeResolver.cs | 16 +- tools/tmt/Loader.cs | 9 +- tools/tmt/tmt.csproj | 9 +- 73 files changed, 1329 insertions(+), 1637 deletions(-) create mode 100644 src/Microsoft.Android.Build.BaseTasks/ZipArchiveExtensions.cs create mode 100644 src/Microsoft.Android.Build.BaseTasks/ZipArchiveMetadataReader.cs create mode 100644 src/Microsoft.Android.Build.Tasks/BuildArchive.cs delete mode 100644 src/Xamarin.Android.Build.Tasks/Tasks/BuildArchive.cs create mode 100644 src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/GlobalUsings.cs delete mode 100644 src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/ZipArchiveExTests.cs delete mode 100644 src/Xamarin.Android.Build.Tasks/Utilities/ZipArchiveDotNet.cs delete mode 100644 src/Xamarin.Android.Build.Tasks/Utilities/ZipArchiveEx.cs create mode 100644 tests/MSBuildDeviceIntegration/GlobalUsings.cs create mode 100644 tests/Microsoft.Android.Build.BaseTasks-Tests/ZipArchiveExtensionsTests.cs diff --git a/.github/skills/extract-android-assemblies/scripts/extract-android-assemblies.cs b/.github/skills/extract-android-assemblies/scripts/extract-android-assemblies.cs index 7d62036449a..d5f0944ce1d 100644 --- a/.github/skills/extract-android-assemblies/scripts/extract-android-assemblies.cs +++ b/.github/skills/extract-android-assemblies/scripts/extract-android-assemblies.cs @@ -5,10 +5,10 @@ using System; using System.Collections.Generic; using System.IO; +using System.IO.Compression; using Xamarin.Android.AssemblyStore; using Xamarin.Android.Tools; -using Xamarin.Tools.Zip; namespace Xamarin.Android.Tools.DecompressAssemblies { @@ -79,7 +79,7 @@ static bool ExtractIndividualEntries (ZipArchive apk, string filePath, string as { bool retVal = true; int assemblyCount = 0; - foreach (ZipEntry entry in apk) { + foreach (var entry in apk.Entries) { if (!TryGetAssemblyOutputPath (entry.FullName, assembliesPath, nativeLibrariesPath, out string assemblyName)) { continue; } @@ -87,7 +87,7 @@ static bool ExtractIndividualEntries (ZipArchive apk, string filePath, string as assemblyCount++; using (var stream = new MemoryStream ()) { - entry.Extract (stream); + Utils.Extract (entry, stream); stream.Seek (0, SeekOrigin.Begin); string outputFile = GetSafeOutputFile (outputDirectory, assemblyName); using var payload = new MemoryStream (); @@ -166,15 +166,15 @@ static string GetAndroidAbi (AndroidTargetArch arch) static bool HasAssemblyStore (ZipArchive apk, string assembliesPath, string nativeLibrariesPath) { - if (apk.ContainsEntry ($"{assembliesPath}assemblies.blob")) { + if (Utils.ContainsEntry (apk, $"{assembliesPath}assemblies.blob", caseSensitive: true)) { return true; } foreach (AndroidTargetArch arch in targetArchitectures) { string abi = GetAndroidAbi (arch); if ( - apk.ContainsEntry ($"{nativeLibrariesPath}{abi}/libassembly-store.so") || - apk.ContainsEntry ($"{nativeLibrariesPath}{abi}/libassemblies.{abi}.blob.so") + Utils.ContainsEntry (apk, $"{nativeLibrariesPath}{abi}/libassembly-store.so", caseSensitive: true) || + Utils.ContainsEntry (apk, $"{nativeLibrariesPath}{abi}/libassemblies.{abi}.blob.so", caseSensitive: true) ) { return true; } @@ -185,7 +185,7 @@ static bool HasAssemblyStore (ZipArchive apk, string assembliesPath, string nati static bool ExtractFromArchive (string filePath, string assembliesPath, string nativeLibrariesPath, string outputDirectory) { - using (ZipArchive apk = ZipArchive.Open (filePath, FileMode.Open)) { + using (ZipArchive apk = Utils.OpenZip (filePath)) { if (HasAssemblyStore (apk, assembliesPath, nativeLibrariesPath)) { return ExtractAssemblyStores (filePath, outputDirectory); } diff --git a/.github/skills/read-assembly-store/src/AssemblyStore/AssemblyStore.csproj b/.github/skills/read-assembly-store/src/AssemblyStore/AssemblyStore.csproj index d9863dffdbd..eb095fbaf9b 100644 --- a/.github/skills/read-assembly-store/src/AssemblyStore/AssemblyStore.csproj +++ b/.github/skills/read-assembly-store/src/AssemblyStore/AssemblyStore.csproj @@ -12,7 +12,6 @@ - diff --git a/.github/skills/read-assembly-store/src/AssemblyStore/AssemblyStoreExplorer.cs b/.github/skills/read-assembly-store/src/AssemblyStore/AssemblyStoreExplorer.cs index 1d6cc65c7ec..45c69adfbb9 100644 --- a/.github/skills/read-assembly-store/src/AssemblyStore/AssemblyStoreExplorer.cs +++ b/.github/skills/read-assembly-store/src/AssemblyStore/AssemblyStoreExplorer.cs @@ -1,9 +1,9 @@ using System; using System.Collections.Generic; using System.IO; +using System.IO.Compression; using Xamarin.Android.Tools; -using Xamarin.Tools.Zip; namespace Xamarin.Android.AssemblyStore; @@ -127,7 +127,7 @@ public static (IList? explorers, string? errorMessage) Op static (IList? explorers, string? errorMessage) OpenArchive (FileInfo fi, IList paths) { string? errorMessage; - using (var zip = ZipArchive.Open (fi.FullName, FileMode.Open)) { + using (var zip = Utils.OpenZip (fi.FullName)) { (IList? explorers, string? loadError, bool pathsFound) = TryLoad (fi, zip, paths); if (pathsFound) { return (explorers, loadError); @@ -172,13 +172,15 @@ static bool IsV1Store (FileInfo info) var ret = new List (); foreach (string path in paths) { - if (!zip.ContainsEntry (path)) { + if (!Utils.ContainsEntry (zip, path, caseSensitive: true)) { continue; } - ZipEntry entry = zip.ReadEntry (path); + var entry = Utils.ReadEntry (zip, path, caseSensitive: true); + if (entry == null) + continue; var stream = new MemoryStream (); - entry.Extract (stream); + Utils.Extract (entry, stream); ret.Add (new AssemblyStoreExplorer (stream, $"{fi.FullName}!{path}")); } diff --git a/.github/skills/read-assembly-store/src/AssemblyStore/Utils.cs b/.github/skills/read-assembly-store/src/AssemblyStore/Utils.cs index 3b502ebaf62..f8decd14e3f 100644 --- a/.github/skills/read-assembly-store/src/AssemblyStore/Utils.cs +++ b/.github/skills/read-assembly-store/src/AssemblyStore/Utils.cs @@ -1,10 +1,11 @@ using System; using System.IO; using System.Buffers; +using System.IO.Compression; +using System.Linq; using ELFSharp.ELF; using ELFSharp.ELF.Sections; -using Xamarin.Tools.Zip; namespace Xamarin.Android.AssemblyStore; @@ -224,7 +225,7 @@ public static (FileFormat format, FileInfo? info) DetectFileFormat (string path) static FileFormat DetectAndroidArchive (FileInfo info, FileFormat defaultFormat) { - using var zip = ZipArchive.Open (info.FullName, FileMode.Open); + using var zip = OpenZip (info.FullName); if (HasAllEntries (zip, aabZipEntries)) { return FileFormat.Aab; @@ -244,11 +245,38 @@ static FileFormat DetectAndroidArchive (FileInfo info, FileFormat defaultFormat) static bool HasAllEntries (ZipArchive zip, string[] entries) { foreach (string entry in entries) { - if (!zip.ContainsEntry (entry, caseSensitive: true)) { + if (!ContainsEntry (zip, entry, caseSensitive: true)) { return false; } } return true; } + + public static ZipArchive OpenZip (string path) + { + return ZipFile.OpenRead (path); + } + + public static ZipArchive OpenZip (Stream stream, bool leaveOpen = false) + { + return new ZipArchive (stream, ZipArchiveMode.Read, leaveOpen); + } + + public static bool ContainsEntry (ZipArchive archive, string entryName, bool caseSensitive = false) + { + return ReadEntry (archive, entryName, caseSensitive) != null; + } + + public static ZipArchiveEntry? ReadEntry (ZipArchive archive, string entryName, bool caseSensitive = false) + { + var comparison = caseSensitive ? StringComparison.Ordinal : StringComparison.OrdinalIgnoreCase; + return archive.Entries.FirstOrDefault (entry => string.Equals (entry.FullName, entryName, comparison)); + } + + public static void Extract (ZipArchiveEntry entry, Stream destination) + { + using var stream = entry.Open (); + stream.CopyTo (destination); + } } diff --git a/.github/skills/read-assembly-store/src/AssemblyStore/V1/AssemblyStoreExplorer.cs b/.github/skills/read-assembly-store/src/AssemblyStore/V1/AssemblyStoreExplorer.cs index 85ebcd22264..94e9cd854f8 100644 --- a/.github/skills/read-assembly-store/src/AssemblyStore/V1/AssemblyStoreExplorer.cs +++ b/.github/skills/read-assembly-store/src/AssemblyStore/V1/AssemblyStoreExplorer.cs @@ -1,8 +1,8 @@ using System; using System.Collections.Generic; using System.IO; - -using Xamarin.Tools.Zip; +using System.IO.Compression; +using Xamarin.Android.AssemblyStore; namespace Xamarin.Android.AssemblyStore.V1 { @@ -215,20 +215,20 @@ void ReadStoreSetFromArchive (string baseName, string archivePath, string extens } basePathInArchive = $"{basePathInArchive}/{baseName}."; - using (ZipArchive archive = ZipArchive.Open (archivePath, FileMode.Open)) { + using (ZipArchive archive = Utils.OpenZip (archivePath)) { ReadStoreSetFromArchive (archive, basePathInArchive); } } void ReadStoreSetFromArchive (ZipArchive archive, string basePathInArchive) { - foreach (ZipEntry entry in archive) { + foreach (var entry in archive.Entries) { if (!entry.FullName.StartsWith (basePathInArchive, StringComparison.Ordinal)) { continue; } using (var stream = new MemoryStream ()) { - entry.Extract (stream); + Utils.Extract (entry, stream); if (entry.FullName.EndsWith (".blob", StringComparison.Ordinal)) { AddStore (new AssemblyStoreReader (stream, GetStoreArch (entry.FullName), keepStoreInMemory)); diff --git a/.github/skills/update-tpn/SKILL.md b/.github/skills/update-tpn/SKILL.md index e87dde58734..430d32bb47e 100644 --- a/.github/skills/update-tpn/SKILL.md +++ b/.github/skills/update-tpn/SKILL.md @@ -82,7 +82,6 @@ Search `.csproj` files for `` elements. Current third-party Nu |---------|------------|-------------| | ELFSharp | KonradKuczynski/ELFSharp | https://elfsharp.it/ (MIT + LLVM) | | K4os.Compression.LZ4 | MiloszKrajewski/K4os.Compression.LZ4 | https://github.com/MiloszKrajewski/K4os.Compression.LZ4/ (MIT) | -| Xamarin.LibZipSharp | xamarin/LibZipSharp | https://github.com/xamarin/LibZipSharp/ (MIT) | | Irony | IronyProject/Irony | https://github.com/IronyProject/Irony (MIT) | | Newtonsoft.Json | JamesNK/Newtonsoft.Json | https://github.com/JamesNK/Newtonsoft.Json (MIT) | | NuGet.ProjectModel | NuGet/NuGet.Client | https://github.com/NuGet/NuGet.Client (Apache 2.0) | @@ -111,13 +110,6 @@ These are downloaded and shipped with the SDK: | r8 | google/r8 | https://r8.googlesource.com/r8/ (BSD-3-Clause) | | binutils | gnu/binutils | https://sourceware.org/git/?p=binutils-gdb.git;a=tree;hb=HEAD (GPLv3) | -#### libzip (via LibZipSharp NuGet) -LibZipSharp bundles libzip internally: - -| Source | Name in TPN | License Location | -|--------|------------|-----------------| -| libzip (in LibZipSharp NuGet) | nih-at/libzip | LibZipSharp NuGet `Licences/libzip/LICENSE` or https://github.com/nih-at/libzip/ (BSD-3-Clause) | - ### Step 2 — Cross-reference Compare the inventory against the current entries in `THIRD-PARTY-NOTICES.TXT`: diff --git a/Directory.Build.props b/Directory.Build.props index baf6a832968..60beac9242b 100644 --- a/Directory.Build.props +++ b/Directory.Build.props @@ -39,7 +39,6 @@ - 3.3.0 1.0.0 13.0.3 5.4.0 diff --git a/THIRD-PARTY-NOTICES.TXT b/THIRD-PARTY-NOTICES.TXT index 5759a3f7d26..533bcb53f48 100644 --- a/THIRD-PARTY-NOTICES.TXT +++ b/THIRD-PARTY-NOTICES.TXT @@ -30,10 +30,8 @@ implication, estoppel or otherwise. 19. MiloszKrajewski/K4os.Compression.LZ4 (https://github.com/MiloszKrajewski/K4os.Compression.LZ4/) 20. mono/cecil (https://github.com/mono/cecil/) 21. mono/linker (https://github.com/mono/linker/) -22. nih-at/libzip (https://github.com/nih-at/libzip/) -23. NuGet/NuGet.Client (https://github.com/NuGet/NuGet.Client) -24. tessil/robin-map (https://github.com/Tessil/robin-map) -25. xamarin/LibZipSharp (https://github.com/xamarin/LibZipSharp/) +22. NuGet/NuGet.Client (https://github.com/NuGet/NuGet.Client) +23. tessil/robin-map (https://github.com/Tessil/robin-map) %% Android API documentation NOTICES AND INFORMATION BEGIN HERE ================================================================ @@ -2401,45 +2399,6 @@ SOFTWARE. END OF mono/linker NOTICES AND INFORMATION -%% nih-at/libzip NOTICES AND INFORMATION BEGIN HERE -=================================================== -Copyright (C) 1999-2020 Dieter Baron and Thomas Klausner - -The authors can be contacted at - -Redistribution and use in source and binary forms, with or without -modification, are permitted provided that the following conditions -are met: - -1. Redistributions of source code must retain the above copyright - notice, this list of conditions and the following disclaimer. - -2. Redistributions in binary form must reproduce the above copyright - notice, this list of conditions and the following disclaimer in - the documentation and/or other materials provided with the - distribution. - -3. The names of the authors may not be used to endorse or promote - products derived from this software without specific prior - written permission. - -THIS SOFTWARE IS PROVIDED BY THE AUTHORS ``AS IS'' AND ANY EXPRESS -OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED -WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE -ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY -DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL -DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE -GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS -INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER -IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR -OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN -IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - - -=================================================== -END OF nih-at/libzip NOTICES AND INFORMATION - - %% NuGet/NuGet.Client NOTICES AND INFORMATION BEGIN HERE ======================================================== Copyright (c) .NET Foundation and Contributors. @@ -2491,33 +2450,3 @@ SOFTWARE. ====================================================== END OF tessil/robin-map NOTICES AND INFORMATION - -%% xamarin/LibZipSharp NOTICES AND INFORMATION BEGIN HERE -========================================================= -The MIT License (MIT) - -Copyright (c) 2016 Marek Habersack - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all -copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -SOFTWARE. - - -========================================================= -END OF xamarin/LibZipSharp NOTICES AND INFORMATION - - diff --git a/build-tools/Xamarin.Android.Tools.BootstrapTasks/Xamarin.Android.Tools.BootstrapTasks.csproj b/build-tools/Xamarin.Android.Tools.BootstrapTasks/Xamarin.Android.Tools.BootstrapTasks.csproj index b84f0c3d1de..bce2837a4af 100644 --- a/build-tools/Xamarin.Android.Tools.BootstrapTasks/Xamarin.Android.Tools.BootstrapTasks.csproj +++ b/build-tools/Xamarin.Android.Tools.BootstrapTasks/Xamarin.Android.Tools.BootstrapTasks.csproj @@ -4,7 +4,6 @@ $(DotNetStableTargetFramework) - true $(BootstrapOutputDirectory) diff --git a/build-tools/Xamarin.Android.Tools.BootstrapTasks/Xamarin.Android.Tools.BootstrapTasks/CheckApiCompatibility.cs b/build-tools/Xamarin.Android.Tools.BootstrapTasks/Xamarin.Android.Tools.BootstrapTasks/CheckApiCompatibility.cs index 44e726086be..720acd7fccc 100644 --- a/build-tools/Xamarin.Android.Tools.BootstrapTasks/Xamarin.Android.Tools.BootstrapTasks/CheckApiCompatibility.cs +++ b/build-tools/Xamarin.Android.Tools.BootstrapTasks/Xamarin.Android.Tools.BootstrapTasks/CheckApiCompatibility.cs @@ -2,10 +2,11 @@ using System.Collections.Generic; using System.Diagnostics; using System.IO; +using System.IO.Compression; using System.Linq; using Microsoft.Build.Framework; using Microsoft.Build.Utilities; -using Xamarin.Tools.Zip; +using Microsoft.Android.Build.Tasks; namespace Xamarin.Android.Tools.BootstrapTasks { @@ -133,11 +134,12 @@ public override bool Execute () var zipFiles = Directory.GetFiles (referenceContractPath.Parent.FullName, "*.zip"); foreach (var zipFile in zipFiles) { var zipDateTime = File.GetLastWriteTimeUtc (zipFile); - using (var zip = ZipArchive.Open (zipFile, FileMode.Open)) { - foreach (var entry in zip) { - var path = Path.Combine (referenceContractPath.FullName, entry.NativeFullName); + using (var zip = ZipArchiveExtensions.OpenZip (zipFile, FileMode.Open)) { + foreach (var entry in zip.Entries) { + var path = Path.Combine (referenceContractPath.FullName, entry.FullName.Replace ('/', Path.DirectorySeparatorChar)); if (!File.Exists (path) || File.GetLastWriteTimeUtc (path) < zipDateTime) { Log.LogMessage ($"Extracting: {path}"); + Directory.CreateDirectory (Path.GetDirectoryName (path)); using (var fileStream = File.Create (path)) { entry.Extract (fileStream); } diff --git a/build-tools/Xamarin.Android.Tools.BootstrapTasks/Xamarin.Android.Tools.BootstrapTasks/UnzipDirectoryChildren.cs b/build-tools/Xamarin.Android.Tools.BootstrapTasks/Xamarin.Android.Tools.BootstrapTasks/UnzipDirectoryChildren.cs index ad0d915b7be..7de5833217a 100644 --- a/build-tools/Xamarin.Android.Tools.BootstrapTasks/Xamarin.Android.Tools.BootstrapTasks/UnzipDirectoryChildren.cs +++ b/build-tools/Xamarin.Android.Tools.BootstrapTasks/Xamarin.Android.Tools.BootstrapTasks/UnzipDirectoryChildren.cs @@ -1,8 +1,9 @@ using Microsoft.Build.Framework; using System.IO; +using System.IO.Compression; using System.Text; using System.Collections.Generic; -using Xamarin.Tools.Zip; +using Microsoft.Android.Build.Tasks; using MTask = Microsoft.Build.Utilities.Task; using TTask = System.Threading.Tasks.Task; @@ -70,17 +71,22 @@ void ExtractFile (string sourceFile, string relativeDestDir, string destinationF { relativeDestDir = relativeDestDir?.Replace ('\\', Path.DirectorySeparatorChar); - using (var zip = ZipArchive.Open (sourceFile, FileMode.Open)) { - foreach (var entry in zip) { - if (!entry.IsDirectory) { + using (var zip = ZipArchiveExtensions.OpenZip (sourceFile, FileMode.Open, encoding)) { + foreach (var entry in zip.Entries) { + if (!entry.IsDirectory ()) { if (filesToExtract.Count > 0 && !filesToExtract.Contains (Path.GetFileName (entry.FullName))) continue; - var entryPath = entry.NativeFullName; + var entryPath = entry.FullName.Replace ('/', Path.DirectorySeparatorChar); if (!NoSubdirectory) { - entryPath = entryPath.Substring (entryPath.IndexOf (Path.DirectorySeparatorChar) + 1); + int separatorIndex = entryPath.IndexOf (Path.DirectorySeparatorChar); + if (separatorIndex < 0) + continue; + entryPath = entryPath.Substring (separatorIndex + 1); + if (entryPath.Length == 0) + continue; } var destinationPath = Path.Combine (destinationFolder, relativeDestDir, entryPath); - Log.LogMessage (MessageImportance.Low, $"Extracting {entry.NativeFullName} to {destinationPath}"); + Log.LogMessage (MessageImportance.Low, $"Extracting {entry.FullName} to {destinationPath}"); entry.Extract (Path.GetDirectoryName (destinationPath), Path.GetFileName (destinationPath)); } } @@ -88,4 +94,3 @@ void ExtractFile (string sourceFile, string relativeDestDir, string destinationF } } } - diff --git a/build-tools/Xamarin.Android.Tools.BootstrapTasks/Xamarin.Android.Tools.BootstrapTasks/Zip.cs b/build-tools/Xamarin.Android.Tools.BootstrapTasks/Xamarin.Android.Tools.BootstrapTasks/Zip.cs index a2c1cbdca69..1c4a0a9c9f4 100644 --- a/build-tools/Xamarin.Android.Tools.BootstrapTasks/Xamarin.Android.Tools.BootstrapTasks/Zip.cs +++ b/build-tools/Xamarin.Android.Tools.BootstrapTasks/Xamarin.Android.Tools.BootstrapTasks/Zip.cs @@ -5,8 +5,7 @@ using Microsoft.Build.Framework; using Microsoft.Build.Utilities; - -using Xamarin.Tools.Zip; +using Microsoft.Android.Build.Tasks; using IOFile = System.IO.File; @@ -43,7 +42,7 @@ public override bool Execute () prefix += Path.DirectorySeparatorChar; } - using (var zip = ZipArchive.Open (File.ItemSpec, FileMode.OpenOrCreate)) { + using (var zip = ZipArchiveExtensions.OpenZip (File.ItemSpec, FileMode.OpenOrCreate)) { if (Entries == null) return !Log.HasLoggedErrors; foreach (var entry in Entries) { @@ -57,17 +56,13 @@ public override bool Execute () if (prefix != null && entryDir.StartsWith (prefix, StringComparison.OrdinalIgnoreCase)) { zipDir = entryDir.Substring (prefix.Length); } - if (string.IsNullOrEmpty (zipDir)) { - // JonP can't figure out how to actually clear the archive directory name - // using AddFileToDirectory(). This works as desired. - zip.AddFile (entryPath, Path.GetFileName (entryPath)); - } else { - zip.AddFileToDirectory (entryPath, zipDir, useFileDirectory: false); - } + var archivePath = string.IsNullOrEmpty (zipDir) + ? Path.GetFileName (entryPath) + : Path.Combine (zipDir, Path.GetFileName (entryPath)); + zip.AddFile (entryPath, archivePath.Replace ('\\', '/')); } } return !Log.HasLoggedErrors; } } } - diff --git a/build-tools/create-packs/SignList.xml b/build-tools/create-packs/SignList.xml index a71ea2c7fcf..352da582aa5 100644 --- a/build-tools/create-packs/SignList.xml +++ b/build-tools/create-packs/SignList.xml @@ -1,7 +1,6 @@ - diff --git a/build-tools/debian-metadata/rules b/build-tools/debian-metadata/rules index 0db5a1752d6..0a73bf108b2 100755 --- a/build-tools/debian-metadata/rules +++ b/build-tools/debian-metadata/rules @@ -21,7 +21,6 @@ override_dh_install: rm -f bin/*/lib/xamarin.android/xbuild/Xamarin/Android/opt.exe rm -f bin/*/lib/xamarin.android/xbuild/Xamarin/Android/aapt2.exe rm -f bin/*/lib/xamarin.android/xbuild/Xamarin/Android/libwinpthread-1.dll - rm -f bin/*/lib/xamarin.android/xbuild/Xamarin/Android/libZipSharpNative-*.dll rm -f bin/*/lib/xamarin.android/xbuild/Xamarin/Android/runtimes/*/libMono.Unix.so dh_install @@ -41,7 +40,6 @@ override_dh_clideps: --exclude-moduleref=libfam.so.0 \ --exclude-moduleref=libgamin-1.so.0 \ --exclude-moduleref=libmono-btls-shared \ - --exclude-moduleref=libZipSharpNative \ --exclude-moduleref=lzo.dll \ --exclude-moduleref=Microsoft.VisualStudio.Setup.Configuration.Native.dll \ --exclude-moduleref=mscoree.dll \ diff --git a/build-tools/installers/create-installers.targets b/build-tools/installers/create-installers.targets index b19283e27ce..df517785566 100644 --- a/build-tools/installers/create-installers.targets +++ b/build-tools/installers/create-installers.targets @@ -96,9 +96,6 @@ <_MSBuildFiles Include="$(MicrosoftAndroidSdkOutDir)java-interop.jar" /> <_MSBuildFiles Include="$(MicrosoftAndroidSdkOutDir)java-source-utils.jar" /> <_MSBuildFiles Include="$(MicrosoftAndroidSdkOutDir)LayoutBinding.cs" /> - <_MSBuildFiles Include="$(MicrosoftAndroidSdkOutDir)libZipSharp.dll" /> - <_MSBuildFiles Include="@(_LocalizationLanguages->'$(MicrosoftAndroidSdkOutDir)%(Identity)\libZipSharp.resources.dll')" /> - <_MSBuildFiles Include="$(MicrosoftAndroidSdkOutDir)libZipSharp.pdb" /> <_MSBuildFiles Include="$(MicrosoftAndroidSdkOutDir)Mono.Unix.dll" /> <_MSBuildFiles Include="$(MicrosoftAndroidSdkOutDir)Mono.Unix.pdb" /> <_MSBuildFiles Include="$(MicrosoftAndroidSdkOutDir)Microsoft.Android.Build.BaseTasks.dll" /> @@ -182,12 +179,6 @@ <_MSBuildTargetsSrcFiles Include="$(MSBuildTargetsSrcDir)\Xamarin.Android.AvailableItems.targets" /> - <_MSBuildFilesWin Include="$(MicrosoftAndroidSdkOutDir)x86\libZipSharpNative-*.dll" /> - <_MSBuildFilesWin Include="$(MicrosoftAndroidSdkOutDir)x86\libZipSharpNative-*.pdb" /> - <_MSBuildFilesWin Include="$(MicrosoftAndroidSdkOutDir)x64\libZipSharpNative-*.dll" /> - <_MSBuildFilesWin Include="$(MicrosoftAndroidSdkOutDir)x64\libZipSharpNative-*.pdb" /> - <_MSBuildFilesWin Include="$(MicrosoftAndroidSdkOutDir)arm64\libZipSharpNative-*.dll" /> - <_MSBuildFilesWin Include="$(MicrosoftAndroidSdkOutDir)arm64\libZipSharpNative-*.pdb" /> <_MSBuildFilesWin Include="$(MicrosoftAndroidSdkOutDir)aapt2.exe" /> <_MSBuildFilesWin Include="$(MicrosoftAndroidSdkOutDir)binutils\bin\as.exe" /> <_MSBuildFilesWin Include="$(MicrosoftAndroidSdkOutDir)binutils\bin\ld.exe" /> @@ -228,7 +219,6 @@ <_MSBuildFilesUnix Include="$(MicrosoftAndroidSdkOutDir)$(HostOS)\jit-times" Permission="755" /> <_MSBuildFilesUnix Include="$(MicrosoftAndroidSdkOutDir)$(HostOS)\mono.config" /> <_MSBuildFilesUnixSignAndHarden Include="$(MicrosoftAndroidSdkOutDir)$(HostOS)\aapt2" /> - <_MSBuildFilesUnixSign Include="$(MicrosoftAndroidSdkOutDir)libZipSharpNative-*.$(LibExtension)" /> <_MSBuildFilesUnixSign Include="$(MicrosoftAndroidSdkOutDir)libMono.Unix.$(LibExtension)" /> diff --git a/src-ThirdParty/android-platform-tools-base/PackagingUtils.cs b/src-ThirdParty/android-platform-tools-base/PackagingUtils.cs index ee48570cc02..1f4efa60212 100644 --- a/src-ThirdParty/android-platform-tools-base/PackagingUtils.cs +++ b/src-ThirdParty/android-platform-tools-base/PackagingUtils.cs @@ -33,7 +33,7 @@ internal class PackagingUtils /// /// Checks if a zip entry is valid for packaging into the .apk as standard Java resource. /// - /// the name of the zip entry from Xamarin.Tools.Zip.ZipEntry.FullName. + /// the full name of the zip entry. /// true if the entry is valid for packaging. public static bool CheckEntryForPackaging (string entryName) { diff --git a/src/Microsoft.Android.Build.BaseTasks/Files.cs b/src/Microsoft.Android.Build.BaseTasks/Files.cs index 514448df475..50a1dafe42e 100644 --- a/src/Microsoft.Android.Build.BaseTasks/Files.cs +++ b/src/Microsoft.Android.Build.BaseTasks/Files.cs @@ -5,11 +5,12 @@ using System.Collections.Generic; using System.Globalization; using System.IO; +using System.IO.Compression; +using System.IO.Hashing; using System.Linq; using System.Runtime.CompilerServices; using System.Security.Cryptography; using System.Text; -using Xamarin.Tools.Zip; using Microsoft.Build.Utilities; using System.Threading; using System.Runtime.InteropServices; @@ -401,23 +402,27 @@ public static bool HasBytesChanged (byte [] bytes, string destination) static string? HashZip (Stream stream) { - string hashes = String.Empty; + var hashes = new StringBuilder (); + var buffer = MemoryStreamPool.Shared.Rent (); try { - using (var zip = ZipArchive.Open (stream)) { - foreach (var item in zip) { - hashes += String.Format (CultureInfo.InvariantCulture, "{0}{1}", item.FullName, item.CRC); + using (var zip = ZipArchiveExtensions.OpenZip (stream, ZipArchiveMode.Read, leaveOpen: true)) { + foreach (var item in zip.Entries) { + hashes.AppendFormat (CultureInfo.InvariantCulture, "{0}{1}", item.FullName, GetEntryCrc32 (item, buffer)); } } } catch { return null; + } finally { + MemoryStreamPool.Shared.Return (buffer); } - return hashes; + return hashes.ToString (); } static string? HashZip (string filename) { - string hashes = String.Empty; + var hashes = new StringBuilder (); + var buffer = MemoryStreamPool.Shared.Rent (); try { // check cache @@ -425,25 +430,38 @@ public static bool HasBytesChanged (byte [] bytes, string destination) return File.ReadAllText (filename + ".hash"); using (var zip = ReadZipFile (filename)) { - foreach (var item in zip) { - hashes += String.Format (CultureInfo.InvariantCulture, "{0}{1}", item.FullName, item.CRC); + foreach (var item in zip.Entries) { + hashes.AppendFormat (CultureInfo.InvariantCulture, "{0}{1}", item.FullName, GetEntryCrc32 (item, buffer)); } } } catch { return null; + } finally { + MemoryStreamPool.Shared.Return (buffer); } - return hashes; + return hashes.ToString (); + } + + static uint GetEntryCrc32 (ZipArchiveEntry entry, MemoryStream buffer) + { + buffer.SetLength (0); + entry.Extract (buffer); + if (buffer.TryGetBuffer (out ArraySegment segment) && segment.Array != null) { + return Crc32.HashToUInt32 (new ReadOnlySpan (segment.Array, segment.Offset, (int) buffer.Length)); + } + + return Crc32.HashToUInt32 (buffer.ToArray ()); } public static ZipArchive ReadZipFile (string filename, bool strictConsistencyChecks = false) { - return ZipArchive.Open (filename, FileMode.Open, strictConsistencyChecks: strictConsistencyChecks); + return ZipArchiveExtensions.OpenZip (filename, FileMode.Open); } - public static bool ZipAny (string filename, Func filter) + public static bool ZipAny (string filename, Func filter) { using (var zip = ReadZipFile (filename)) { - return zip.Any (filter); + return zip.Entries.Any (filter); } } @@ -458,15 +476,15 @@ public static bool ExtractAll (ZipArchive zip, string destination, Action? deleteCallback = null, Func? skipCallback = null, TaskLoggingHelper? log = null) { int i = 0; - int total = (int)zip.EntryCount; + int total = zip.Entries.Count; bool updated = false; var files = new HashSet (); var memoryStream = MemoryStreamPool.Shared.Rent (); var fullDestination = Path.GetFullPath (destination + Path.DirectorySeparatorChar); try { - foreach (var entry in zip) { + foreach (var entry in zip.Entries) { progressCallback?.Invoke (i++, total); - if (entry.IsDirectory) + if (entry.IsDirectory ()) continue; if (entry.FullName.Contains ("/__MACOSX/") || entry.FullName.EndsWith ("/__MACOSX", StringComparison.OrdinalIgnoreCase) || diff --git a/src/Microsoft.Android.Build.BaseTasks/HexUtilities.cs b/src/Microsoft.Android.Build.BaseTasks/HexUtilities.cs index ea69f86dfeb..64b83fc9106 100644 --- a/src/Microsoft.Android.Build.BaseTasks/HexUtilities.cs +++ b/src/Microsoft.Android.Build.BaseTasks/HexUtilities.cs @@ -11,7 +11,7 @@ namespace Microsoft.Android.Build.Tasks /// /// This file is also linked into Microsoft.Android.Sdk.TrimmableTypeMap, which /// deliberately does not reference Microsoft.Android.Build.BaseTasks (that would drag - /// Microsoft.Build.*, LibZipSharp, K4os.LZ4 and Mono.Unix into it). Only the copy compiled + /// Microsoft.Build.*, System.IO.Hashing, K4os.LZ4 and Mono.Unix into it). Only the copy compiled /// into Microsoft.Android.Build.BaseTasks is public; the linked copy stays /// internal, otherwise Xamarin.Android.Build.Tasks — which references both /// assemblies — fails with CS0433. diff --git a/src/Microsoft.Android.Build.BaseTasks/MSBuildReferences.projitems b/src/Microsoft.Android.Build.BaseTasks/MSBuildReferences.projitems index e5ccd74facc..ed8c779f8a0 100644 --- a/src/Microsoft.Android.Build.BaseTasks/MSBuildReferences.projitems +++ b/src/Microsoft.Android.Build.BaseTasks/MSBuildReferences.projitems @@ -6,7 +6,6 @@ 18.7.1 10.0.4 - 3.3.0 7.1.0-final.1.21458.1 @@ -17,7 +16,6 @@ - diff --git a/src/Microsoft.Android.Build.BaseTasks/Microsoft.Android.Build.BaseTasks.csproj b/src/Microsoft.Android.Build.BaseTasks/Microsoft.Android.Build.BaseTasks.csproj index 8f3f3b7be8e..830a52662bd 100644 --- a/src/Microsoft.Android.Build.BaseTasks/Microsoft.Android.Build.BaseTasks.csproj +++ b/src/Microsoft.Android.Build.BaseTasks/Microsoft.Android.Build.BaseTasks.csproj @@ -34,6 +34,7 @@ + diff --git a/src/Microsoft.Android.Build.BaseTasks/ZipArchiveExtensions.cs b/src/Microsoft.Android.Build.BaseTasks/ZipArchiveExtensions.cs new file mode 100644 index 00000000000..ab5cc3e4f13 --- /dev/null +++ b/src/Microsoft.Android.Build.BaseTasks/ZipArchiveExtensions.cs @@ -0,0 +1,247 @@ +using System; +using System.IO; +using System.IO.Compression; +using System.Linq; +using System.Text; + +namespace Microsoft.Android.Build.Tasks +{ + public static class ZipArchiveExtensions + { + public static CompressionLevel ToCompressionLevel (this ZipEntryCompressionMethod compressionMethod) + { + return compressionMethod switch { + ZipEntryCompressionMethod.Store => CompressionLevel.NoCompression, + ZipEntryCompressionMethod.Deflate => CompressionLevel.Optimal, + _ => throw new NotSupportedException ($"Unsupported ZIP compression method: {(ushort) compressionMethod}"), + }; + } + + public static ZipArchive OpenZip (string archivePath, FileMode fileMode, Encoding? entryNameEncoding = null) + { + if (archivePath == null) + throw new ArgumentNullException (nameof (archivePath)); + + FileMode actualFileMode = fileMode; + ZipArchiveMode archiveMode; + + switch (fileMode) { + case FileMode.Create: + case FileMode.CreateNew: + case FileMode.Truncate: + archiveMode = ZipArchiveMode.Create; + break; + case FileMode.Open: + archiveMode = ZipArchiveMode.Update; + break; + case FileMode.OpenOrCreate: + archiveMode = ZipArchiveMode.Update; + break; + default: + throw new ArgumentOutOfRangeException (nameof (fileMode), fileMode, null); + } + + var stream = new FileStream (archivePath, actualFileMode, FileAccess.ReadWrite, FileShare.Read); + return OpenZip (stream, archiveMode, leaveOpen: false, entryNameEncoding); + } + + public static ZipArchive OpenZip (Stream stream, ZipArchiveMode mode = ZipArchiveMode.Read, bool leaveOpen = false, Encoding? entryNameEncoding = null) + { + if (stream == null) + throw new ArgumentNullException (nameof (stream)); + + return new ZipArchive (stream, mode, leaveOpen, entryNameEncoding); + } + + public static bool ContainsEntry (this ZipArchive archive, string entryName, StringComparison comparison = StringComparison.Ordinal) + => archive.ReadEntry (entryName, comparison) != null; + + public static ZipArchiveEntry? ReadEntry (this ZipArchive archive, string entryName, StringComparison comparison = StringComparison.Ordinal) + { + if (archive == null) + throw new ArgumentNullException (nameof (archive)); + if (entryName == null) + throw new ArgumentNullException (nameof (entryName)); + + return archive.Entries.FirstOrDefault (entry => string.Equals (entry.FullName, entryName, comparison)); + } + + public static bool IsDirectory (this ZipArchiveEntry entry) + { + if (entry == null) + throw new ArgumentNullException (nameof (entry)); + + return entry.FullName.EndsWith ("/", StringComparison.Ordinal) || entry.FullName.EndsWith ("\\", StringComparison.Ordinal); + } + + public static void Extract (this ZipArchiveEntry entry, Stream destination) + { + if (entry == null) + throw new ArgumentNullException (nameof (entry)); + if (destination == null) + throw new ArgumentNullException (nameof (destination)); + + using var source = entry.Open (); + source.CopyTo (destination); + } + + public static void Extract (this ZipArchiveEntry entry, string destinationDirectory, string? destinationFileName = null) + { + if (entry == null) + throw new ArgumentNullException (nameof (entry)); + if (destinationDirectory == null) + throw new ArgumentNullException (nameof (destinationDirectory)); + + var fileName = destinationFileName ?? entry.FullName.Replace ('/', Path.DirectorySeparatorChar); + var destinationPath = Path.Combine (destinationDirectory, fileName); + var destinationFolder = Path.GetDirectoryName (destinationPath); + if (!string.IsNullOrEmpty (destinationFolder)) + Directory.CreateDirectory (destinationFolder); + + using var output = File.Create (destinationPath); + entry.Extract (output); + } + + public static void AddEntry (this ZipArchive archive, string entryName, string contents, Encoding encoding, CompressionLevel compressionLevel = CompressionLevel.Optimal) + { + if (archive == null) + throw new ArgumentNullException (nameof (archive)); + if (entryName == null) + throw new ArgumentNullException (nameof (entryName)); + if (contents == null) + throw new ArgumentNullException (nameof (contents)); + if (encoding == null) + throw new ArgumentNullException (nameof (encoding)); + + DeleteEntry (archive, entryName); + var entry = archive.CreateEntry (entryName, compressionLevel); + using var writer = new StreamWriter (entry.Open (), encoding); + writer.Write (contents); + } + + public static void AddStream (this ZipArchive archive, Stream source, string entryName, CompressionLevel compressionLevel = CompressionLevel.Optimal) + { + if (archive == null) + throw new ArgumentNullException (nameof (archive)); + if (source == null) + throw new ArgumentNullException (nameof (source)); + if (entryName == null) + throw new ArgumentNullException (nameof (entryName)); + + DeleteEntry (archive, entryName); + var entry = archive.CreateEntry (entryName, compressionLevel); + using var destination = entry.Open (); + source.CopyTo (destination); + } + + public static void AddFile (this ZipArchive archive, string filePath, string entryName, CompressionLevel compressionLevel = CompressionLevel.Optimal) + { + if (archive == null) + throw new ArgumentNullException (nameof (archive)); + if (filePath == null) + throw new ArgumentNullException (nameof (filePath)); + if (entryName == null) + throw new ArgumentNullException (nameof (entryName)); + + DeleteEntry (archive, entryName); + ZipFileExtensions.CreateEntryFromFile (archive, filePath, entryName, compressionLevel); + } + + public static void AddDirectory (this ZipArchive archive, string directory, string directoryPathInArchive = "", CompressionLevel compressionLevel = CompressionLevel.Optimal) + { + if (archive == null) + throw new ArgumentNullException (nameof (archive)); + if (directory == null) + throw new ArgumentNullException (nameof (directory)); + + directory = directory.Replace ('/', Path.DirectorySeparatorChar).Replace ('\\', Path.DirectorySeparatorChar); + directory = Path.GetFullPath (directory); + if (directory [directory.Length - 1] == Path.DirectorySeparatorChar) + directory = directory.Substring (0, directory.Length - 1); + + AddDirectoryContents (directory); + + void AddDirectoryContents (string currentDirectory) + { + foreach (var filePath in Directory.GetFiles (currentDirectory, "*.*", SearchOption.TopDirectoryOnly).OrderBy (path => path, StringComparer.Ordinal)) { + var fileInfo = new FileInfo (filePath); + if ((fileInfo.Attributes & FileAttributes.Hidden) != 0) + continue; + + var relativePath = filePath.Substring (directory.Length).TrimStart (Path.DirectorySeparatorChar, Path.AltDirectorySeparatorChar).Replace ('\\', '/'); + var entryName = string.IsNullOrEmpty (directoryPathInArchive) ? relativePath : $"{directoryPathInArchive.TrimEnd ('/')}/{relativePath}"; + archive.AddFile (filePath, entryName, compressionLevel); + } + + foreach (var childDirectory in Directory.GetDirectories (currentDirectory, "*", SearchOption.TopDirectoryOnly).OrderBy (path => path, StringComparer.Ordinal)) { + var directoryInfo = new DirectoryInfo (childDirectory); + if ((directoryInfo.Attributes & FileAttributes.Hidden) != 0) + continue; + + AddDirectoryContents (childDirectory); + } + } + } + + public static bool MoveEntry (this ZipArchive archive, string oldEntryName, string newEntryName, CompressionLevel compressionLevel) + { + if (archive == null) + throw new ArgumentNullException (nameof (archive)); + if (oldEntryName == null) + throw new ArgumentNullException (nameof (oldEntryName)); + if (newEntryName == null) + throw new ArgumentNullException (nameof (newEntryName)); + + var source = archive.ReadEntry (oldEntryName, StringComparison.Ordinal); + if (source == null) + return false; + + using var buffer = MemoryStreamPool.Shared.Rent (); + source.Extract (buffer); + buffer.Position = 0; + + DeleteEntry (archive, newEntryName); + var destination = archive.CreateEntry (newEntryName, compressionLevel); + destination.LastWriteTime = source.LastWriteTime; + using (var destinationStream = destination.Open ()) { + buffer.CopyTo (destinationStream); + } + + source.Delete (); + return true; + } + + public static void FixupWindowsPathSeparators (this ZipArchive archive, Func compressionLevelSelector, Action? onRename = null) + { + if (archive == null) + throw new ArgumentNullException (nameof (archive)); + if (compressionLevelSelector == null) + throw new ArgumentNullException (nameof (compressionLevelSelector)); + + foreach (var entryName in archive.Entries + .Where (entry => entry.FullName.Contains ('\\')) + .Select (entry => entry.FullName) + .ToArray ()) { + var entry = archive.ReadEntry (entryName, StringComparison.Ordinal); + if (entry == null) + continue; + + var normalizedName = entryName.Replace ('\\', '/'); + if (normalizedName == entryName) + continue; + + var compressionLevel = compressionLevelSelector (entry); + onRename?.Invoke (entryName, normalizedName); + archive.MoveEntry (entryName, normalizedName, compressionLevel); + } + } + + static void DeleteEntry (ZipArchive archive, string entryName) + { + if (archive.Mode == ZipArchiveMode.Create) + return; + + archive.ReadEntry (entryName, StringComparison.Ordinal)?.Delete (); + } + } +} diff --git a/src/Microsoft.Android.Build.BaseTasks/ZipArchiveMetadataReader.cs b/src/Microsoft.Android.Build.BaseTasks/ZipArchiveMetadataReader.cs new file mode 100644 index 00000000000..985f273a0dd --- /dev/null +++ b/src/Microsoft.Android.Build.BaseTasks/ZipArchiveMetadataReader.cs @@ -0,0 +1,157 @@ +using System; +using System.Collections.Generic; +using System.IO; +using System.Text; + +namespace Microsoft.Android.Build.Tasks +{ + public enum ZipEntryCompressionMethod : ushort + { + Store = 0, + Deflate = 8, + } + + public readonly struct ZipEntryMetadata + { + public string FullName { get; } + public uint Crc32 { get; } + public long CompressedSize { get; } + public long UncompressedSize { get; } + public ZipEntryCompressionMethod CompressionMethod { get; } + + public ZipEntryMetadata (string fullName, uint crc32, long compressedSize, long uncompressedSize, ZipEntryCompressionMethod compressionMethod) + { + FullName = fullName; + Crc32 = crc32; + CompressedSize = compressedSize; + UncompressedSize = uncompressedSize; + CompressionMethod = compressionMethod; + } + } + + public static class ZipArchiveMetadataReader + { + const uint EndOfCentralDirectorySignature = 0x06054b50; + const uint CentralDirectoryFileHeaderSignature = 0x02014b50; + const int EndOfCentralDirectoryMinimumSize = 22; + const int EndOfCentralDirectorySearchWindow = ushort.MaxValue + EndOfCentralDirectoryMinimumSize; + static readonly Encoding Cp437 = CreateCp437Encoding (); + + public static IReadOnlyDictionary Read (string archivePath) + { + if (archivePath == null) + throw new ArgumentNullException (nameof (archivePath)); + + using var stream = File.OpenRead (archivePath); + return Read (stream); + } + + public static IReadOnlyDictionary Read (Stream stream) + { + if (stream == null) + throw new ArgumentNullException (nameof (stream)); + if (!stream.CanSeek) + throw new NotSupportedException ("ZIP metadata requires a seekable stream."); + + long originalPosition = stream.Position; + try { + return ReadCore (stream); + } finally { + stream.Seek (originalPosition, SeekOrigin.Begin); + } + } + + static IReadOnlyDictionary ReadCore (Stream stream) + { + long endOfCentralDirectoryOffset = FindEndOfCentralDirectory (stream); + stream.Seek (endOfCentralDirectoryOffset + 10, SeekOrigin.Begin); + + using var reader = new BinaryReader (stream, Encoding.UTF8, leaveOpen: true); + ushort entryCount = reader.ReadUInt16 (); + uint centralDirectorySize = reader.ReadUInt32 (); + uint centralDirectoryOffset = reader.ReadUInt32 (); + + stream.Seek (centralDirectoryOffset, SeekOrigin.Begin); + var entries = new Dictionary ((int) entryCount, StringComparer.Ordinal); + long centralDirectoryEnd = centralDirectoryOffset + centralDirectorySize; + while (stream.Position < centralDirectoryEnd && entries.Count < entryCount) { + if (reader.ReadUInt32 () != CentralDirectoryFileHeaderSignature) + throw new InvalidDataException ("Invalid ZIP central directory header."); + + reader.ReadUInt16 (); // version made by + reader.ReadUInt16 (); // version needed to extract + ushort flags = reader.ReadUInt16 (); + ushort compressionMethod = reader.ReadUInt16 (); + reader.ReadUInt16 (); // last mod file time + reader.ReadUInt16 (); // last mod file date + uint crc32 = reader.ReadUInt32 (); + uint compressedSize = reader.ReadUInt32 (); + uint uncompressedSize = reader.ReadUInt32 (); + ushort fileNameLength = reader.ReadUInt16 (); + ushort extraFieldLength = reader.ReadUInt16 (); + ushort fileCommentLength = reader.ReadUInt16 (); + reader.ReadUInt16 (); // disk number start + reader.ReadUInt16 (); // internal file attributes + reader.ReadUInt32 (); // external file attributes + reader.ReadUInt32 (); // relative offset of local header + + var fileNameBytes = reader.ReadBytes (fileNameLength); + var encoding = (flags & (1 << 11)) != 0 ? Encoding.UTF8 : Cp437; + var fullName = encoding.GetString (fileNameBytes); + + if (stream.Position + extraFieldLength + fileCommentLength > stream.Length) + throw new InvalidDataException ("ZIP central directory entry exceeds the available data."); + + stream.Seek (extraFieldLength + fileCommentLength, SeekOrigin.Current); + entries [fullName] = new ZipEntryMetadata ( + fullName, + crc32, + compressedSize, + uncompressedSize, + (ZipEntryCompressionMethod) compressionMethod + ); + } + + return entries; + } + + static long FindEndOfCentralDirectory (Stream stream) + { + long searchLength = Math.Min (stream.Length, EndOfCentralDirectorySearchWindow); + var buffer = new byte [searchLength]; + stream.Seek (-searchLength, SeekOrigin.End); + ReadExactly (stream, buffer, 0, buffer.Length); + + for (int index = buffer.Length - EndOfCentralDirectoryMinimumSize; index >= 0; index--) { + if ( + buffer [index] == 0x50 && + buffer [index + 1] == 0x4b && + buffer [index + 2] == 0x05 && + buffer [index + 3] == 0x06 + ) { + return stream.Length - searchLength + index; + } + } + + throw new InvalidDataException ("Could not locate the ZIP end of central directory record."); + } + + static void ReadExactly (Stream stream, byte [] buffer, int offset, int count) + { + while (count > 0) { + int bytesRead = stream.Read (buffer, offset, count); + if (bytesRead == 0) + throw new EndOfStreamException (); + + offset += bytesRead; + count -= bytesRead; + } + } + + static Encoding CreateCp437Encoding () + { + Encoding.RegisterProvider (CodePagesEncodingProvider.Instance); + return Encoding.GetEncoding (437); + } + } +} diff --git a/src/Microsoft.Android.Build.Tasks/BuildArchive.cs b/src/Microsoft.Android.Build.Tasks/BuildArchive.cs new file mode 100644 index 00000000000..134928f49dd --- /dev/null +++ b/src/Microsoft.Android.Build.Tasks/BuildArchive.cs @@ -0,0 +1,347 @@ +#nullable enable +using System; +using System.Collections.Generic; +using System.IO; +using System.IO.Compression; +using System.IO.Hashing; +using Microsoft.Android.Build.Tasks; +using Microsoft.Build.Framework; + +namespace Microsoft.Android.Tasks; + +/// +/// Takes a list of files and adds them to an APK archive. If the APK archive already +/// exists, files are only added if they were changed. Note *ALL* files to be in the final +/// APK must be passed in via @(FilesToAddToArchive). This task will determine any unchanged files +/// and skip them, as well as remove any existing files in the APK that are no longer required. +/// +public class BuildArchive : AndroidTask +{ + public override string TaskPrefix => "BAA"; + + public string? AndroidPackageFormat { get; set; } + + public string? ApkInputPath { get; set; } + + [Required] + public string ApkOutputPath { get; set; } = ""; + + [Required] + public ITaskItem [] FilesToAddToArchive { get; set; } = []; + + public string? UncompressedFileExtensions { get; set; } + + HashSet? uncompressedFileExtensions; + + HashSet UncompressedFileExtensionsSet => uncompressedFileExtensions ??= ParseUncompressedFileExtensions (); + + CompressionLevel uncompressedFileCompression = CompressionLevel.NoCompression; + + public override bool RunTask () + { + bool isAab = string.Equals (AndroidPackageFormat, "aab", StringComparison.OrdinalIgnoreCase); + if (isAab) { + uncompressedFileCompression = CompressionLevel.Optimal; + } + + Directory.CreateDirectory (Path.GetDirectoryName (ApkOutputPath) ?? "."); + + bool refreshExistingOutput = true; + if (!string.IsNullOrEmpty (ApkInputPath) && File.Exists (ApkInputPath) && !File.Exists (ApkOutputPath)) { + Log.LogDebugMessage ($"Copying {ApkInputPath} to {ApkOutputPath}"); + File.Copy (ApkInputPath, ApkOutputPath, overwrite: true); + refreshExistingOutput = false; + } + + using var apk = ZipArchiveExtensions.OpenZip (ApkOutputPath, FileMode.OpenOrCreate); + var existingEntries = new List (); + + if (refreshExistingOutput) { + foreach (var entry in apk.Entries) { + Log.LogDebugMessage ($"Registering item {entry.FullName}"); + existingEntries.Add (entry.FullName); + } + } + + if (!string.IsNullOrEmpty (ApkInputPath) && File.Exists (ApkInputPath) && refreshExistingOutput) { + RefreshEntriesFromInputArchive (apk, existingEntries, isAab); + } + + apk.FixupWindowsPathSeparators ( + entry => ToCompressionLevel (entry.CompressionMethod), + (source, destination) => { + Log.LogDebugMessage ($"Fixing up malformed entry `{source}` -> `{destination}`"); + existingEntries.Remove (source); + existingEntries.Add (destination); + } + ); + + foreach (var file in FilesToAddToArchive) { + if (!AddItemToArchive (apk, file, existingEntries)) + return false; + } + + foreach (var entry in existingEntries) { + if (string.Equals (Path.GetFileName (entry), "AndroidManifest.xml", StringComparison.OrdinalIgnoreCase)) + continue; + + Log.LogDebugMessage ($"Removing {entry} as it is no longer required."); + apk.ReadEntry (entry, StringComparison.Ordinal)?.Delete (); + } + + if (isAab) { + FixupBundleManifest (apk); + } + + return !Log.HasLoggedErrors; + } + + void RefreshEntriesFromInputArchive (ZipArchive apk, List existingEntries, bool isAab) + { + if (ApkInputPath == null) + throw new InvalidOperationException ("ApkInputPath must not be null when refreshing the output archive."); + + DateTime lastWriteOutput = File.Exists (ApkOutputPath) ? File.GetLastWriteTimeUtc (ApkOutputPath) : DateTime.MinValue; + DateTime lastWriteInput = File.GetLastWriteTimeUtc (ApkInputPath); + var inputMetadata = ZipArchiveMetadataReader.Read (ApkInputPath); + + using var packaged = ZipArchiveExtensions.OpenZip (ApkInputPath, FileMode.Open); + foreach (var entry in packaged.Entries) { + if (entry.IsDirectory ()) { + continue; + } + + string entryName = entry.FullName; + if (entryName.Contains ("\\")) { + entryName = entryName.Replace ('\\', '/'); + Log.LogDebugMessage ($"Fixing up malformed entry `{entry.FullName}` -> `{entryName}`"); + } + + if (entryName == "AndroidManifest.xml" && isAab) { + Log.LogDebugMessage ("Renaming AndroidManifest.xml to manifest/AndroidManifest.xml"); + entryName = "manifest/AndroidManifest.xml"; + } + + Log.LogDebugMessage ($"Deregistering item {entryName}"); + existingEntries.Remove (entryName); + + if (lastWriteInput <= lastWriteOutput) { + Log.LogDebugMessage ($"Skipping to next item. {lastWriteInput} <= {lastWriteOutput}."); + continue; + } + + if (!inputMetadata.TryGetValue (entry.FullName, out ZipEntryMetadata metadata)) { + throw new InvalidDataException ($"Unable to read ZIP metadata for '{entry.FullName}' in '{ApkInputPath}'."); + } + + var currentEntry = apk.ReadEntry (entryName, StringComparison.Ordinal); + if (currentEntry != null && metadata.Crc32 == GetEntryCrc32 (currentEntry) && metadata.CompressedSize == currentEntry.CompressedLength) { + Log.LogDebugMessage ($"Skipping {entryName} from {ApkInputPath} as its up to date."); + continue; + } + + if (currentEntry != null) { + currentEntry.Delete (); + } + + Log.LogDebugMessage ($"Refreshing {entryName} from {ApkInputPath}"); + CopyEntryToArchive (apk, entryName, entry, metadata.CompressionMethod.ToCompressionLevel ()); + } + } + + bool AddItemToArchive (ZipArchive apk, ITaskItem item, List existingEntries) + { + string diskPath = item.ItemSpec; + string? archivePath = GetRequiredMetadata (item, "FilesToAddToArchive", "ArchivePath"); + if (archivePath == null) + return false; + + archivePath = archivePath.Replace ('\\', '/'); + + string jarEntryName = GetMetadataOrDefault (item, "JavaArchiveEntry", string.Empty); + if (!string.IsNullOrEmpty (jarEntryName)) { + AddJarEntryToArchive (apk, diskPath, archivePath, jarEntryName, existingEntries); + return !Log.HasLoggedErrors; + } + + AddFileToArchiveIfNewer (apk, diskPath, archivePath, item, existingEntries); + return !Log.HasLoggedErrors; + } + + void AddJarEntryToArchive (ZipArchive apk, string diskPath, string archivePath, string jarEntryName, List existingEntries) + { + string jarFilePath = diskPath.Substring (0, diskPath.Length - (jarEntryName.Length + 1)); + bool wasExistingOutputEntry = existingEntries.Remove (archivePath); + var currentEntry = apk.ReadEntry (archivePath, StringComparison.Ordinal); + + if (currentEntry != null && !wasExistingOutputEntry) { + Log.LogDebugMessage ("Failed to add jar entry {0} from {1}: the same file already exists in the apk", jarEntryName, Path.GetFileName (jarFilePath)); + return; + } + + using var jar = ZipArchiveExtensions.OpenZip (jarFilePath, FileMode.Open); + var jarEntry = jar.ReadEntry (jarEntryName, StringComparison.Ordinal); + if (jarEntry == null) { + Log.LogDebugMessage ("Failed to add jar entry {0} from {1}: entry not found in jar.", jarEntryName, jarFilePath); + if (wasExistingOutputEntry) + existingEntries.Add (archivePath); + return; + } + + if (currentEntry != null && GetEntryCrc32 (currentEntry) == GetEntryCrc32 (jarEntry)) { + Log.LogDebugMessage ("Skipping {0} from {1} as it is up to date.", jarEntryName, jarFilePath); + return; + } + + currentEntry?.Delete (); + + using var buffer = MemoryStreamPool.Shared.Rent (); + jarEntry.Extract (buffer); + buffer.Position = 0; + Log.LogDebugMessage ($"Adding {jarEntryName} from {jarFilePath} as the archive file is out of date."); + apk.AddStream (buffer, archivePath); + } + + bool AddFileToArchiveIfNewer (ZipArchive apk, string file, string archivePath, ITaskItem item, List existingEntries) + { + ZipCompressionMethod compressionMethod = GetCompressionMethod (item); + existingEntries.Remove (archivePath); + + var entry = apk.ReadEntry (archivePath, StringComparison.Ordinal); + if (entry == null) { + apk.AddFile (file, archivePath, ToCompressionLevel (compressionMethod)); + Log.LogDebugMessage ($"Adding {file} as it doesn't already exist."); + return true; + } + + if (GetExistingCompressionMethod (entry) != compressionMethod) { + Log.LogDebugMessage ($"Updating {file} as the compression level changed."); + entry.Delete (); + apk.AddFile (file, archivePath, ToCompressionLevel (compressionMethod)); + return true; + } + + uint existingDosTime = DateTimeToDosTime (entry.LastWriteTime.UtcDateTime); + uint fileDosTime = DateTimeToDosTime (File.GetLastWriteTimeUtc (file)); + if (existingDosTime < fileDosTime) { + Log.LogDebugMessage ($"Updating {file} as the file write time is newer: file in zip - '{existingDosTime}', file on disk - '{fileDosTime}'."); + entry.Delete (); + apk.AddFile (file, archivePath, ToCompressionLevel (compressionMethod)); + return true; + } + + Log.LogDebugMessage ($"Skipping {file} as the archive file is up to date."); + return false; + } + + void FixupBundleManifest (ZipArchive apk) + { + var manifest = apk.ReadEntry ("AndroidManifest.xml", StringComparison.Ordinal); + if (manifest == null) { + Log.LogDebugMessage ("No AndroidManifest.xml. Skipping Fixup"); + return; + } + + Log.LogDebugMessage ("Fixing up AndroidManifest.xml to be manifest/AndroidManifest.xml."); + apk.MoveEntry ("AndroidManifest.xml", "manifest/AndroidManifest.xml", ToCompressionLevel (manifest.CompressionMethod)); + } + + void CopyEntryToArchive (ZipArchive archive, string destinationEntryName, ZipArchiveEntry sourceEntry, CompressionLevel compressionLevel) + { + var destinationEntry = archive.CreateEntry (destinationEntryName, compressionLevel); + destinationEntry.LastWriteTime = sourceEntry.LastWriteTime; + using var source = sourceEntry.Open (); + using var destination = destinationEntry.Open (); + source.CopyTo (destination); + } + + ZipCompressionMethod GetCompressionMethod (ITaskItem item) + { + if (UncompressedFileExtensionsSet.Contains (Path.GetExtension (item.ItemSpec))) { + return uncompressedFileCompression == CompressionLevel.NoCompression ? ZipCompressionMethod.Stored : ZipCompressionMethod.Deflate; + } + + return ZipCompressionMethod.Deflate; + } + + static CompressionLevel ToCompressionLevel (ZipCompressionMethod compressionMethod) + { + return compressionMethod switch { + ZipCompressionMethod.Stored => CompressionLevel.NoCompression, + ZipCompressionMethod.Deflate => CompressionLevel.Optimal, + _ => throw new NotSupportedException ($"Unsupported ZIP compression method: {compressionMethod}"), + }; + } + + static ZipCompressionMethod GetExistingCompressionMethod (ZipArchiveEntry entry) + { + return entry.CompressionMethod switch { + ZipCompressionMethod.Stored => ZipCompressionMethod.Stored, + ZipCompressionMethod.Deflate => ZipCompressionMethod.Deflate, + _ => throw new NotSupportedException ($"Unsupported ZIP compression method: {entry.CompressionMethod}"), + }; + } + + static uint GetEntryCrc32 (ZipArchiveEntry entry) + { + using var buffer = MemoryStreamPool.Shared.Rent (); + entry.Extract (buffer); + if (buffer.TryGetBuffer (out ArraySegment segment) && segment.Array != null) { + return Crc32.HashToUInt32 (new ReadOnlySpan (segment.Array, segment.Offset, (int) buffer.Length)); + } + + return Crc32.HashToUInt32 (buffer.ToArray ()); + } + + HashSet ParseUncompressedFileExtensions () + { + var parsedExtensions = new HashSet (StringComparer.OrdinalIgnoreCase); + + foreach (var extension in UncompressedFileExtensions?.Split ([';', ','], StringSplitOptions.RemoveEmptyEntries) ?? []) { + var normalized = extension.Trim (); + if (string.IsNullOrEmpty (normalized)) { + continue; + } + + if (normalized [0] != '.') { + normalized = $".{normalized}"; + } + + parsedExtensions.Add (normalized); + } + + return parsedExtensions; + } + + string? GetRequiredMetadata (ITaskItem item, string itemName, string metadataName) + { + string metadataValue = item.GetMetadata (metadataName) ?? ""; + if (!string.IsNullOrWhiteSpace (metadataValue)) + return metadataValue; + + Log.LogError ($"The '{metadataName}' metadata on '{itemName}' is required for '{item.ItemSpec}'."); + return null; + } + + static string GetMetadataOrDefault (ITaskItem item, string metadataName, string defaultValue) + { + string metadataValue = item.GetMetadata (metadataName) ?? ""; + if (string.IsNullOrEmpty (metadataValue)) + return defaultValue; + + return metadataValue; + } + + const int ValidZipDate_YearMin = 1980; + + static uint DateTimeToDosTime (DateTime dateTime) + { + int ret = ((dateTime.Year - ValidZipDate_YearMin) & 0x7F); + ret = (ret << 4) + dateTime.Month; + ret = (ret << 5) + dateTime.Day; + ret = (ret << 5) + dateTime.Hour; + ret = (ret << 6) + dateTime.Minute; + ret = (ret << 5) + (dateTime.Second / 2); + return (uint) ret; + } +} diff --git a/src/Microsoft.Android.Build.Tasks/Microsoft.Android.Build.Tasks.csproj b/src/Microsoft.Android.Build.Tasks/Microsoft.Android.Build.Tasks.csproj index 245e9c7c077..106fa0b815a 100644 --- a/src/Microsoft.Android.Build.Tasks/Microsoft.Android.Build.Tasks.csproj +++ b/src/Microsoft.Android.Build.Tasks/Microsoft.Android.Build.Tasks.csproj @@ -2,14 +2,14 @@ - $(DotNetTargetFramework) + net12.0 Microsoft.Android.Tasks enable Nullable diff --git a/src/Xamarin.Android.Build.Debugging.Tasks/Tasks/GetGdbSymbols.cs b/src/Xamarin.Android.Build.Debugging.Tasks/Tasks/GetGdbSymbols.cs index e26489c7c63..b0d9eb43178 100644 --- a/src/Xamarin.Android.Build.Debugging.Tasks/Tasks/GetGdbSymbols.cs +++ b/src/Xamarin.Android.Build.Debugging.Tasks/Tasks/GetGdbSymbols.cs @@ -1,9 +1,9 @@ #nullable enable using System; using System.IO; +using System.IO.Compression; using System.Linq; using System.Threading; -using Xamarin.Tools.Zip; using Microsoft.Build.Framework; using Microsoft.Build.Utilities; using Xamarin.AndroidTools; @@ -48,8 +48,8 @@ public override bool RunTask () void ExtractFilesFromPath(string apk, string path) { - using (var zip = ZipArchive.Open (apk, FileMode.Open)) { - foreach (var e in zip.Where (x => x.FullName.StartsWith (path, StringComparison.OrdinalIgnoreCase))) { + using (var zip = ZipArchiveExtensions.OpenZip (apk, FileMode.Open)) { + foreach (var e in zip.Entries.Where (x => x.FullName.StartsWith (path, StringComparison.OrdinalIgnoreCase))) { Log.LogDebugMessage ("Extracting {0} from {1}", e.FullName, apk); using (var fs = new FileStream (Path.Combine (GdbSymbolsPath, Path.GetFileName (e.FullName)), FileMode.OpenOrCreate)) { e.Extract (fs); diff --git a/src/Xamarin.Android.Build.Debugging.Tasks/Xamarin.Android.Build.Debugging.Tasks.csproj b/src/Xamarin.Android.Build.Debugging.Tasks/Xamarin.Android.Build.Debugging.Tasks.csproj index 37f23860cd5..ec09e3f2318 100644 --- a/src/Xamarin.Android.Build.Debugging.Tasks/Xamarin.Android.Build.Debugging.Tasks.csproj +++ b/src/Xamarin.Android.Build.Debugging.Tasks/Xamarin.Android.Build.Debugging.Tasks.csproj @@ -8,7 +8,6 @@ 12 netstandard2.0 - true $(DefineConstants);MSBUILD $(MicrosoftAndroidSdkOutDir) false diff --git a/src/Xamarin.Android.Build.Tasks/Tasks/Aapt2LinkAssetPack.cs b/src/Xamarin.Android.Build.Tasks/Tasks/Aapt2LinkAssetPack.cs index 204c7dfb7dd..c1038a728db 100644 --- a/src/Xamarin.Android.Build.Tasks/Tasks/Aapt2LinkAssetPack.cs +++ b/src/Xamarin.Android.Build.Tasks/Tasks/Aapt2LinkAssetPack.cs @@ -41,12 +41,23 @@ public async override System.Threading.Tasks.Task RunTaskAsync () RunAapt (GenerateCommandLineCommands (Manifest, OutputArchive), OutputArchive.ItemSpec); ProcessOutput (); if (File.Exists (OutputArchive.ItemSpec)) { + var zipMetadata = ZipArchiveMetadataReader.Read (OutputArchive.ItemSpec); // move the manifest to the right place. - using (var zip = new ZipArchiveEx (OutputArchive.ItemSpec, File.Exists (OutputArchive.ItemSpec) ? FileMode.Open : FileMode.Create)) { - zip.MoveEntry ("AndroidManifest.xml", "manifest/AndroidManifest.xml"); - zip.Archive.DeleteEntry ("resources.pb"); - // Fix up aapt2 not dealing with '\' in subdirectories for assets. - zip.FixupWindowsPathSeparators ((a, b) => LogDebugMessage ($"Fixing up malformed entry `{a}` -> `{b}`")); + using (var zip = ZipArchiveExtensions.OpenZip (OutputArchive.ItemSpec, FileMode.Open)) { + if (zipMetadata.TryGetValue ("AndroidManifest.xml", out var manifestMetadata)) { + zip.MoveEntry ("AndroidManifest.xml", "manifest/AndroidManifest.xml", manifestMetadata.CompressionMethod.ToCompressionLevel ()); + } + zip.ReadEntry ("resources.pb", StringComparison.Ordinal)?.Delete (); + zip.FixupWindowsPathSeparators ( + entry => { + if (!zipMetadata.TryGetValue (entry.FullName, out var metadata)) { + throw new InvalidDataException ($"Unable to read ZIP metadata for '{entry.FullName}' in '{OutputArchive.ItemSpec}'."); + } + + return metadata.CompressionMethod.ToCompressionLevel (); + }, + (a, b) => LogDebugMessage ($"Fixing up malformed entry `{a}` -> `{b}`") + ); } } await System.Threading.Tasks.Task.CompletedTask; diff --git a/src/Xamarin.Android.Build.Tasks/Tasks/BuildArchive.cs b/src/Xamarin.Android.Build.Tasks/Tasks/BuildArchive.cs deleted file mode 100644 index 813bd2ea859..00000000000 --- a/src/Xamarin.Android.Build.Tasks/Tasks/BuildArchive.cs +++ /dev/null @@ -1,305 +0,0 @@ -#nullable enable -using System; -using System.Collections.Generic; -using System.IO; -using System.Linq; -using System.Runtime.InteropServices; -using Microsoft.Android.Build.Tasks; -using Microsoft.Build.Framework; -using Xamarin.Tools.Zip; - -namespace Xamarin.Android.Tasks; - -/// -/// Takes a list of files and adds them to an APK archive. If the APK archive already -/// exists, files are only added if they were changed. Note *ALL* files to be in the final -/// APK must be passed in via @(FilesToAddToArchive). This task will determine any unchanged files -/// and skip them, as well as remove any existing files in the APK that are no longer required. -/// -public class BuildArchive : AndroidTask -{ - public override string TaskPrefix => "BAA"; - - public string? AndroidPackageFormat { get; set; } - - public string? ApkInputPath { get; set; } - - [Required] - public string ApkOutputPath { get; set; } = ""; - - [Required] - public ITaskItem [] FilesToAddToArchive { get; set; } = []; - - public string? UncompressedFileExtensions { get; set; } - - public bool UseLibZipSharp { get; set; } - - public string? ZipFlushFilesLimit { get; set; } - - public string? ZipFlushSizeLimit { get; set; } - - HashSet? uncompressedFileExtensions; - HashSet UncompressedFileExtensionsSet => uncompressedFileExtensions ??= ParseUncompressedFileExtensions (); - - CompressionMethod uncompressedMethod = CompressionMethod.Store; - - public override bool RunTask () - { - var is_aab = string.Compare (AndroidPackageFormat, "aab", true) == 0; - - // Nothing needs to be compressed with app bundles. BundleConfig.json specifies the final compression mode. - if (is_aab) - uncompressedMethod = CompressionMethod.Default; - - var refresh = true; - - // If we have an input apk but no output apk, copy it to the output - // so we don't modify the original. - if (ApkInputPath is not null && File.Exists (ApkInputPath) && !File.Exists (ApkOutputPath)) { - Log.LogDebugMessage ($"Copying {ApkInputPath} to {ApkOutputPath}"); - File.Copy (ApkInputPath, ApkOutputPath, overwrite: true); - refresh = false; - } - - using var apk = ZipArchiveDotNet.Create (Log, ApkOutputPath, System.IO.Compression.ZipArchiveMode.Update, ShouldFallbackToLibZipSharp ()); - - // Set up AutoFlush - if (int.TryParse (ZipFlushFilesLimit, out int flushFilesLimit)) { - apk.ZipFlushFilesLimit = flushFilesLimit; - } - - if (int.TryParse (ZipFlushSizeLimit, out int flushSizeLimit)) { - apk.ZipFlushSizeLimit = flushSizeLimit; - } - - // If we're modifying an existing APK we need to track what entries we started - // with so we can remove any existing entries that are no longer used. - var existingEntries = new List (); - - if (refresh) { - foreach (var entry in apk.GetAllEntryNames ()) { - Log.LogDebugMessage ($"Registering item {entry}"); - existingEntries.Add (entry); - } - } - - // If we're modifying an existing APK we need to update any out - // of date entries in the output APK from the input APK. - if (ApkInputPath is not null && File.Exists (ApkInputPath) && refresh) { - - var lastWriteOutput = File.Exists (ApkOutputPath) ? File.GetLastWriteTimeUtc (ApkOutputPath) : DateTime.MinValue; - var lastWriteInput = File.GetLastWriteTimeUtc (ApkInputPath); - - using (var packaged = new ZipArchiveEx (ApkInputPath, FileMode.Open)) { - foreach (var entry in packaged.Archive) { - - // NOTE: aapt2 is creating zip entries on Windows such as `assets\subfolder/asset2.txt` - var entryName = entry.FullName; - - if (entryName.Contains ("\\")) { - entryName = entryName.Replace ('\\', '/'); - Log.LogDebugMessage ($"Fixing up malformed entry `{entry.FullName}` -> `{entryName}`"); - } - - if (entryName == "AndroidManifest.xml" && is_aab) { - Log.LogDebugMessage ("Renaming AndroidManifest.xml to manifest/AndroidManifest.xml"); - entryName = "manifest/AndroidManifest.xml"; - } - - Log.LogDebugMessage ($"Deregistering item {entryName}"); - existingEntries.Remove (entryName); - - if (lastWriteInput <= lastWriteOutput) { - Log.LogDebugMessage ($"Skipping to next item. {lastWriteInput} <= {lastWriteOutput}."); - continue; - } - - if (apk.ContainsEntry (entryName)) { - var e = apk.GetEntry (entryName); - // check the CRC values as the ModifiedDate is always 01/01/1980 in the aapt generated file. - if (entry.CRC == e.CRC && entry.CompressedSize == e.CompressedSize) { - Log.LogDebugMessage ($"Skipping {entryName} from {ApkInputPath} as its up to date."); - continue; - } - - // Delete the existing entry so we can replace it with the new one. - apk.DeleteEntry (entryName); - } - - var ms = new MemoryStream (); - entry.Extract (ms); - ms.Position = 0; - Log.LogDebugMessage ($"Refreshing {entryName} from {ApkInputPath}"); - apk.AddEntry (ms, entryName, entry.CompressionMethod.ToCompressionLevel ()); - } - } - } - - apk.FixupWindowsPathSeparators (Log); - - // Add the files to the apk - foreach (var file in FilesToAddToArchive) { - var disk_path = file.ItemSpec; - var apk_path = file.GetRequiredMetadata ("FilesToAddToArchive", "ArchivePath", Log); - - // An error will already be logged - if (apk_path is null) { - return !Log.HasLoggedErrors; - } - - apk_path = apk_path.Replace ('\\', '/'); - - // This is a temporary hack for adding files directly from inside a .jar/.aar - // into the APK. Eventually another task should be writing them to disk and just - // passing us a filename like everything else. - var jar_entry_name = file.GetMetadataOrDefault ("JavaArchiveEntry", string.Empty); - - if (jar_entry_name.HasValue ()) { - // ItemSpec for these will be "# - // eg: "obj/myjar.jar#myfile.txt" - var jar_file_path = disk_path.Substring (0, disk_path.Length - (jar_entry_name.Length + 1)); - var wasExistingOutputEntry = existingEntries.Remove (apk_path); - var hasApkEntry = apk.ContainsEntry (apk_path); - - if (hasApkEntry && !wasExistingOutputEntry) { - Log.LogDebugMessage ("Failed to add jar entry {0} from {1}: the same file already exists in the apk", jar_entry_name, Path.GetFileName (jar_file_path)); - continue; - } - - using (var stream = File.OpenRead (jar_file_path)) - using (var jar = ZipArchive.Open (stream)) { - if (!jar.ContainsEntry (jar_entry_name)) { - Log.LogDebugMessage ("Failed to add jar entry {0} from {1}: entry not found in jar.", jar_entry_name, jar_file_path); - if (wasExistingOutputEntry) - existingEntries.Add (apk_path); - continue; - } - - var jar_item = jar.ReadEntry (jar_entry_name); - - if (hasApkEntry) { - // CRC is computed on uncompressed data — matching CRC means identical content regardless of compression settings. - if (apk.GetEntry (apk_path).CRC == jar_item.CRC) { - Log.LogDebugMessage ("Skipping {0} from {1} as it is up to date.", jar_entry_name, jar_file_path); - continue; - } - - apk.DeleteEntry (apk_path); - } - - byte [] data; - var d = MemoryStreamPool.Shared.Rent (); - - try { - jar_item.Extract (d); - data = d.ToArray (); - } finally { - MemoryStreamPool.Shared.Return (d); - } - - Log.LogDebugMessage ($"Adding {jar_entry_name} from {jar_file_path} as the archive file is out of date."); - apk.AddEntry (data, apk_path); - } - - continue; - } - - AddFileToArchiveIfNewer (apk, disk_path, apk_path, file, existingEntries); - } - - // Clean up Removed files. - foreach (var entry in existingEntries) { - // Never remove an AndroidManifest. It may be renamed when using aab. - if (string.Compare (Path.GetFileName (entry), "AndroidManifest.xml", StringComparison.OrdinalIgnoreCase) == 0) - continue; - - Log.LogDebugMessage ($"Removing {entry} as it is no longer required."); - apk.DeleteEntry (entry); - } - - if (is_aab) - FixupArchive (apk); - - return !Log.HasLoggedErrors; - } - - // .NET Framework has a bug where it doesn't handle uncompressed files correctly. - // It writes them as "compressed" (DEFLATE) but with a compression level of 0. This causes - // issues with Android, which expect uncompressed files to be stored correctly. - // We can work around this by using LibZipSharp, which doesn't have this bug. - // This is only necessary if we're on .NET Framework (MSBuild in VSWin) and we have uncompressed files. - bool ShouldFallbackToLibZipSharp () - { - // Explicitly requested via MSBuild property. - if (UseLibZipSharp) { - Log.LogDebugMessage ("Falling back to LibZipSharp because '$(_AndroidUseLibZipSharp)' is 'true'."); - return true; - } - - // Always fallback on .NET Framework - var frameworkDescription = RuntimeInformation.FrameworkDescription; - Log.LogDebugMessage ($"RuntimeInformation.FrameworkDescription: {frameworkDescription}"); - if (Environment.Version.Major < 6) { - Log.LogDebugMessage ($"Falling back to LibZipSharp because we are *not* running on .NET 6+, Environment.Version.Major: {Environment.Version.Major}"); - return true; - } - - // .NET 6+ handles uncompressed files correctly, so we don't need to fallback. - Log.LogDebugMessage ("Using System.IO.Compression because we're running on .NET 6+."); - return false; - } - - bool AddFileToArchiveIfNewer (IZipArchive apk, string file, string inArchivePath, ITaskItem item, List existingEntries) - { - var compressionMethod = GetCompressionLevel (item); - existingEntries.Remove (inArchivePath.Replace (Path.DirectorySeparatorChar, '/')); - - return apk.AddFileIfChanged (Log, file, inArchivePath, compressionMethod); - } - - /// - /// aapt2 is putting AndroidManifest.xml in the root of the archive instead of at manifest/AndroidManifest.xml that bundletool expects. - /// I see no way to change this behavior, so we can move the file for now: - /// https://github.com/aosp-mirror/platform_frameworks_base/blob/e80b45506501815061b079dcb10bf87443bd385d/tools/aapt2/LoadedApk.h#L34 - /// - void FixupArchive (IZipArchive zip) - { - if (!zip.ContainsEntry ("AndroidManifest.xml")) { - Log.LogDebugMessage ($"No AndroidManifest.xml. Skipping Fixup"); - return; - } - - Log.LogDebugMessage ($"Fixing up AndroidManifest.xml to be manifest/AndroidManifest.xml."); - - if (zip.ContainsEntry ("manifest/AndroidManifest.xml")) - zip.DeleteEntry ("manifest/AndroidManifest.xml"); - - zip.MoveEntry ("AndroidManifest.xml", "manifest/AndroidManifest.xml"); - } - - System.IO.Compression.CompressionLevel GetCompressionLevel (ITaskItem item) - { - return (UncompressedFileExtensionsSet.Contains (Path.GetExtension (item.ItemSpec)) ? uncompressedMethod : CompressionMethod.Default).ToCompressionLevel (); - } - - HashSet ParseUncompressedFileExtensions () - { - var uncompressedFileExtensions = new HashSet (StringComparer.OrdinalIgnoreCase); - - foreach (var extension in UncompressedFileExtensions?.Split ([';', ','], StringSplitOptions.RemoveEmptyEntries) ?? []) { - var ext = extension.Trim (); - - if (ext.IsNullOrEmpty ()) { - continue; - } - - if (ext [0] != '.') { - ext = $".{ext}"; - } - - uncompressedFileExtensions.Add (ext); - } - - return uncompressedFileExtensions; - } -} diff --git a/src/Xamarin.Android.Build.Tasks/Tasks/CollectJarContentFilesForArchive.cs b/src/Xamarin.Android.Build.Tasks/Tasks/CollectJarContentFilesForArchive.cs index 544fa391cbb..f26d49ca9b7 100644 --- a/src/Xamarin.Android.Build.Tasks/Tasks/CollectJarContentFilesForArchive.cs +++ b/src/Xamarin.Android.Build.Tasks/Tasks/CollectJarContentFilesForArchive.cs @@ -2,12 +2,12 @@ using System; using System.Collections.Generic; using System.IO; +using System.IO.Compression; using System.Linq; using System.Text; using System.Text.RegularExpressions; using Microsoft.Android.Build.Tasks; using Microsoft.Build.Framework; -using Xamarin.Tools.Zip; namespace Xamarin.Android.Tasks; @@ -71,9 +71,9 @@ public override bool RunTask () foreach (var jarFile in jarFilePaths) { using (var stream = File.OpenRead (jarFile)) - using (var jar = ZipArchive.Open (stream)) { - foreach (var jarItem in jar) { - if (jarItem.IsDirectory) + using (var jar = ZipArchiveExtensions.OpenZip (stream, ZipArchiveMode.Read, leaveOpen: false)) { + foreach (var jarItem in jar.Entries) { + if (jarItem.IsDirectory ()) continue; var name = jarItem.FullName; if (!PackagingUtils.CheckEntryForPackaging (name)) { diff --git a/src/Xamarin.Android.Build.Tasks/Tasks/CreateAar.cs b/src/Xamarin.Android.Build.Tasks/Tasks/CreateAar.cs index a984bc643ba..ebec2cb3edf 100644 --- a/src/Xamarin.Android.Build.Tasks/Tasks/CreateAar.cs +++ b/src/Xamarin.Android.Build.Tasks/Tasks/CreateAar.cs @@ -3,11 +3,11 @@ using System; using System.Collections.Generic; using System.IO; +using System.IO.Compression; using System.Text; using System.Xml.Linq; using Microsoft.Build.Framework; using Xamarin.Android.Tools; -using Xamarin.Tools.Zip; using Microsoft.Android.Build.Tasks; namespace Xamarin.Android.Tasks @@ -47,13 +47,7 @@ public override bool RunTask () } Directory.CreateDirectory (Path.GetDirectoryName (OutputFile)); - using (var stream = File.Create (OutputFile)) - using (var aar = ZipArchive.Open (stream)) { - var existingEntries = new HashSet (StringComparer.Ordinal); - foreach (var entry in aar) { - Log.LogDebugMessage ("Existing entry: " + entry.FullName); - existingEntries.Add (entry.FullName); - } + using (var aar = ZipArchiveExtensions.OpenZip (OutputFile, FileMode.Create)) { if (AndroidAssets != null) { foreach (var asset in AndroidAssets) { // See: https://github.com/xamarin/xamarin-android/commit/665cb59205f8ac565b6acbda740624844bc1cbd9 @@ -63,8 +57,8 @@ public override bool RunTask () } var relative = MonoAndroidHelper.GetRelativePathForAndroidAsset (AssetDirectory, asset); var archivePath = "assets/" + relative.Replace ('\\', '/'); - aar.AddStream (File.OpenRead (asset.ItemSpec), archivePath); - existingEntries.Remove (archivePath); + using var input = File.OpenRead (asset.ItemSpec); + aar.AddStream (input, archivePath); } } if (AndroidResources != null) { @@ -78,24 +72,22 @@ public override bool RunTask () var directory = Path.GetDirectoryName (resource.ItemSpec); var resourcePath = Path.GetFileName (directory) + "/" + Path.GetFileName (resource.ItemSpec); var archivePath = "res/" + resourcePath; - aar.AddStream (File.OpenRead (resource.ItemSpec), archivePath); - existingEntries.Remove (archivePath); + using var input = File.OpenRead (resource.ItemSpec); + aar.AddStream (input, archivePath); nameCaseMap.Append (resource.GetMetadata ("LogicalName").Replace ('\\', '/')); nameCaseMap.Append (';'); nameCaseMap.AppendLine (resourcePath); } if (nameCaseMap.Length > 0) { - var archivePath = ".net/__res_name_case_map.txt"; - aar.AddEntry (archivePath, nameCaseMap.ToString (), Files.UTF8withoutBOM); - existingEntries.Remove (archivePath); + aar.AddEntry (".net/__res_name_case_map.txt", nameCaseMap.ToString (), Files.UTF8withoutBOM); } } if (AndroidEnvironment != null) { foreach (var env in AndroidEnvironment) { var archivePath = $".net/env/{GetHashedFileName (env)}.env"; - aar.AddStream (File.OpenRead (env.ItemSpec), archivePath); - existingEntries.Remove (archivePath); + using var input = File.OpenRead (env.ItemSpec); + aar.AddStream (input, archivePath); } } if (JarFiles != null) { @@ -106,8 +98,8 @@ public override bool RunTask () continue; } var archivePath = $"libs/{GetHashedFileName (jar)}.jar"; - aar.AddStream (File.OpenRead (jar.ItemSpec), archivePath); - existingEntries.Remove (archivePath); + using var input = File.OpenRead (jar.ItemSpec); + aar.AddStream (input, archivePath); } } if (NativeLibraries != null) { @@ -118,8 +110,8 @@ public override bool RunTask () continue; } var archivePath = "jni/" + abi + "/" + Path.GetFileName (lib.ItemSpec); - aar.AddStream (File.OpenRead (lib.ItemSpec), archivePath); - existingEntries.Remove (archivePath); + using var input = File.OpenRead (lib.ItemSpec); + aar.AddStream (input, archivePath); } } if (ProguardConfigurationFiles != null) { @@ -138,10 +130,6 @@ public override bool RunTask () Log.LogDebugMessage ($"Skipping {AndroidManifest.ItemSpec}. The `manifest` does not have a `package` attribute."); } } - foreach (var entry in existingEntries) { - Log.LogDebugMessage ($"Removing {entry} as it is not longer required."); - aar.DeleteEntry (entry); - } } // Delete the archive on failure diff --git a/src/Xamarin.Android.Build.Tasks/Tasks/ExtractJarsFromAar.cs b/src/Xamarin.Android.Build.Tasks/Tasks/ExtractJarsFromAar.cs index 7d34a66016b..03488ac8d79 100644 --- a/src/Xamarin.Android.Build.Tasks/Tasks/ExtractJarsFromAar.cs +++ b/src/Xamarin.Android.Build.Tasks/Tasks/ExtractJarsFromAar.cs @@ -2,9 +2,9 @@ using System; using System.Collections.Generic; using System.IO; +using System.IO.Compression; using Microsoft.Build.Framework; using Xamarin.Android.Tools; -using Xamarin.Tools.Zip; using Microsoft.Android.Build.Tasks; namespace Xamarin.Android.Tasks @@ -55,8 +55,8 @@ void ExtractLibraries (string []? libraries, string outputJarsDirectory, string var jarOutputDirectory = Path.Combine (outputJarsDirectory, Path.GetFileName (library)); var annotationOutputDirectory = Path.Combine (outputAnnotationsDirectory, Path.GetFileName (library)); using (var zip = MonoAndroidHelper.ReadZipFile (library)) { - foreach (var entry in zip) { - if (entry.IsDirectory) + foreach (var entry in zip.Entries) { + if (entry.IsDirectory ()) continue; var entryFullName = entry.FullName; var fileName = Path.GetFileName (entryFullName); @@ -94,7 +94,7 @@ bool IsUnderDirectory (string resolvedPath, string targetDirectory, string entry return false; } - static void Extract (ZipEntry entry, MemoryStream stream, string destination) + static void Extract (ZipArchiveEntry entry, MemoryStream stream, string destination) { stream.SetLength (0); //Reuse the stream entry.Extract (stream); diff --git a/src/Xamarin.Android.Build.Tasks/Tasks/GenerateResourceCaseMap.cs b/src/Xamarin.Android.Build.Tasks/Tasks/GenerateResourceCaseMap.cs index 4c417583d12..25e6e5b4532 100644 --- a/src/Xamarin.Android.Build.Tasks/Tasks/GenerateResourceCaseMap.cs +++ b/src/Xamarin.Android.Build.Tasks/Tasks/GenerateResourceCaseMap.cs @@ -9,7 +9,6 @@ using Microsoft.Build.Utilities; using Microsoft.Android.Build.Tasks; using Xamarin.Android.Tools; -//using Xamarin.Tools.Zip; using System.IO.Compression; namespace Xamarin.Android.Tasks diff --git a/src/Xamarin.Android.Build.Tasks/Tasks/Javac.cs b/src/Xamarin.Android.Build.Tasks/Tasks/Javac.cs index 4dfbfa17629..89ebcde2c97 100644 --- a/src/Xamarin.Android.Build.Tasks/Tasks/Javac.cs +++ b/src/Xamarin.Android.Build.Tasks/Tasks/Javac.cs @@ -5,11 +5,11 @@ using System; using System.Linq; using System.IO; +using System.IO.Compression; using Microsoft.Build.Framework; using Microsoft.Build.Utilities; using System.Text; using System.Collections.Generic; -using Xamarin.Tools.Zip; using Xamarin.Android.Tools; using Microsoft.Android.Build.Tasks; @@ -42,10 +42,10 @@ public override bool RunTask () return result; // compress all the class files if (!ClassesZip.IsNullOrEmpty ()) { - using (var zip = new ZipArchiveEx (ClassesZip, FileMode.OpenOrCreate)) { - zip.AutoFlush = false; - zip.AddDirectory (ClassesOutputDirectory, "", CompressionMethod.Store); - } + Files.ArchiveZip (ClassesZip, archivePath => { + using var zip = ZipArchiveExtensions.OpenZip (archivePath, FileMode.Create); + zip.AddDirectory (ClassesOutputDirectory, compressionLevel: CompressionLevel.NoCompression); + }); } return result; } diff --git a/src/Xamarin.Android.Build.Tasks/Tasks/ResolveLibraryProjectImports.cs b/src/Xamarin.Android.Build.Tasks/Tasks/ResolveLibraryProjectImports.cs index bb6ceac5dd8..22df17a3412 100644 --- a/src/Xamarin.Android.Build.Tasks/Tasks/ResolveLibraryProjectImports.cs +++ b/src/Xamarin.Android.Build.Tasks/Tasks/ResolveLibraryProjectImports.cs @@ -2,14 +2,14 @@ using System; using System.Collections.Generic; using System.IO; +using System.IO.Compression; using System.Linq; +using System.Reflection.Metadata; +using System.Reflection.PortableExecutable; using System.Xml.Linq; using Microsoft.Build.Utilities; using Microsoft.Build.Framework; -using Xamarin.Tools.Zip; using Xamarin.Android.Tools; -using System.Reflection.Metadata; -using System.Reflection.PortableExecutable; using Microsoft.Android.Build.Tasks; namespace Xamarin.Android.Tasks @@ -105,7 +105,7 @@ public override bool RunTask () assemblyMap.Load (AssemblyIdentityMapFile); try { Extract (jars, resolvedResourceDirectories, resolvedAssetDirectories, resolvedEnvironmentFiles, proguardConfigFiles, extractedDirectories); - } catch (ZipIOException ex) { + } catch (InvalidDataException ex) { Log.LogCodedError ("XA1004", ex.Message); Log.LogDebugMessage (ex.ToString ()); } @@ -277,7 +277,7 @@ void Extract ( else if (name == "__AndroidNativeLibraries__.zip") { List files = new List (); using (var stream = pe.GetEmbeddedResourceStream (resource)) - using (var zip = Xamarin.Tools.Zip.ZipArchive.Open (stream)) { + using (var zip = ZipArchiveExtensions.OpenZip (stream, ZipArchiveMode.Read, leaveOpen: false)) { try { updated |= Files.ExtractAll (zip, nativeimportsDir, modifyCallback: (entryFullName) => { files.Add (Path.GetFullPath (Path.Combine (nativeimportsDir, entryFullName))); @@ -301,7 +301,7 @@ void Extract ( // temporarily extracted directory will look like: // __library_projects__/[dllname]/[library_project_imports | jlibs]/bin using (var stream = pe.GetEmbeddedResourceStream (resource)) - using (var zip = Xamarin.Tools.Zip.ZipArchive.Open (stream)) { + using (var zip = ZipArchiveExtensions.OpenZip (stream, ZipArchiveMode.Read, leaveOpen: false)) { try { updated |= Files.ExtractAll (zip, importsDir, modifyCallback: (entryFullName) => { var path = entryFullName @@ -498,11 +498,9 @@ void Extract ( void CreateResourceArchive (string resDir, string outputFile) { - var fileMode = File.Exists (outputFile) ? FileMode.Open : FileMode.CreateNew; - Files.ArchiveZipUpdate (outputFile, f => { - using (var zip = new ZipArchiveEx (f, fileMode)) { - zip.AddDirectory (resDir, "res"); - } + Files.ArchiveZip (outputFile, archivePath => { + using var zip = ZipArchiveExtensions.OpenZip (archivePath, FileMode.Create); + zip.AddDirectory (resDir, "res"); }); } diff --git a/src/Xamarin.Android.Build.Tasks/Tasks/UnzipToFolder.cs b/src/Xamarin.Android.Build.Tasks/Tasks/UnzipToFolder.cs index 758f2e6cd57..bd548c7d988 100644 --- a/src/Xamarin.Android.Build.Tasks/Tasks/UnzipToFolder.cs +++ b/src/Xamarin.Android.Build.Tasks/Tasks/UnzipToFolder.cs @@ -1,10 +1,10 @@ #nullable enable using System; using System.IO; +using System.IO.Compression; using System.Linq; using Microsoft.Build.Framework; using Microsoft.Build.Utilities; -using Xamarin.Tools.Zip; using Microsoft.Android.Build.Tasks; namespace Xamarin.Android.Tasks @@ -22,12 +22,12 @@ public override bool RunTask () foreach (var pair in Sources.Zip (DestinationDirectories, (s, d) => new { Source = s, Destination = d })) { if (!Directory.Exists (pair.Destination.ItemSpec)) Directory.CreateDirectory (pair.Destination.ItemSpec); - using (var z = ZipArchive.Open (pair.Source.ItemSpec, FileMode.Open)) { + using (var z = ZipArchiveExtensions.OpenZip (pair.Source.ItemSpec, FileMode.Open)) { if (Files == null || Files.Length == 0) { - z.ExtractAll (pair.Destination.ItemSpec); + Microsoft.Android.Build.Tasks.Files.ExtractAll (z, pair.Destination.ItemSpec); } else { foreach (var file in Files) { - ZipEntry entry = z.ReadEntry (file.ItemSpec); + var entry = z.ReadEntry (file.ItemSpec, StringComparison.Ordinal); if (entry == null) { Log.LogDebugMessage ($"Skipping not existant file {file.ItemSpec}"); continue; diff --git a/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/AndroidUpdateResourcesTest.cs b/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/AndroidUpdateResourcesTest.cs index e206f1deb68..014a8c7b56e 100644 --- a/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/AndroidUpdateResourcesTest.cs +++ b/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/AndroidUpdateResourcesTest.cs @@ -558,7 +558,7 @@ protected override void OnClick() } } - void CheckCustomView (Xamarin.Tools.Zip.ZipArchive zip, params string [] paths) + void CheckCustomView (System.IO.Compression.ZipArchive zip, params string [] paths) { var customViewPath = Path.Combine (paths); FileAssert.Exists (customViewPath, $"custom_text.xml should exist at {customViewPath}"); @@ -571,11 +571,13 @@ void CheckCustomView (Xamarin.Tools.Zip.ZipArchive zip, params string [] paths) //Now check the zip var customViewInZip = "res/layout/" + Path.GetFileName (customViewPath); - var entry = zip.ReadEntry (customViewInZip); + var entry = zip.GetEntry (customViewInZip); Assert.IsNotNull (entry, $"`{customViewInZip}` should exist in packaged_resources!"); using (var stream = new MemoryStream ()) { - entry.Extract (stream); + using (var entryStream = entry.Open ()) { + entryStream.CopyTo (stream); + } stream.Position = 0; using (var reader = new StreamReader (stream)) { diff --git a/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/BindingBuildTest.cs b/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/BindingBuildTest.cs index 7e3cf931004..7fc016445da 100644 --- a/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/BindingBuildTest.cs +++ b/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/BindingBuildTest.cs @@ -3,13 +3,13 @@ using System; using System.Collections.Generic; using System.IO; +using System.IO.Compression; using System.Linq; using System.Text; using System.Xml.Linq; using Xamarin.ProjectTools; using Microsoft.Android.Build.Tasks; using Microsoft.Build.Framework; -using Xamarin.Tools.Zip; using Xamarin.Android.Tasks; namespace Xamarin.Android.Build.Tests @@ -547,13 +547,13 @@ public void AndroidLibraryPackMetadataIsPreservedInNuGet ([Values (AndroidRuntim var nupkgPath = Path.Combine (Root, bindingBuilder.ProjectDirectory, binding.OutputPath, "UnnamedProject.1.0.0.nupkg"); FileAssert.Exists (nupkgPath); - using (var nupkg = ZipArchive.Open (nupkgPath, FileMode.Open)) { - var aarEntry = nupkg.Single (entry => entry.FullName.EndsWith ("/UnnamedProject.aar", StringComparison.Ordinal)); + using (var nupkg = ZipFile.OpenRead (nupkgPath)) { + var aarEntry = nupkg.Entries.Single (entry => entry.FullName.EndsWith ("/UnnamedProject.aar", StringComparison.Ordinal)); using var aarStream = new MemoryStream (); aarEntry.Extract (aarStream); aarStream.Position = 0; - using var aar = ZipArchive.Open (aarStream); - Assert.AreEqual (1, aar.Count (entry => entry.FullName.StartsWith ("libs/", StringComparison.Ordinal) && entry.FullName.EndsWith (".jar", StringComparison.Ordinal)), + using var aar = new ZipArchive (aarStream, ZipArchiveMode.Read, leaveOpen: true); + Assert.AreEqual (1, aar.Entries.Count (entry => entry.FullName.StartsWith ("libs/", StringComparison.Ordinal) && entry.FullName.EndsWith (".jar", StringComparison.Ordinal)), "The generated AAR should contain only the Bind='false', Pack='true' JAR."); } } @@ -629,7 +629,7 @@ public JavaSourceJarTest echo (JavaSourceJarTest value) { static byte [] CreateAar (byte [] classesJar) { using var stream = new MemoryStream (); - using (var aar = ZipArchive.Open (stream)) { + using (var aar = new ZipArchive (stream, ZipArchiveMode.Create, leaveOpen: true)) { aar.AddStream (new MemoryStream (classesJar), "classes.jar"); } return stream.ToArray (); diff --git a/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/BuildTest.cs b/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/BuildTest.cs index 63a575f42a2..79806220dda 100644 --- a/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/BuildTest.cs +++ b/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/BuildTest.cs @@ -1811,9 +1811,9 @@ public void KotlinServiceLoader ([Values ("apk", "aab")] string packageFormat, [ }; var manifest = prefix + "META-INF/MANIFEST.MF"; using (var zip = ZipHelper.OpenZip (archive)) { - Assert.IsFalse (zip.ContainsEntry (manifest, caseSensitive: true), $"{manifest} should *not* exist in {archive}"); + Assert.IsFalse (zip.ContainsEntry (manifest, StringComparison.Ordinal), $"{manifest} should *not* exist in {archive}"); foreach (var expected in expectedFiles) { - Assert.IsTrue (zip.ContainsEntry (expected, caseSensitive: true), $"{expected} should exist in {archive}"); + Assert.IsTrue (zip.ContainsEntry (expected, StringComparison.Ordinal), $"{expected} should exist in {archive}"); } } } diff --git a/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/BuildTest2.cs b/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/BuildTest2.cs index f20b791a80d..ef7742d9980 100644 --- a/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/BuildTest2.cs +++ b/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/BuildTest2.cs @@ -672,8 +672,8 @@ public void ClassLibraryHasNoWarnings ([Values (AndroidRuntime.CoreCLR, AndroidR // $(AndroidEnableMultiDex) should not add android-support-multidex.jar! var aarPath = Path.Combine (Root, b.ProjectDirectory, proj.OutputPath, $"{proj.ProjectName}.aar"); - using var zip = Xamarin.Tools.Zip.ZipArchive.Open (aarPath, FileMode.Open); - Assert.IsFalse (zip.Any (e => e.FullName.EndsWith (".jar", StringComparison.OrdinalIgnoreCase)), + using var zip = System.IO.Compression.ZipFile.OpenRead (aarPath); + Assert.IsFalse (zip.Entries.Any (e => e.FullName.EndsWith (".jar", StringComparison.OrdinalIgnoreCase)), $"{aarPath} should not contain a .jar file!"); } } diff --git a/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/GlobalUsings.cs b/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/GlobalUsings.cs new file mode 100644 index 00000000000..913a3038f2e --- /dev/null +++ b/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/GlobalUsings.cs @@ -0,0 +1 @@ +global using Microsoft.Android.Build.Tasks; diff --git a/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/ManifestTest.TestCaseSource.cs b/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/ManifestTest.TestCaseSource.cs index db90ab95de1..c4015f422aa 100644 --- a/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/ManifestTest.TestCaseSource.cs +++ b/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/ManifestTest.TestCaseSource.cs @@ -6,7 +6,6 @@ using System.Xml; using System.Xml.Linq; using System.Xml.XPath; -using Xamarin.Tools.Zip; using Xamarin.Android.Tasks; using System.Collections.Generic; diff --git a/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/ManifestTest.cs b/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/ManifestTest.cs index 0f5d6141d80..8c93bf97ab2 100644 --- a/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/ManifestTest.cs +++ b/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/ManifestTest.cs @@ -1,6 +1,7 @@ using System; using System.Collections.Generic; using System.IO; +using System.IO.Compression; using System.Linq; using System.Xml; using System.Xml.Linq; @@ -13,7 +14,6 @@ using Xamarin.Android.Tasks; using Xamarin.Android.Tools; using Xamarin.ProjectTools; -using Xamarin.Tools.Zip; using PropertyAttribute = Android.App.PropertyAttribute; namespace Xamarin.Android.Build.Tests @@ -890,7 +890,7 @@ public void MergeLibraryManifest ([Values (AndroidRuntime.CoreCLR, AndroidRuntim byte [] classesJar = XamarinAndroidCommonProject.GetResourceContents ("Xamarin.ProjectTools.Resources.Base.classes.jar"); byte [] data; using (var ms = new MemoryStream ()) { - using (var zip = ZipArchive.Create (ms)) { + using (var zip = new ZipArchive (ms, ZipArchiveMode.Create, leaveOpen: true)) { zip.AddEntry ("AndroidManifest.xml", @" @@ -906,8 +906,10 @@ public void MergeLibraryManifest ([Values (AndroidRuntime.CoreCLR, AndroidRuntim ", encoding: System.Text.Encoding.UTF8); - zip.CreateDirectory ("res"); - zip.AddEntry (classesJar, "classes.jar"); + zip.CreateEntry ("res/"); + using (var classesJarStream = new MemoryStream (classesJar)) { + zip.AddStream (classesJarStream, "classes.jar"); + } zip.AddEntry ("R.txt", " ", encoding: System.Text.Encoding.UTF8); } data = ms.ToArray (); diff --git a/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/PackagingTest.cs b/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/PackagingTest.cs index ee0ff31d17e..e40b592d286 100644 --- a/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/PackagingTest.cs +++ b/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/PackagingTest.cs @@ -1,15 +1,16 @@ using System; using System.Collections.Generic; using System.IO; +using System.IO.Compression; using System.Linq; using System.Text; using System.Xml.Linq; using Microsoft.Build.Framework; +using Microsoft.Android.Build.Tasks; using NUnit.Framework; using Xamarin.Android.Tasks; using Xamarin.Android.Tools; using Xamarin.ProjectTools; -using Xamarin.Tools.Zip; namespace Xamarin.Android.Build.Tests { @@ -259,13 +260,14 @@ public void CheckIncludedNativeLibraries ([Values] bool compressNativeLibraries, Assert.IsTrue (b.Build (proj), "build failed"); var apk = Path.Combine (Root, b.ProjectDirectory, proj.OutputPath, $"{proj.PackageName}-Signed.apk"); - CompressionMethod method = compressNativeLibraries ? CompressionMethod.Deflate : CompressionMethod.Store; + var metadata = ZipArchiveMetadataReader.Read (apk); + var expectedCompression = compressNativeLibraries ? ZipEntryCompressionMethod.Deflate : ZipEntryCompressionMethod.Store; using (var zip = ZipHelper.OpenZip (apk)) { - var libFiles = zip.Where (x => x.FullName.StartsWith("lib/", StringComparison.Ordinal) && !x.FullName.Equals("lib/", StringComparison.InvariantCultureIgnoreCase)); + var libFiles = zip.Entries.Where (x => x.FullName.StartsWith("lib/", StringComparison.Ordinal) && !x.FullName.Equals("lib/", StringComparison.InvariantCultureIgnoreCase)); var abiPaths = new string[] { "lib/x86_64/" }; foreach (var file in libFiles) { Assert.IsTrue (abiPaths.Any (x => file.FullName.Contains (x)), $"Apk contains an unnesscary lib file: {file.FullName}"); - Assert.IsTrue (file.CompressionMethod == method, $"{file.FullName} should have been CompressionMethod.{method} in the apk, but was CompressionMethod.{file.CompressionMethod}"); + Assert.AreEqual (expectedCompression, metadata [file.FullName].CompressionMethod, $"{file.FullName} should have been {expectedCompression} in the apk."); } } } @@ -316,22 +318,23 @@ void AssertEmbeddedDSOs (string apk) Assert.That (RunCommand (zipAlignPath, $"-c -v -p 4 {apk}"), Is.True, $"{apk} does not contain page-aligned .so files"); using (var zip = ZipHelper.OpenZip (apk)) { - foreach (var entry in zip) { + foreach (var entry in zip.Entries) { if (entry.FullName.EndsWith (".so", StringComparison.Ordinal)) { - AssertCompression (entry, compressed: false); + AssertCompression (apk, entry, compressed: false); } } } } - void AssertCompression (ZipEntry entry, bool compressed) + void AssertCompression (string archivePath, ZipArchiveEntry entry, bool compressed) { + var compressionMethod = ZipArchiveMetadataReader.Read (archivePath) [entry.FullName].CompressionMethod; if (compressed) { - Assert.AreNotEqual (CompressionMethod.Store, entry.CompressionMethod, $"`{entry.FullName}` should be compressed!"); - Assert.AreNotEqual (entry.Size, entry.CompressedSize, $"`{entry.FullName}` should be compressed!"); + Assert.AreNotEqual (ZipEntryCompressionMethod.Store, compressionMethod, $"`{entry.FullName}` should be compressed!"); + Assert.AreNotEqual (entry.Length, entry.CompressedLength, $"`{entry.FullName}` should be compressed!"); } else { - Assert.AreEqual (CompressionMethod.Store, entry.CompressionMethod, $"`{entry.FullName}` should be uncompressed!"); - Assert.AreEqual (entry.Size, entry.CompressedSize, $"`{entry.FullName}` should be uncompressed!"); + Assert.AreEqual (ZipEntryCompressionMethod.Store, compressionMethod, $"`{entry.FullName}` should be uncompressed!"); + Assert.AreEqual (entry.Length, entry.CompressedLength, $"`{entry.FullName}` should be uncompressed!"); } } @@ -362,7 +365,7 @@ public void IncrementalCompression ([Values (AndroidRuntime.CoreCLR, AndroidRunt using (var zip = ZipHelper.OpenZip (apk)) { foreach (var entry in zip) { if (entry.FullName.EndsWith (".so", StringComparison.Ordinal) || entry.FullName.EndsWith (".bar", StringComparison.Ordinal)) { - AssertCompression (entry, compressed: true); + AssertCompression (apk, entry, compressed: true); } } } @@ -379,7 +382,7 @@ public void IncrementalCompression ([Values (AndroidRuntime.CoreCLR, AndroidRunt using (var zip = ZipHelper.OpenZip (apk)) { foreach (var entry in zip) { if (entry.FullName.EndsWith (".so", StringComparison.Ordinal) || entry.FullName.EndsWith (".bar", StringComparison.Ordinal)) { - AssertCompression (entry, compressed: false); + AssertCompression (apk, entry, compressed: false); } } } @@ -900,58 +903,6 @@ public void CheckIncludedFilesArePresent ([Values (AndroidRuntime.CoreCLR, Andro } } - static IEnumerable Get_BuildApkWithZipFlushLimits_Data () - { - var ret = new List (); - - foreach (AndroidRuntime runtime in new[] { AndroidRuntime.CoreCLR, AndroidRuntime.NativeAOT }) { - AddTestData (1, -1, runtime); - AddTestData (5, -1, runtime); - AddTestData (50, -1, runtime); - AddTestData (100, -1, runtime); - AddTestData (512, -1, runtime); - AddTestData (1024, -1, runtime); - AddTestData (-1, 1, runtime); - AddTestData (-1, 5, runtime); - AddTestData (-1, 10, runtime); - AddTestData (-1, 100, runtime); - AddTestData (-1, 200, runtime); - } - - return ret; - - void AddTestData (int filesLimit, int sizeLimit, AndroidRuntime runtime) - { - ret.Add (new object[] { - filesLimit, - sizeLimit, - runtime, - }); - } - } - - [Test] - [TestCaseSource (nameof (Get_BuildApkWithZipFlushLimits_Data))] - public void BuildApkWithZipFlushLimits (int filesLimit, int sizeLimit, AndroidRuntime runtime) - { - const bool isRelease = false; - if (IgnoreUnsupportedConfiguration (runtime, release: isRelease)) { - return; - } - var proj = new XamarinFormsAndroidApplicationProject { - IsRelease = isRelease, - }; - proj.SetRuntime (runtime); - proj.SetProperty ("EmbedAssembliesIntoApk", "true"); - if (filesLimit > 0) - proj.SetProperty ("_ZipFlushFilesLimit", filesLimit.ToString ()); - if (sizeLimit > 0) - proj.SetProperty ("_ZipFlushSizeLimit", (sizeLimit * 1024 * 1024).ToString ()); - using (var b = CreateApkBuilder ()) { - Assert.IsTrue (b.Build (proj), "Build should have succeeded."); - - } - } [Test] public void ExtractNativeLibsTrue ([Values (AndroidRuntime.CoreCLR, AndroidRuntime.NativeAOT)] AndroidRuntime runtime) @@ -982,7 +933,7 @@ public void ExtractNativeLibsTrue ([Values (AndroidRuntime.CoreCLR, AndroidRunti using (var zip = ZipHelper.OpenZip (apk)) { foreach (var entry in zip) { if (entry.FullName.EndsWith (".so", StringComparison.Ordinal)) { - AssertCompression (entry, compressed: true); + AssertCompression (apk, entry, compressed: true); } } } diff --git a/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/Tasks/Aapt2Tests.cs b/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/Tasks/Aapt2Tests.cs index b9443fd44af..56ca2f5b2c8 100644 --- a/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/Tasks/Aapt2Tests.cs +++ b/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/Tasks/Aapt2Tests.cs @@ -180,7 +180,7 @@ public void Aapt2Link ([Values (true, false)] bool compilePerFile) Assert.LessOrEqual (0, warnings.Count, "There should be 0 warnings."); Assert.True (File.Exists (outputFile), $"{outputFile} should have been created."); using (var apk = ZipHelper.OpenZip (outputFile)) { - Assert.AreEqual (3, apk.EntryCount, $"{outputFile} should have 3 entries."); + Assert.AreEqual (3, apk.Entries.Count, $"{outputFile} should have 3 entries."); } Directory.Delete (Path.Combine (Root, path), recursive: true); } @@ -218,7 +218,7 @@ public void Aapt2Compile () var flatArchive = Path.Combine (archivePath, "compiled.flata"); Assert.True (File.Exists (flatArchive), $"{flatArchive} should have been created."); using (var apk = ZipHelper.OpenZip (flatArchive)) { - Assert.AreEqual (2, apk.EntryCount, $"{flatArchive} should have 2 entries."); + Assert.AreEqual (2, apk.Entries.Count, $"{flatArchive} should have 2 entries."); } Directory.Delete (Path.Combine (Root, path), recursive: true); } diff --git a/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/Tasks/BuildArchiveTests.cs b/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/Tasks/BuildArchiveTests.cs index 7af60ef932c..f4a2a7fcdc3 100644 --- a/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/Tasks/BuildArchiveTests.cs +++ b/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/Tasks/BuildArchiveTests.cs @@ -2,14 +2,14 @@ using System; using System.Collections.Generic; using System.IO; +using System.IO.Compression; using System.Linq; using System.Text; using Microsoft.Android.Build.Tasks; using Microsoft.Build.Framework; using Microsoft.Build.Utilities; using NUnit.Framework; -using Xamarin.Android.Tasks; -using Xamarin.Tools.Zip; +using Microsoft.Android.Tasks; namespace Xamarin.Android.Build.Tests; @@ -30,7 +30,7 @@ public void Setup () [TearDown] public void TearDown () { - if (!tempDirectory.IsNullOrEmpty () && Directory.Exists (tempDirectory)) + if (!string.IsNullOrEmpty (tempDirectory) && Directory.Exists (tempDirectory)) Directory.Delete (tempDirectory, recursive: true); } @@ -62,7 +62,7 @@ public void ConsecutiveUnchangedBuildsKeepJavaArchiveEntries () Assert.AreEqual (previousSnapshot, snapshot, $"build {build} should match the previous unchanged build"); previousSnapshot = snapshot; - using (var archive = ZipArchive.Open (apk, FileMode.Open)) { + using (var archive = ZipFile.OpenRead (apk)) { archive.AssertEntryContents (apk, "commonMain/default/manifest", "current"); archive.AssertDoesNotContainEntry (apk, "stale.txt"); } @@ -93,7 +93,7 @@ public void ExistingJavaArchiveEntriesAreSkippedWhenUpToDate () Assert.That (messages, Has.Some.Property (nameof (BuildMessageEventArgs.Message)).EqualTo ($"Skipping commonMain/default/manifest from {jar} as it is up to date.")); - using (var archive = ZipArchive.Open (apk, FileMode.Open)) { + using (var archive = ZipFile.OpenRead (apk)) { archive.AssertEntryContents (apk, "commonMain/default/manifest", "current"); } } @@ -127,7 +127,7 @@ public void DuplicateJavaArchiveEntriesKeepFirstCurrentBuildItem () Assert.That (messages, Has.Some.Property (nameof (BuildMessageEventArgs.Message)).EqualTo ("Failed to add jar entry commonMain/default/manifest from second.jar: the same file already exists in the apk")); - using (var archive = ZipArchive.Open (apk, FileMode.Open)) { + using (var archive = ZipFile.OpenRead (apk)) { archive.AssertEntryContents (apk, "commonMain/default/manifest", "first"); archive.AssertDoesNotContainEntry (apk, "stale.txt"); } @@ -159,30 +159,96 @@ public void MissingJarEntryIsSkippedAndExistingOutputEntryIsRemoved () // The entry should be removed. If the APK itself no longer exists, all entries were cleared (also satisfies the assertion). if (File.Exists (apk)) { - using (var archive = ZipArchive.Open (apk, FileMode.Open)) { + using (var archive = ZipFile.OpenRead (apk)) { archive.AssertDoesNotContainEntry (apk, "commonMain/default/manifest"); } } } - static void CreateArchive (string path, params (string name, string contents) [] entries) + [Test] + public void StoredBundleManifestRelocationPreservesCompressionMethod () + { + var bundle = Path.Combine (TempDirectory, "app.aab"); + + CreateArchive (bundle, ("AndroidManifest.xml", "manifest", CompressionLevel.NoCompression)); + + var task = new BuildArchive { + BuildEngine = new MockBuildEngine (TestContext.Out), + AndroidPackageFormat = "aab", + ApkOutputPath = bundle, + FilesToAddToArchive = [], + }; + + Assert.IsTrue (task.RunTask (), "task should have succeeded"); + + var metadata = ZipArchiveMetadataReader.Read (bundle); + Assert.IsFalse (metadata.ContainsKey ("AndroidManifest.xml"), "Original manifest entry should be moved."); + Assert.AreEqual (ZipEntryCompressionMethod.Store, metadata ["manifest/AndroidManifest.xml"].CompressionMethod, "Moved manifest should stay stored."); + } + + [Test] + public void ZeroByteStoredFileStabilizesAcrossBuilds () + { + var apk = Path.Combine (TempDirectory, "app.apk"); + var emptyFile = Path.Combine (TempDirectory, "empty.dat"); + File.WriteAllBytes (emptyFile, []); + + var item = new TaskItem (emptyFile); + item.SetMetadata ("ArchivePath", "empty.dat"); + var secondRunMessages = new List (); + + var firstRun = new BuildArchive { + BuildEngine = new MockBuildEngine (TestContext.Out), + ApkOutputPath = apk, + FilesToAddToArchive = [item], + UncompressedFileExtensions = ".dat", + }; + Assert.IsTrue (firstRun.RunTask (), "first build should have succeeded"); + + var firstSnapshot = GetArchiveSnapshot (apk); + Assert.AreEqual (ZipEntryCompressionMethod.Store, ZipArchiveMetadataReader.Read (apk) ["empty.dat"].CompressionMethod, "Entry should be stored."); + + var secondRun = new BuildArchive { + BuildEngine = new MockBuildEngine (TestContext.Out, messages: secondRunMessages), + ApkOutputPath = apk, + FilesToAddToArchive = [item], + UncompressedFileExtensions = ".dat", + }; + Assert.IsTrue (secondRun.RunTask (), "second build should have succeeded"); + + var secondSnapshot = GetArchiveSnapshot (apk); + Assert.AreEqual (firstSnapshot, secondSnapshot, "Archive contents should be stable across unchanged builds."); + Assert.That (secondRunMessages, Has.Some.Property (nameof (BuildMessageEventArgs.Message)).EqualTo ($"Skipping {emptyFile} as the archive file is up to date.")); + } + + static void CreateArchive (string path, params (string name, string contents, CompressionLevel compressionLevel) [] entries) { - using (var stream = File.Create (path)) - using (var archive = ZipArchive.Create (stream)) { + using (var stream = new FileStream (path, FileMode.Create, FileAccess.ReadWrite)) + using (var archive = new ZipArchive (stream, ZipArchiveMode.Create)) { foreach (var entry in entries) { - archive.AddEntry (entry.name, entry.contents, encoding: Encoding.UTF8); + var zipEntry = archive.CreateEntry (entry.name, entry.compressionLevel); + using (var writer = new StreamWriter (zipEntry.Open (), Encoding.UTF8)) { + writer.Write (entry.contents); + } } } } + static void CreateArchive (string path, params (string name, string contents) [] entries) + { + CreateArchive (path, entries.Select (entry => (entry.name, entry.contents, CompressionLevel.Optimal)).ToArray ()); + } + static string GetArchiveSnapshot (string path) { - using var archive = ZipArchive.Open (path, FileMode.Open); - return string.Join ("\n", archive + using var archive = ZipFile.OpenRead (path); + return string.Join ("\n", archive.Entries .OrderBy (entry => entry.FullName, StringComparer.Ordinal) .Select (entry => { using var stream = new MemoryStream (); - entry.Extract (stream); + using (var entryStream = entry.Open ()) { + entryStream.CopyTo (stream); + } return $"{entry.FullName}:{Convert.ToBase64String (stream.ToArray ())}"; })); } diff --git a/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/Tasks/FilterAssembliesTests.cs b/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/Tasks/FilterAssembliesTests.cs index 3f1cae67499..2fa8414fb50 100644 --- a/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/Tasks/FilterAssembliesTests.cs +++ b/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/Tasks/FilterAssembliesTests.cs @@ -2,11 +2,11 @@ using System; using System.Collections.Generic; using System.IO; +using System.IO.Compression; using System.Linq; using System.Threading.Tasks; using Xamarin.Android.Tasks; using Xamarin.ProjectTools; -using Xamarin.Tools.Zip; using TaskItem = Microsoft.Build.Utilities.TaskItem; namespace Xamarin.Android.Build.Tests @@ -52,11 +52,11 @@ async Task GetAssembliesFromNuGet (string url, string filename, strin { var assemblies = new List (); var nuget = await DownloadFromNuGet (url, filename); - using (var zip = ZipArchive.Open (nuget, FileMode.Open)) { - foreach (var entry in zip) { + using (var zip = ZipFile.OpenRead (nuget)) { + foreach (var entry in zip.Entries) { if (entry.FullName.StartsWith (path, StringComparison.OrdinalIgnoreCase) && entry.FullName.EndsWith (".dll", StringComparison.OrdinalIgnoreCase)) { - var temp = Path.Combine (tempDirectory, Path.GetFileName (entry.NativeFullName)); + var temp = Path.Combine (tempDirectory, Path.GetFileName (entry.FullName)); assemblies.Add (temp); using (var fileStream = File.Create (temp)) { entry.Extract (fileStream); diff --git a/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/Utilities/ArchiveAssemblyHelper.cs b/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/Utilities/ArchiveAssemblyHelper.cs index 780a4612a14..12d460a3420 100644 --- a/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/Utilities/ArchiveAssemblyHelper.cs +++ b/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/Utilities/ArchiveAssemblyHelper.cs @@ -2,13 +2,13 @@ using System.Buffers; using System.Collections.Generic; using System.IO; +using System.IO.Compression; using System.Linq; using Xamarin.Android.AssemblyStore; using Xamarin.Android.Tools; using Xamarin.Android.Tasks; using Xamarin.ProjectTools; -using Xamarin.Tools.Zip; namespace Xamarin.Android.Build.Tests { @@ -125,14 +125,18 @@ public ArchiveAssemblyHelper (string archivePath, bool useAssemblyStores = true, } using var zip = ZipHelper.OpenZip (archivePath); + if (zip == null) + return null; foreach (string assemblyPath in potentialEntries) { - if (!zip.ContainsEntry (assemblyPath)) { + var entry = zip.GetEntry (assemblyPath); + if (entry == null) { continue; } - ZipEntry entry = zip.ReadEntry (assemblyPath); var ret = new MemoryStream (); - entry.Extract (ret); + using (var stream = entry.Open ()) { + stream.CopyTo (ret); + } ret.Flush (); return ret; } @@ -192,8 +196,8 @@ public List ListArchiveContents (string storeEntryPrefix = DefaultAssemb } var entries = new List (); - using (var zip = ZipArchive.Open (archivePath, FileMode.Open)) { - foreach (var entry in zip) { + using (var zip = ZipFile.OpenRead (archivePath)) { + foreach (var entry in zip.Entries) { entries.Add (entry.FullName); } } diff --git a/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/Utilities/AssertionExtensions.cs b/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/Utilities/AssertionExtensions.cs index 4edbfc7797e..84906eaddd7 100644 --- a/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/Utilities/AssertionExtensions.cs +++ b/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/Utilities/AssertionExtensions.cs @@ -1,11 +1,11 @@ using System.Collections.Generic; using System.Diagnostics; using System.IO; +using System.IO.Compression; using System.Linq; using NUnit.Framework; using Xamarin.Android.Tasks; using Xamarin.ProjectTools; -using Xamarin.Tools.Zip; namespace Xamarin.Android.Build.Tests { @@ -61,9 +61,12 @@ public static void AssertEntryEquals (this ZipArchive zip, string zipPath, strin { zip.AssertContainsEntry (zipPath, archivePath); - var entry = zip.ReadEntry (archivePath); + var entry = zip.GetEntry (archivePath); + Assert.IsNotNull (entry, $"{zipPath} should contain {archivePath}"); using var stream = new MemoryStream (); - entry.Extract (stream); + using (var entryStream = entry.Open ()) { + entryStream.CopyTo (stream); + } stream.Position = 0; using var reader = new StreamReader (stream); Assert.AreEqual (expected, reader.ReadToEnd ().Trim ()); @@ -72,7 +75,7 @@ public static void AssertEntryEquals (this ZipArchive zip, string zipPath, strin [DebuggerHidden] public static void AssertContainsEntry (this ZipArchive zip, string zipPath, string archivePath) { - Assert.IsTrue (zip.ContainsEntry (archivePath), $"{zipPath} should contain {archivePath}:\n{string.Join (",\n", zip.Select (e => e.FullName))}"); + Assert.IsNotNull (zip.GetEntry (archivePath), $"{zipPath} should contain {archivePath}:\n{string.Join (",\n", zip.Entries.Select (e => e.FullName))}"); } [DebuggerHidden] @@ -84,7 +87,7 @@ public static void AssertContainsEntry (this ArchiveAssemblyHelper helper, strin [DebuggerHidden] public static void AssertDoesNotContainEntry (this ZipArchive zip, string zipPath, string archivePath) { - Assert.IsFalse (zip.ContainsEntry (archivePath), $"{zipPath} should *not* contain {archivePath}"); + Assert.IsNull (zip.GetEntry (archivePath), $"{zipPath} should *not* contain {archivePath}"); } [DebuggerHidden] @@ -117,11 +120,13 @@ public static void AssertContainsEntry (this ArchiveAssemblyHelper helper, strin public static void AssertEntryContents (this ZipArchive zip, string zipPath, string archivePath, string contents) { zip.AssertContainsEntry (zipPath, archivePath); - var entry = zip.ReadEntry (archivePath); + var entry = zip.GetEntry (archivePath); Assert.IsNotNull (entry, $"{zipPath} should contain {archivePath}"); using (var stream = new MemoryStream ()) using (var reader = new StreamReader (stream)) { - entry.Extract (stream); + using (var entryStream = entry.Open ()) { + entryStream.CopyTo (stream); + } stream.Position = 0; var actual = reader.ReadToEnd (); Assert.AreEqual (contents, actual, $"{archivePath} should contain {contents}"); diff --git a/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/XASdkTests.cs b/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/XASdkTests.cs index bb4f44a1fe4..09ae9d6891c 100644 --- a/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/XASdkTests.cs +++ b/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/XASdkTests.cs @@ -1,13 +1,13 @@ using System; using System.Collections.Generic; using System.IO; +using System.IO.Compression; using System.Linq; using System.Reflection; using System.Text; using NUnit.Framework; using Xamarin.Android.Tools; using Xamarin.ProjectTools; -using Xamarin.Tools.Zip; using Microsoft.Android.Build.Tasks; using Xamarin.Android.Tasks; @@ -210,7 +210,7 @@ public String Say (String quote) { aarStream.Seek (0, SeekOrigin.Begin); // Look for the Maven dependency, foo.jar, and the compiled AndroidJavaSource output under libs/ - using var aar = ZipArchive.Open (aarStream); + using var aar = new ZipArchive (aarStream, ZipArchiveMode.Read, leaveOpen: true); int count = aar.Count (e => e.FullName.StartsWith ("libs/", StringComparison.OrdinalIgnoreCase) && e.FullName.EndsWith (".jar", StringComparison.OrdinalIgnoreCase)); diff --git a/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/Xamarin.Android.Build.Tests.csproj b/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/Xamarin.Android.Build.Tests.csproj index b412625aebc..e37ae83ced7 100644 --- a/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/Xamarin.Android.Build.Tests.csproj +++ b/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/Xamarin.Android.Build.Tests.csproj @@ -1,8 +1,7 @@ - $(DotNetStableTargetFramework) - true + net12.0 ..\..\..\..\bin\Test$(Configuration) true ..\..\..\..\product.snk @@ -26,6 +25,7 @@ + diff --git a/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/ZipArchiveExTests.cs b/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/ZipArchiveExTests.cs deleted file mode 100644 index 729991223b7..00000000000 --- a/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/ZipArchiveExTests.cs +++ /dev/null @@ -1,219 +0,0 @@ -using NUnit.Framework; -using System; -using System.Collections.Concurrent; -using System.IO; -using System.Text; -using System.Threading; -using Xamarin.Android.Tasks; -using Xamarin.Tools.Zip; - -namespace Xamarin.Android.Build.Tests -{ - [TestFixture] - public class ZipArchiveExTests - { - ConcurrentDictionary paths = new ConcurrentDictionary (); - string root; - - [SetUp] - public void SetUp () - { - root = Path.GetDirectoryName (GetType ().Assembly.Location); - string temp = Path.Combine (root, "temp", TestContext.CurrentContext.Test.Name); - if (Directory.Exists (temp)) - Directory.Delete (temp, recursive: true); - if (File.Exists (Zip)) - File.Delete (Zip); - Directory.CreateDirectory (temp); - paths.TryAdd (TestContext.CurrentContext.Test.Name, temp); - } - - [TearDown] - public void TearDown () - { - if (TestContext.CurrentContext.Result.Outcome.Status != NUnit.Framework.Interfaces.TestStatus.Passed) { - return; - } - try { - Directory.Delete (TestPath, recursive: true); - } catch (Exception ex) { - TestContext.WriteLine ($"Error deleting '{TestPath}': {ex}"); - } - try { - File.Delete (Zip); - } catch (Exception ex) { - TestContext.WriteLine ($"Error deleting '{Zip}': {ex}"); - } - paths.TryRemove (TestContext.CurrentContext.Test.Name, out _); - } - - public string TestPath { - get { - paths.TryGetValue (TestContext.CurrentContext.Test.Name, out string value); - return value; - } - } - - public string Zip { - get { - return Path.Combine (root, "temp", $"{TestContext.CurrentContext.Test.Name}.zip"); - } - } - - void CreateDirectories (params string [] paths) - { - foreach (var path in paths) { - var dest = Path.Combine (TestPath, path); - Directory.CreateDirectory (Path.GetDirectoryName (dest)); - //Just put the path in the test file, for testing purposes - File.WriteAllText (dest, path); - } - } - - static DateTime WithoutMilliseconds (DateTime t) => - new DateTime (t.Year, t.Month, t.Day, t.Hour, t.Minute, t.Second, t.Kind); - - void AssertZip (string expected) - { - FileAssert.Exists (Zip, "Zip file should exist!"); - - var builder = new StringBuilder (); - using (var archive = new ZipArchiveEx (Zip, FileMode.Open)) { - foreach (var entry in archive.Archive) { - builder.AppendLine (entry.FullName); - } - } - - Assert.AreEqual (expected.Trim ().Replace("\r\n", "\n"), builder.ToString ().Trim ().Replace("\r\n", "\n")); - } - - void AssertSkipExisting (string file, string fileInArchive, bool expected, CompressionMethod method = CompressionMethod.Deflate) - { - DateTime lastWrite = File.GetLastWriteTimeUtc (file); - string path = Path.GetFullPath (file); - using (var archive = new ZipArchiveEx (Zip, FileMode.Open)) { - var entry = archive.Archive.ContainsEntry (fileInArchive) ? archive.Archive.ReadEntry (fileInArchive) : null; - var modTime = entry?.ModificationTime ?? DateTime.MinValue; - var entryMethod = entry?.CompressionMethod; - bool result = archive.SkipExistingFile (file, fileInArchive, method); - Assert.AreEqual (expected, result, - $"SkipExistingFile returned unexpected value for {Zip} {path} {fileInArchive}\n" + - $"{method} != {entryMethod} or\n" + - $"{WithoutMilliseconds(lastWrite)} (disk {lastWrite:MM/dd/yyyy HH:mm:ss:fff}) <= {WithoutMilliseconds (modTime)} (zip {modTime:MM/dd/yyyy HH:mm:ss:fff}) = {result}"); - } - } - - [Test] - public void AddDirectory () - { - CreateDirectories ("A.txt", Path.Combine ("B", "B.txt")); - - using (var archive = new ZipArchiveEx (Zip, FileMode.Create)) { - archive.AddDirectory (TestPath, "temp", CompressionMethod.Deflate); - } - - AssertZip ( -@"temp/A.txt -temp/B/B.txt"); - } - - [Test] - public void AddDirectoryOddPaths () - { - CreateDirectories ("A.txt", Path.Combine ("B", "B.txt")); - - using (var archive = new ZipArchiveEx (Zip, FileMode.Create)) { - archive.AddDirectory (TestPath + $@"\../{nameof (AddDirectoryOddPaths)}/", "temp", CompressionMethod.Deflate); - } - - AssertZip ( -@"temp/A.txt -temp/B/B.txt"); - } - - [Test] - public void AddDirectoryCurrentDirectory () - { - CreateDirectories ("A.txt", Path.Combine ("B", "B.txt")); - - string cwd = Directory.GetCurrentDirectory (); - try { - Directory.SetCurrentDirectory (TestPath); - - using (var archive = new ZipArchiveEx (Zip, FileMode.Create)) { - archive.AddDirectory (".", "temp", CompressionMethod.Deflate); - } - - AssertZip ( -@"temp/A.txt -temp/B/B.txt"); - } finally { - Directory.SetCurrentDirectory (cwd); - } - } - - [Test] - [Repeat(100)] - public void SkipExistingFile_Exist () - { - CreateDirectories ("A.txt", Path.Combine ("B", "B.txt")); - - using (var archive = new ZipArchiveEx (Zip, FileMode.Create)) { - archive.AddDirectory (TestPath, folderInArchive: "", CompressionMethod.Deflate); - } - - AssertZip ( -@"A.txt -B/B.txt"); - AssertSkipExisting (Path.Combine (TestPath, "A.txt"), "A.txt", expected: true); - AssertSkipExisting (Path.Combine (TestPath, "B", "B.txt"), "B/B.txt", expected: true); - AssertSkipExisting (Path.Combine (TestPath, "C.txt"), "C.txt", expected: false); - AssertSkipExisting (Path.Combine (TestPath, "C", "C.txt"), "C/C.txt", expected: false); - } - - [Test] - [Repeat(100)] - public void SkipExistingFile_TimeStamp () - { - CreateDirectories ("A.txt", Path.Combine ("B", "B.txt")); - - using (var archive = new ZipArchiveEx (Zip, FileMode.Create)) { - archive.AddDirectory (TestPath, folderInArchive: "", CompressionMethod.Deflate); - } - - AssertZip ( -@"A.txt -B/B.txt"); - var file = Path.Combine (TestPath, "A.txt"); - File.SetLastWriteTimeUtc (file, DateTime.UtcNow.AddSeconds (1)); - AssertSkipExisting (file, "A.txt", expected: false); - AssertSkipExisting (Path.Combine (TestPath, "B", "B.txt"), "B/B.txt", expected: true); - } - - [Test] - [Repeat (100)] - public void SkipExistingFile_Compression () - { - var fileName = "A.txt"; - var filePath = Path.Combine (TestPath, fileName); - CreateDirectories (fileName); - - using (var archive = new ZipArchiveEx (Zip, FileMode.Create)) { - archive.AddDirectory (TestPath, folderInArchive: "", CompressionMethod.Deflate); - } - - AssertSkipExisting (filePath, fileName, expected: true, method: CompressionMethod.Deflate); - AssertSkipExisting (filePath, fileName, expected: true, method: CompressionMethod.Default); - AssertSkipExisting (filePath, fileName, expected: false, method: CompressionMethod.Store); - - File.Delete (Zip); - using (var archive = new ZipArchiveEx (Zip, FileMode.Create)) { - archive.AddDirectory (TestPath, folderInArchive: "", method: CompressionMethod.Store); - } - - AssertSkipExisting (filePath, fileName, expected: true, method: CompressionMethod.Store); - AssertSkipExisting (filePath, fileName, expected: false, method: CompressionMethod.Deflate); - AssertSkipExisting (filePath, fileName, expected: false, method: CompressionMethod.Default); - } - } -} diff --git a/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.ProjectTools/Common/BuildOutput.cs b/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.ProjectTools/Common/BuildOutput.cs index 668ecef3299..060643cb9f7 100644 --- a/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.ProjectTools/Common/BuildOutput.cs +++ b/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.ProjectTools/Common/BuildOutput.cs @@ -1,10 +1,10 @@ using System; using System.Collections.Generic; using System.IO; +using System.IO.Compression; using System.Linq; using System.Text.RegularExpressions; using Microsoft.Build.Framework; -using Xamarin.Tools.Zip; namespace Xamarin.ProjectTools { @@ -247,20 +247,21 @@ public void Dispose () apk.Dispose (); } - ZipEntry GetEntry (string file) + ZipArchiveEntry GetEntry (string file) { - return apk.First (e => e.FullName == file); + return apk.Entries.First (e => e.FullName == file); } public bool Exists (string file) { - return apk.Any (e => e.FullName == file); + return apk.Entries.Any (e => e.FullName == file); } public string GetText (string file) { - using (var ms = new MemoryStream ()) { - GetEntry (file).Extract (ms); + using (var ms = new MemoryStream ()) + using (var stream = GetEntry (file).Open ()) { + stream.CopyTo (ms); ms.Position = 0; using (var sr = new StreamReader (ms)) return sr.ReadToEnd (); @@ -270,8 +271,9 @@ public string GetText (string file) public byte [] GetRaw (string file) { var e = GetEntry (file); - using (var ms = new MemoryStream ()) { - e.Extract (ms); + using (var ms = new MemoryStream ()) + using (var stream = e.Open ()) { + stream.CopyTo (ms); return ms.ToArray (); } } diff --git a/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.ProjectTools/Utilities/ZipHelper.cs b/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.ProjectTools/Utilities/ZipHelper.cs index 2ce9f070f14..75b31f62b12 100644 --- a/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.ProjectTools/Utilities/ZipHelper.cs +++ b/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.ProjectTools/Utilities/ZipHelper.cs @@ -1,7 +1,9 @@ using System; +using System.Collections.Generic; using System.IO; +using System.IO.Compression; using System.Linq; -using Xamarin.Tools.Zip; +using System.Text; namespace Xamarin.ProjectTools { @@ -12,29 +14,46 @@ public static ZipArchive OpenZip (string zipFile) { if (!File.Exists (zipFile)) return null; - return ZipArchive.Open (zipFile, FileMode.Open); + return ZipFile.OpenRead (zipFile); } public static byte [] ReadFileFromZip (ZipArchive zip, string filename) { - if (zip.ContainsEntry (filename)) { - var entry = zip.FirstOrDefault (x => x.FullName == filename); - if (entry != null) { - using (var ms = new MemoryStream ()) { - entry.Extract (ms); - return ms.ToArray (); - } + var entry = zip.Entries.FirstOrDefault (x => x.FullName == filename); + if (entry != null) { + using (var ms = new MemoryStream ()) + using (var stream = entry.Open ()) { + stream.CopyTo (ms); + return ms.ToArray (); } } + return null; } public static byte [] ReadFileFromZip (string zipFile, string filename) { - using (var zip = ZipArchive.Open (zipFile, FileMode.Open)) { + using (var zip = ZipFile.OpenRead (zipFile)) { return ReadFileFromZip (zip, filename); } } + + public static IEnumerator GetEnumerator (this ZipArchive zip) + => zip.Entries.GetEnumerator (); + + public static bool Any (this ZipArchive zip, Func predicate) + => zip.Entries.Any (predicate); + + public static int Count (this ZipArchive zip, Func predicate) + => zip.Entries.Count (predicate); + + public static ZipArchiveEntry Single (this ZipArchive zip, Func predicate) + => zip.Entries.Single (predicate); + + public static IEnumerable Where (this ZipArchive zip, Func predicate) + => zip.Entries.Where (predicate); + + public static IEnumerable Select (this ZipArchive zip, Func selector) + => zip.Entries.Select (selector); } } - diff --git a/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.ProjectTools/Xamarin.ProjectTools.csproj b/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.ProjectTools/Xamarin.ProjectTools.csproj index 45edebac50b..bbed96c3430 100644 --- a/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.ProjectTools/Xamarin.ProjectTools.csproj +++ b/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.ProjectTools/Xamarin.ProjectTools.csproj @@ -2,7 +2,6 @@ {2DD1EE75-6D8D-4653-A800-0A24367F7F38} - true $(DotNetStableTargetFramework) true ..\..\..\..\product.snk diff --git a/src/Xamarin.Android.Build.Tasks/Utilities/MonoAndroidHelper.cs b/src/Xamarin.Android.Build.Tasks/Utilities/MonoAndroidHelper.cs index 08455232c0d..96b6031bb63 100644 --- a/src/Xamarin.Android.Build.Tasks/Utilities/MonoAndroidHelper.cs +++ b/src/Xamarin.Android.Build.Tasks/Utilities/MonoAndroidHelper.cs @@ -5,12 +5,12 @@ using System.Diagnostics; using System.Linq; using System.IO; +using System.IO.Compression; using System.Reflection.Metadata; using System.Reflection.PortableExecutable; using System.Text; using System.Threading; using Xamarin.Android.Tools; -using Xamarin.Tools.Zip; using Java.Interop.Tools.JavaCallableWrappers; using Mono.Cecil; @@ -432,8 +432,8 @@ public static ZipArchive ReadZipFile (string filename) { try { return Files.ReadZipFile (filename); - } catch (ZipIOException ex) { - throw new ZipIOException ($"There was an error opening {filename}. The file is probably corrupt. Try deleting it and building again. {ex.Message}", ex); + } catch (InvalidDataException ex) { + throw new InvalidDataException ($"There was an error opening {filename}. The file is probably corrupt. Try deleting it and building again. {ex.Message}", ex); } } diff --git a/src/Xamarin.Android.Build.Tasks/Utilities/UtilityExtensions.cs b/src/Xamarin.Android.Build.Tasks/Utilities/UtilityExtensions.cs index ce5e689496c..ed0d7f556c3 100644 --- a/src/Xamarin.Android.Build.Tasks/Utilities/UtilityExtensions.cs +++ b/src/Xamarin.Android.Build.Tasks/Utilities/UtilityExtensions.cs @@ -1,52 +1,13 @@ #nullable enable using System; using System.IO; -using System.IO.Compression; using System.Xml; using System.Xml.Linq; -using Xamarin.Tools.Zip; namespace Xamarin.Android.Tasks; static class UtilityExtensions { - public static System.IO.Compression.CompressionLevel ToCompressionLevel (this CompressionMethod method) - { - switch (method) { - case CompressionMethod.Store: - return System.IO.Compression.CompressionLevel.NoCompression; - case CompressionMethod.Default: - case CompressionMethod.Deflate: - return System.IO.Compression.CompressionLevel.Optimal; - default: - throw new ArgumentOutOfRangeException (nameof (method), method, null); - } - } - - public static CompressionMethod ToCompressionMethod (this System.IO.Compression.CompressionLevel level) - { - switch (level) { - case System.IO.Compression.CompressionLevel.NoCompression: - return CompressionMethod.Store; - case System.IO.Compression.CompressionLevel.Optimal: - return CompressionMethod.Deflate; - default: - throw new ArgumentOutOfRangeException (nameof (level), level, null); - } - } - - public static FileMode ToFileMode (this ZipArchiveMode mode) - { - switch (mode) { - case ZipArchiveMode.Create: - return FileMode.Create; - case ZipArchiveMode.Update: - return FileMode.Open; - default: - throw new ArgumentOutOfRangeException (nameof (mode), mode, null); - } - } - public static T GetAttributeOrDefault (this XElement xml, string name, T defaultValue) { var value = xml.Attribute (name)?.Value; diff --git a/src/Xamarin.Android.Build.Tasks/Utilities/ZipArchiveDotNet.cs b/src/Xamarin.Android.Build.Tasks/Utilities/ZipArchiveDotNet.cs deleted file mode 100644 index 12a84f83ab4..00000000000 --- a/src/Xamarin.Android.Build.Tasks/Utilities/ZipArchiveDotNet.cs +++ /dev/null @@ -1,263 +0,0 @@ -using System; -using System.Collections.Generic; -using System.IO; -using System.IO.Compression; -using System.Linq; -using System.Reflection; -using Microsoft.Android.Build.Tasks; -using Microsoft.Build.Utilities; - -namespace Xamarin.Android.Tasks; - -interface IZipArchive : IDisposable -{ - int ZipFlushFilesLimit { get; set; } - int ZipFlushSizeLimit { get; set; } - - void AddEntry (byte [] data, string apkPath); - void AddEntry (Stream stream, string apkPath, CompressionLevel compression); - bool AddFileIfChanged (TaskLoggingHelper log, string filename, string archiveFileName, CompressionLevel compression); - bool ContainsEntry (string entryPath); - void DeleteEntry (string entry); - void FixupWindowsPathSeparators (TaskLoggingHelper log); - IEnumerable GetAllEntryNames (); - IZipArchiveEntry GetEntry (string entryName); - void MoveEntry (string oldEntry, string newEntry); -} - -interface IZipArchiveEntry -{ - uint CRC { get; } - ulong CompressedSize { get; } -} - -class ZipArchiveDotNet : IZipArchive -{ - const int DEFAULT_FLUSH_SIZE_LIMIT = 100 * 1024 * 1024; - const int DEFAULT_FLUSH_FILES_LIMIT = 512; - - // ZipFile seems to be performant enough that we have not currently implemented a flush mechanism. - public int ZipFlushSizeLimit { get; set; } = DEFAULT_FLUSH_SIZE_LIMIT; - public int ZipFlushFilesLimit { get; set; } = DEFAULT_FLUSH_FILES_LIMIT; - - public ZipArchive Archive { get; } - - static readonly FieldInfo? crc_field; - static readonly FieldInfo? comp_field; - - // This is private to ensure the only way to create an instance is through the Create method. - ZipArchiveDotNet (string archive, ZipArchiveMode mode) - { - Archive = ZipFile.Open (archive, mode); - } - - static ZipArchiveDotNet () - { - // netstandard2.0 does not provide a way to access a ZipArchiveEntry's CRC or compresion level. - // We need to use reflection to access the private fields. - - // These private fields exist on both .NET Framework 4.7.2 and .NET 9.0. If they are not found, - // we will return a ZipArchiveEx which uses LibZipSharp instead. - crc_field = typeof (ZipArchiveEntry).GetField ("_crc32", BindingFlags.NonPublic | BindingFlags.Instance); - comp_field = typeof (ZipArchiveEntry).GetField ("_storedCompressionMethod", BindingFlags.NonPublic | BindingFlags.Instance); - } - - public static IZipArchive Create (TaskLoggingHelper log, string archive, ZipArchiveMode mode, bool fallback) - { - if (fallback) { - log.LogDebugMessage ($"ZipArchiveDotNet: Falling back to LibZipSharp as requested."); - return new ZipArchiveEx (archive, mode.ToFileMode ()); - } - - if (crc_field is null) { - log.LogDebugMessage ($"ZipArchiveDotNet: Could not find private CRC field, falling back to LibZipSharp."); - return new ZipArchiveEx (archive, mode.ToFileMode ()); - } - - if (comp_field is null) { - log.LogDebugMessage ($"ZipArchiveDotNet: Could not find private CompressionMethod field, falling back to LibZipSharp."); - return new ZipArchiveEx (archive, mode.ToFileMode ()); - } - - log.LogDebugMessage ($"ZipArchiveDotNet: Using ZipArchiveDotNet."); - return new ZipArchiveDotNet (archive, mode); - } - - public void AddEntry (byte [] data, string apkPath) - { - var entry = Archive.CreateEntry (apkPath); - - using (var stream = entry.Open ()) - stream.Write (data, 0, data.Length); - } - - public void AddEntry (Stream stream, string apkPath, CompressionLevel compression) - { - var entry = Archive.CreateEntry (apkPath, compression); - - using (var entry_stream = entry.Open ()) - stream.CopyTo (entry_stream); - } - - public bool AddFileIfChanged (TaskLoggingHelper log, string filename, string archiveFileName, CompressionLevel compression) - { - if (!FileNeedsUpdating (log, filename, archiveFileName, compression)) - return false; - - DeleteEntry (archiveFileName); - Archive.CreateEntryFromFile (filename, archiveFileName, compression); - - return true; - } - - public bool ContainsEntry (string entryName) - { - return Archive.GetEntry (entryName) is not null; - } - - public void DeleteEntry (string entryName) - { - var entry = Archive.GetEntry (entryName); - - entry?.Delete (); - } - - /// - /// HACK: aapt2 is creating zip entries on Windows such as `assets\subfolder/asset2.txt` - /// - public void FixupWindowsPathSeparators (TaskLoggingHelper log) - { - var malformed_entries = Archive.Entries.Where (entry => entry.FullName.Contains ('\\')).ToList (); - - foreach (var entry in malformed_entries) { - var name = entry.FullName.Replace ('\\', '/'); - if (name != entry.FullName) { - log.LogDebugMessage ($"Fixing up malformed entry `{entry.FullName}` -> `{name}`"); - MoveEntry (entry.FullName, name); - } - } - } - - public IEnumerable GetAllEntryNames () - { - return Archive.Entries.Select (entry => entry.FullName); - } - - public IZipArchiveEntry GetEntry (string entryName) - { - var entry = Archive.GetEntry (entryName); - - if (entry is null) - throw new ArgumentOutOfRangeException (nameof (entryName)); - - return new ZipArchiveEntryDotNet (entry, GetCrc32 (entry)); - } - - public void Dispose () - { - Archive.Dispose (); - } - - public void MoveEntry (string oldPath, string newPath) - { - var old_entry = Archive.GetEntry (oldPath); - - if (old_entry is null) - return; - - var new_entry = Archive.CreateEntry (newPath); - - using (var oldStream = old_entry.Open ()) - using (var newStream = new_entry.Open ()) - oldStream.CopyTo (newStream); - - old_entry.Delete (); - } - - bool FileNeedsUpdating (TaskLoggingHelper log, string filename, string archiveFileName, CompressionLevel compression) - { - var entry = Archive.GetEntry (archiveFileName); - - if (entry is null) { - log.LogDebugMessage ($"Adding {filename} as it doesn't already exist."); - return true; - } - - var stored_compression = GetCompressionLevel (entry); - - if (stored_compression != compression) { - log.LogDebugMessage ($"Updating {filename} as the compression level changed: existing - '{stored_compression}', requested - '{compression}'."); - return true; - } - - var last_write = File.GetLastWriteTimeUtc (filename); - var file_write_dos_time = DateTimeToDosTime (last_write); - var zip_write_dos_time = DateTimeToDosTime (entry.LastWriteTime.UtcDateTime); - - if (DateTimeToDosTime (entry.LastWriteTime.UtcDateTime) < DateTimeToDosTime (last_write)) { - log.LogDebugMessage ($"Updating {filename} as the file write time is newer: file in zip - '{zip_write_dos_time}', file on disk - '{file_write_dos_time}'."); - return true; - } - - log.LogDebugMessage ($"Skipping {filename} as the archive file is up to date."); - return false; - } - - static uint GetCrc32 (ZipArchiveEntry entry) - { - if (crc_field is null) - throw new NotSupportedException ("This method is not supported on this platform."); - - return (uint) crc_field.GetValue (entry); - } - - static CompressionLevel GetCompressionLevel (ZipArchiveEntry entry) - { - if (comp_field is null) - throw new NotSupportedException ("This method is not supported on this platform."); - - var level = comp_field.GetValue (entry).ToString (); - - switch (level) { - case "Stored": - return CompressionLevel.NoCompression; - case "Deflate": - return CompressionLevel.Optimal; - default: - throw new NotSupportedException ($"Unsupported compression level: {level}"); - } - } - - // System.IO.Compression.ZipArchive apparently only provides a 2 second granularity for the LastWriteTime. - // This should be fine, it would be nearly impossible for someone to complete a build, make a change, - // and rebuild in under 2 seconds. - // This is a port of the DateTimeToDosTime method from System.IO.Compression.ZipHelper - // https://github.com/dotnet/runtime/blob/373f048bae3c46810bc030ed7c1ee0568ee5ecc0/src/libraries/System.IO.Compression/src/System/IO/Compression/ZipHelper.cs#L88 - const int ValidZipDate_YearMin = 1980; - - static uint DateTimeToDosTime (DateTime dateTime) - { - int ret = ((dateTime.Year - ValidZipDate_YearMin) & 0x7F); - ret = (ret << 4) + dateTime.Month; - ret = (ret << 5) + dateTime.Day; - ret = (ret << 5) + dateTime.Hour; - ret = (ret << 6) + dateTime.Minute; - ret = (ret << 5) + (dateTime.Second / 2); // only 5 bits for second, so we only have a granularity of 2 sec. - return (uint) ret; - } - - class ZipArchiveEntryDotNet : IZipArchiveEntry - { - readonly ZipArchiveEntry entry; - readonly uint crc; - - public uint CRC => crc; - public ulong CompressedSize => (ulong) entry.CompressedLength; - - public ZipArchiveEntryDotNet (ZipArchiveEntry entry, uint crc) - { - this.entry = entry; - this.crc = crc; - } - } -} diff --git a/src/Xamarin.Android.Build.Tasks/Utilities/ZipArchiveEx.cs b/src/Xamarin.Android.Build.Tasks/Utilities/ZipArchiveEx.cs deleted file mode 100644 index 986e0b845a5..00000000000 --- a/src/Xamarin.Android.Build.Tasks/Utilities/ZipArchiveEx.cs +++ /dev/null @@ -1,309 +0,0 @@ -#nullable enable -using System; -using System.Collections.Generic; -using System.IO; -using System.Linq; -using Microsoft.Android.Build.Tasks; -using Microsoft.Build.Utilities; -using Xamarin.Tools.Zip; - -namespace Xamarin.Android.Tasks -{ - public class ZipArchiveEx : IZipArchive - { - - const int DEFAULT_FLUSH_SIZE_LIMIT = 100 * 1024 * 1024; - const int DEFAULT_FLUSH_FILES_LIMIT = 512; - - ZipArchive zip; - string archive; - long filesWrittenTotalSize = 0; - long filesWrittenTotalCount = 0; - - public ZipArchive Archive { - get { return zip; } - } - - public bool AutoFlush { get; set; } = true; - - public bool CreateDirectoriesInZip { get; set; } = true; - - public int ZipFlushSizeLimit { get; set; } = DEFAULT_FLUSH_SIZE_LIMIT; - public int ZipFlushFilesLimit { get; set; } = DEFAULT_FLUSH_FILES_LIMIT; - - public ZipArchiveEx (string archive) : this (archive, FileMode.CreateNew) - { - } - - public ZipArchiveEx(string archive, FileMode filemode) - { - this.archive = archive; - zip = ZipArchive.Open(archive, filemode); - } - - public void Flush () - { - if (zip != null) { - zip.Close (); - zip.Dispose (); - zip = null!; - } - zip = ZipArchive.Open (archive, FileMode.Open); - filesWrittenTotalSize = 0; - filesWrittenTotalCount = 0; - } - - string ArchiveNameForFile (string filename, string directoryPathInZip) - { - if (filename.IsNullOrEmpty ()) { - throw new ArgumentNullException (nameof (filename)); - } - string pathName; - if (directoryPathInZip.IsNullOrEmpty ()) { - pathName = Path.GetFileName (filename); - } - else { - pathName = Path.Combine (directoryPathInZip, Path.GetFileName (filename)); - } - return pathName.Replace ("\\", "/").TrimStart ('/'); - } - - void AddFileAndFlush (string filename, long fileLength, string archiveFileName, CompressionMethod compressionMethod) - { - filesWrittenTotalSize += fileLength; - zip.AddFile (filename, archiveFileName, compressionMethod: compressionMethod); - if ((filesWrittenTotalSize >= ZipFlushSizeLimit || filesWrittenTotalCount >= ZipFlushFilesLimit) && AutoFlush) { - Flush (); - } - } - - public void AddFileAndFlush (string filename, string archiveFileName, CompressionMethod compressionMethod) - { - var fi = new FileInfo (filename); - AddFileAndFlush (filename, fi.Length, archiveFileName, compressionMethod); - } - - public void AddEntryAndFlush (byte[] data, string archiveFileName) - { - filesWrittenTotalSize += data.Length; - zip.AddEntry (data, archiveFileName); - if ((filesWrittenTotalSize >= ZipFlushSizeLimit || filesWrittenTotalCount >= ZipFlushFilesLimit) && AutoFlush) { - Flush (); - } - } - - public void AddEntryAndFlush (string archiveFileName, Stream data, CompressionMethod method) - { - filesWrittenTotalSize += data.Length; - zip.AddEntry (archiveFileName, data, method); - if ((filesWrittenTotalSize >= ZipFlushSizeLimit || filesWrittenTotalCount >= ZipFlushFilesLimit) && AutoFlush) { - Flush (); - } - } - - void AddFiles (string folder, string folderInArchive, CompressionMethod method) - { - foreach (string fileName in Directory.GetFiles (folder, "*.*", SearchOption.TopDirectoryOnly)) { - var fi = new FileInfo (fileName); - if ((fi.Attributes & FileAttributes.Hidden) != 0) - continue; - var archiveFileName = ArchiveNameForFile (fileName, folderInArchive); - long index = -1; - if (zip.ContainsEntry (archiveFileName, out index)) { - var e = zip.First (x => x.FullName == archiveFileName); - if (e.ModificationTime < fi.LastWriteTimeUtc || e.Size != (ulong)fi.Length) { - AddFileAndFlush (fileName, fi.Length, archiveFileName, compressionMethod: method); - } - } else { - AddFileAndFlush (fileName, fi.Length, archiveFileName, compressionMethod: method); - } - } - } - - public void RemoveFile (string folder, string file) - { - var archiveName = ArchiveNameForFile (file, Path.Combine (folder, Path.GetDirectoryName (file))); - long index = -1; - if (zip.ContainsEntry (archiveName, out index)) - zip.DeleteEntry ((ulong)index); - } - - public bool MoveEntry (string from, string to) - { - if (!zip.ContainsEntry (from)) { - return false; - } - var entry = zip.ReadEntry (from); - using (var stream = new MemoryStream ()) { - entry.Extract (stream); - stream.Position = 0; - zip.AddEntry (to, stream); - zip.DeleteEntry (entry); - Flush (); - } - return true; - } - - public void AddDirectory (string folder, string folderInArchive, CompressionMethod method = CompressionMethod.Default) - { - if (!folder.IsNullOrEmpty ()) { - folder = folder.Replace ('/', Path.DirectorySeparatorChar).Replace ('\\', Path.DirectorySeparatorChar); - folder = Path.GetFullPath (folder); - if (folder [folder.Length - 1] == Path.DirectorySeparatorChar) { - folder = folder.Substring (0, folder.Length - 1); - } - } - - AddFiles (folder, folderInArchive, method); - foreach (string dir in Directory.GetDirectories (folder, "*", SearchOption.AllDirectories)) { - var di = new DirectoryInfo (dir); - if ((di.Attributes & FileAttributes.Hidden) != 0) - continue; - var internalDir = dir.Replace (folder, string.Empty); - string fullDirPath = folderInArchive + internalDir; - AddFiles (dir, fullDirPath, method); - } - } - - /// - /// HACK: aapt2 is creating zip entries on Windows such as `assets\subfolder/asset2.txt` - /// - public void FixupWindowsPathSeparators (Action onRename) - { - bool modified = false; - foreach (var entry in zip) { - if (entry.FullName.Contains ("\\")) { - var name = entry.FullName.Replace ('\\', '/'); - onRename?.Invoke (entry.FullName, name); - entry.Rename (name); - modified = true; - } - } - if (modified) { - Flush (); - } - } - - public bool SkipExistingFile (string file, string fileInArchive, CompressionMethod compressionMethod) - { - if (!zip.ContainsEntry (fileInArchive)) { - return false; - } - var entry = zip.ReadEntry (fileInArchive); - switch (compressionMethod) { - case CompressionMethod.Unknown: - // If incoming value is Unknown, don't check anything - break; - case CompressionMethod.Default: - // For Default, existing entries could have CompressionMethod.Deflate - // Only compare against CompressionMethod.Store - if (entry.CompressionMethod == CompressionMethod.Store) - return false; - break; - default: - // Other values can just compare CompressionMethod - if (entry.CompressionMethod != compressionMethod) - return false; - break; - } - var lastWrite = File.GetLastWriteTimeUtc (file); - return WithoutMilliseconds (lastWrite) <= WithoutMilliseconds (entry.ModificationTime); - } - - public bool SkipExistingEntry (ZipEntry sourceEntry, string fileInArchive) - { - if (!zip.ContainsEntry (fileInArchive)) { - return false; - } - var entry = zip.ReadEntry (fileInArchive); - return WithoutMilliseconds (sourceEntry.ModificationTime) <= WithoutMilliseconds (entry.ModificationTime); - } - - // The zip file and macOS/mono does not contain milliseconds - // Windows *does* contain milliseconds - static DateTime WithoutMilliseconds (DateTime t) => - new DateTime (t.Year, t.Month, t.Day, t.Hour, t.Minute, t.Second, t.Kind); - - public void Dispose () - { - Dispose(true); - GC.SuppressFinalize (this); - } - - protected virtual void Dispose(bool disposing) { - if (disposing) { - if (zip != null) { - zip.Close (); - zip.Dispose (); - zip = null!; - } - } - } - - public void AddEntry (byte [] data, string apkPath) - => AddEntryAndFlush (data, apkPath); - - public void AddEntry (Stream stream, string apkPath, System.IO.Compression.CompressionLevel compression) - => AddEntryAndFlush (apkPath, stream, compression.ToCompressionMethod ()); - - public bool AddFileIfChanged (TaskLoggingHelper log, string filename, string archiveFileName, System.IO.Compression.CompressionLevel compression) - { - var compressionMethod = compression.ToCompressionMethod (); - - if (!SkipExistingFile (filename, archiveFileName, compressionMethod)) { - AddFileAndFlush (filename, archiveFileName, compressionMethod); - log.LogDebugMessage ($"Adding {filename} as the archive file is out of date."); - return true; - } - - log.LogDebugMessage ($"Skipping {filename} as the archive file is up to date."); - - return false; - } - - public bool ContainsEntry (string entryPath) - => zip.ContainsEntry (entryPath); - - public void DeleteEntry (string entry) - => zip.DeleteEntry (entry); - - public void FixupWindowsPathSeparators (TaskLoggingHelper log) - => FixupWindowsPathSeparators ((a, b) => log.LogDebugMessage ($"Fixing up malformed entry `{a}` -> `{b}`")); - - public IEnumerable GetAllEntryNames () - { - for (var i = 0; i < Archive.EntryCount; i++) { - var entry = Archive.ReadEntry ((ulong) i); - yield return entry.FullName; - } - } - - IZipArchiveEntry IZipArchive.GetEntry (string entryName) - { - return new ZipArchiveEntryEx (zip.ReadEntry (entryName)); - } - - void IZipArchive.MoveEntry (string oldEntry, string newEntry) - { - if (Archive.ContainsEntry (newEntry)) - Archive.DeleteEntry (Archive.ReadEntry (newEntry)); - - var entry = zip.ReadEntry (oldEntry); - entry.Rename (newEntry); - } - } - - class ZipArchiveEntryEx : IZipArchiveEntry - { - readonly ZipEntry entry; - - public ZipArchiveEntryEx (ZipEntry entry) - { - this.entry = entry; - } - - public uint CRC => entry.CRC; - - public ulong CompressedSize => entry.CompressedSize; - } -} diff --git a/src/Xamarin.Android.Build.Tasks/Xamarin.Android.Build.Tasks.csproj b/src/Xamarin.Android.Build.Tasks/Xamarin.Android.Build.Tasks.csproj index 2be97ec0043..929109e315b 100644 --- a/src/Xamarin.Android.Build.Tasks/Xamarin.Android.Build.Tasks.csproj +++ b/src/Xamarin.Android.Build.Tasks/Xamarin.Android.Build.Tasks.csproj @@ -10,7 +10,6 @@ netstandard2.0 Xamarin.Android.Tasks - true true $(MicrosoftAndroidSdkOutDir) false diff --git a/src/Xamarin.Android.Build.Tasks/Xamarin.Android.Build.Tasks.targets b/src/Xamarin.Android.Build.Tasks/Xamarin.Android.Build.Tasks.targets index 833bce9d05a..448abdad0d1 100644 --- a/src/Xamarin.Android.Build.Tasks/Xamarin.Android.Build.Tasks.targets +++ b/src/Xamarin.Android.Build.Tasks/Xamarin.Android.Build.Tasks.targets @@ -234,8 +234,6 @@ - <_ExtraPackageSource Include="$(PkgXamarin_LibZipSharp)\lib\$(TargetFrameworkNETStandard)\libZipSharp.pdb" /> - <_ExtraPackageTarget Include="$(OutputPath)\libZipSharp.pdb" /> <_ExtraPackageSource Include="$(PkgMono_Cecil)\lib\$(TargetFrameworkNETStandard)\Mono.Cecil.pdb" /> <_ExtraPackageTarget Include="$(OutputPath)\Mono.Cecil.pdb" /> <_ExtraPackageSource Include="$(PkgMono_Cecil)\lib\$(TargetFrameworkNETStandard)\Mono.Cecil.Mdb.pdb" /> @@ -264,10 +262,6 @@ /> - - - - diff --git a/src/Xamarin.Android.Build.Tasks/Xamarin.Android.Common.targets b/src/Xamarin.Android.Build.Tasks/Xamarin.Android.Common.targets index f043a72abc5..744cb1053c4 100644 --- a/src/Xamarin.Android.Build.Tasks/Xamarin.Android.Common.targets +++ b/src/Xamarin.Android.Build.Tasks/Xamarin.Android.Common.targets @@ -30,7 +30,8 @@ Copyright (C) 2011-2012 Xamarin. All rights reserved. - + + @@ -2328,10 +2329,7 @@ because xbuild doesn't support framework reference assemblies. ApkInputPath="$(_PackagedResources)" ApkOutputPath="$(_ApkOutputPath)" FilesToAddToArchive="@(FilesToAddToArchive)" - UncompressedFileExtensions="$(AndroidStoreUncompressedFileExtensions)" - UseLibZipSharp="$(_AndroidUseLibZipSharp)" - ZipFlushFilesLimit="$(_ZipFlushFilesLimit)" - ZipFlushSizeLimit="$(_ZipFlushSizeLimit)" /> + UncompressedFileExtensions="$(AndroidStoreUncompressedFileExtensions)" /> + UncompressedFileExtensions="$(AndroidStoreUncompressedFileExtensions)" /> netstandard2.0 - true false true $(NoWarn);CA1305 @@ -48,7 +47,6 @@ - diff --git a/src/Xamarin.Android.Tools.JavadocImporter/samples.cs b/src/Xamarin.Android.Tools.JavadocImporter/samples.cs index 40907040e44..49b20d7df41 100644 --- a/src/Xamarin.Android.Tools.JavadocImporter/samples.cs +++ b/src/Xamarin.Android.Tools.JavadocImporter/samples.cs @@ -5,8 +5,8 @@ using System.Security.Cryptography; using System.Text; using System.Xml.Serialization; +using System.IO.Compression; using Java.Interop.Tools.JavaCallableWrappers; -using Xamarin.Tools.Zip; public struct SampleDesc { public string ID { get; set; } @@ -23,25 +23,33 @@ public override string ToString () public class SampleRepository { ZipArchive archive; + Stream archiveStream; HashAlgorithm hasher = new Crc64 (); HashSet validFiles = new HashSet (); Dictionary index = new Dictionary (); Dictionary updates = new Dictionary (); - public SampleRepository (string name) : this (ZipArchive.Open (name.EndsWith (".zip", StringComparison.OrdinalIgnoreCase) ? name : name + ".zip", FileMode.Open)) + public SampleRepository (string name) { - + var path = name.EndsWith (".zip", StringComparison.OrdinalIgnoreCase) ? name : name + ".zip"; + archiveStream = new FileStream (path, FileMode.OpenOrCreate, FileAccess.ReadWrite, FileShare.Read); + archive = new ZipArchive (archiveStream, ZipArchiveMode.Update); + LoadIndex (); } protected SampleRepository (ZipArchive archive) { this.archive = archive; - if (archive.ContainsEntry ("index.xml")) - using (var stream = new MemoryStream ()) { - archive.First (e => e.FullName.Equals ("index.xml", StringComparison.OrdinalIgnoreCase)).Extract (stream); - stream.Position = 0; - index = ((ICollection)IndexSerializer.Deserialize (stream)).ToDictionary (sd => sd.ID, sd => sd); - } + LoadIndex (); + } + + void LoadIndex () + { + var indexEntry = archive.GetEntry ("index.xml"); + if (indexEntry != null) { + using var stream = indexEntry.Open (); + index = ((ICollection) IndexSerializer.Deserialize (stream)).ToDictionary (sd => sd.ID, sd => sd); + } } // Returns an id that can be used to register the sample position in the documentation flow @@ -51,9 +59,9 @@ public string RegisterSample (string source, SampleDesc desc) validFiles.Add (hash); desc.ID = hash; - if (!archive.ContainsEntry (hash)) { + if (archive.GetEntry (hash) == null) { updates.Add (hash, source); - archive.AddEntry (hash, source, Encoding.UTF8); + AddEntry (archive, hash, source); index[hash] = desc; } @@ -62,7 +70,7 @@ public string RegisterSample (string source, SampleDesc desc) public void OverwriteSample (string hash, string content, SampleDesc newDesc) { - if (!archive.ContainsEntry (hash)) + if (archive.GetEntry (hash) == null) return; UpdateEntry (archive, hash, content); @@ -71,10 +79,10 @@ public void OverwriteSample (string hash, string content, SampleDesc newDesc) void UpdateEntry (ZipArchive archive, string file, string content) { - var existing = archive.FirstOrDefault (e => e.FullName == file); + var existing = archive.GetEntry (file); if (existing != null) - archive.DeleteEntry (existing); - archive.AddEntry (file, content, Encoding.UTF8); + existing.Delete (); + AddEntry (archive, file, content); } public string GetSampleFromID (string id, out SampleDesc desc) @@ -85,16 +93,13 @@ public string GetSampleFromID (string id, out SampleDesc desc) if (updates.TryGetValue (id, out source)) return source; - var entry = archive.FirstOrDefault (e => e.FullName == id); + var entry = archive.GetEntry (id); if (entry == null) return null; desc = index[id]; - using (var stream = new MemoryStream ()) { - entry.Extract (stream); - stream.Position = 0; - return new StreamReader (stream).ReadToEnd (); - } + using var stream = entry.Open (); + return new StreamReader (stream).ReadToEnd (); } public string GetSampleFromContent (string content, out SampleDesc desc) @@ -109,20 +114,17 @@ public SampleDesc GetSampleDescFromID (string id) public static SampleRepository LoadFrom (string file) { - using (var archive = ZipArchive.Open (file, FileMode.Open)) { - var samples = new SampleRepository (archive); - return samples; - } + return new SampleRepository (file); } public void Close (bool removeOldEntries) { // See if we have any stale file if (removeOldEntries) { - var list = new List (archive); + var list = new List (archive.Entries); foreach (var entry in list) { if (!validFiles.Contains (entry.FullName)) - archive.DeleteEntry (entry); + entry.Delete (); } } // Serialize index @@ -130,7 +132,8 @@ public void Close (bool removeOldEntries) IndexSerializer.Serialize (writer, new List (index.Values)); UpdateEntry (archive, "index.xml", writer.ToString ()); - archive.Close (); + archive.Dispose (); + archiveStream?.Dispose (); } public IEnumerable AllIDs { @@ -156,6 +159,13 @@ XmlSerializer IndexSerializer { } } + static void AddEntry (ZipArchive archive, string entryName, string content) + { + var entry = archive.CreateEntry (entryName); + using var writer = new StreamWriter (entry.Open (), Encoding.UTF8); + writer.Write (content); + } + #if STANDALONE_EXPORTER public static void Main (string[] args) { diff --git a/src/androidsdk/androidsdk.targets b/src/androidsdk/androidsdk.targets index 4768da3cf37..0af897f4d79 100644 --- a/src/androidsdk/androidsdk.targets +++ b/src/androidsdk/androidsdk.targets @@ -299,10 +299,10 @@ + - net12.0 + $(DotNetTargetFramework) Microsoft.Android.Tasks enable Nullable diff --git a/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/SampleRepositoryTests.cs b/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/SampleRepositoryTests.cs new file mode 100644 index 00000000000..a99bd7fbb8e --- /dev/null +++ b/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/SampleRepositoryTests.cs @@ -0,0 +1,46 @@ +#nullable enable +using System.IO; +using NUnit.Framework; + +namespace Xamarin.Android.Build.Tests; + +[TestFixture] +public class SampleRepositoryTests : BaseTest +{ + [Test] + public void CreatesMissingRepository () + { + var directory = Path.Combine (Root, "temp", TestName); + var archivePath = Path.Combine (directory, "samples.zip"); + Directory.CreateDirectory (directory); + + try { + const string source = "Console.WriteLine (\"Hello\");"; + var repository = new global::SampleRepository (archivePath); + string id; + try { + id = repository.RegisterSample (source, new global::SampleDesc { + Language = "C#", + FullTypeName = "Example.Widget", + DocumentationFilePath = "Example.Widget.xml", + }); + } finally { + repository.Close (removeOldEntries: false); + } + + FileAssert.Exists (archivePath); + + var reopened = global::SampleRepository.LoadFrom (archivePath); + try { + Assert.AreEqual (source, reopened.GetSampleFromID (id, out var description)); + Assert.AreEqual ("C#", description.Language); + Assert.AreEqual ("Example.Widget", description.FullTypeName); + Assert.AreEqual ("Example.Widget.xml", description.DocumentationFilePath); + } finally { + reopened.Close (removeOldEntries: false); + } + } finally { + Directory.Delete (directory, recursive: true); + } + } +} diff --git a/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/Tasks/BuildArchiveTests.cs b/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/Tasks/BuildArchiveTests.cs index a55e5f76d88..68d149fc20c 100644 --- a/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/Tasks/BuildArchiveTests.cs +++ b/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/Tasks/BuildArchiveTests.cs @@ -259,6 +259,33 @@ public void ArchiveRootDirectoryProvidesRelativeEntryPath () Assert.AreEqual (ZipEntryCompressionMethod.Store, ZipArchiveMetadataReader.Read (apk) ["com/example/Main.class"].CompressionMethod); } + [Test] + public void FlushesArchiveUpdates () + { + var firstFile = Path.Combine (TempDirectory, "first.txt"); + var secondFile = Path.Combine (TempDirectory, "second.txt"); + File.WriteAllText (firstFile, "first"); + File.WriteAllText (secondFile, "second"); + + var firstItem = new TaskItem (firstFile); + firstItem.SetMetadata ("ArchivePath", "first.txt"); + var secondItem = new TaskItem (secondFile); + secondItem.SetMetadata ("ArchivePath", "second.txt"); + + var apk = Path.Combine (TempDirectory, "app.apk"); + var task = new BuildArchive { + BuildEngine = new MockBuildEngine (TestContext.Out), + ApkOutputPath = apk, + FilesToAddToArchive = [firstItem, secondItem], + ZipFlushFilesLimit = "1", + }; + + Assert.IsTrue (task.RunTask (), "task should have succeeded"); + using var archive = ZipFile.OpenRead (apk); + archive.AssertEntryContents (apk, "first.txt", "first"); + archive.AssertEntryContents (apk, "second.txt", "second"); + } + static void CreateArchive (string path, params (string name, string contents, CompressionLevel compressionLevel) [] entries) { using (var stream = new FileStream (path, FileMode.Create, FileAccess.ReadWrite)) diff --git a/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/Xamarin.Android.Build.Tests.csproj b/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/Xamarin.Android.Build.Tests.csproj index e37ae83ced7..7e948d303db 100644 --- a/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/Xamarin.Android.Build.Tests.csproj +++ b/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/Xamarin.Android.Build.Tests.csproj @@ -1,7 +1,7 @@ - net12.0 + $(DotNetTargetFramework) ..\..\..\..\bin\Test$(Configuration) true ..\..\..\..\product.snk @@ -26,6 +26,7 @@ + diff --git a/src/Xamarin.Android.Build.Tasks/Xamarin.Android.Common.targets b/src/Xamarin.Android.Build.Tasks/Xamarin.Android.Common.targets index 744cb1053c4..6b0ad7d325d 100644 --- a/src/Xamarin.Android.Build.Tasks/Xamarin.Android.Common.targets +++ b/src/Xamarin.Android.Build.Tasks/Xamarin.Android.Common.targets @@ -2329,7 +2329,9 @@ because xbuild doesn't support framework reference assemblies. ApkInputPath="$(_PackagedResources)" ApkOutputPath="$(_ApkOutputPath)" FilesToAddToArchive="@(FilesToAddToArchive)" - UncompressedFileExtensions="$(AndroidStoreUncompressedFileExtensions)" /> + UncompressedFileExtensions="$(AndroidStoreUncompressedFileExtensions)" + ZipFlushFilesLimit="$(_ZipFlushFilesLimit)" + ZipFlushSizeLimit="$(_ZipFlushSizeLimit)" /> + UncompressedFileExtensions="$(AndroidStoreUncompressedFileExtensions)" + ZipFlushFilesLimit="$(_ZipFlushFilesLimit)" + ZipFlushSizeLimit="$(_ZipFlushSizeLimit)" /> (index.Values)); UpdateEntry (archive, "index.xml", writer.ToString ()); @@ -166,6 +177,11 @@ static void AddEntry (ZipArchive archive, string entryName, string content) writer.Write (content); } + sealed class Utf8StringWriter : StringWriter + { + public override Encoding Encoding => Encoding.UTF8; + } + #if STANDALONE_EXPORTER public static void Main (string[] args) { diff --git a/tests/Microsoft.Android.Build.BaseTasks-Tests/ZipArchiveExtensionsTests.cs b/tests/Microsoft.Android.Build.BaseTasks-Tests/ZipArchiveExtensionsTests.cs index 77cf5b4fe58..6e28815fab2 100644 --- a/tests/Microsoft.Android.Build.BaseTasks-Tests/ZipArchiveExtensionsTests.cs +++ b/tests/Microsoft.Android.Build.BaseTasks-Tests/ZipArchiveExtensionsTests.cs @@ -78,6 +78,42 @@ public void OpenZipRead_OpensReadOnlyArchive () } } + [TestCase (ZipArchiveMode.Read)] + [TestCase (ZipArchiveMode.Update)] + [TestCase (ZipArchiveMode.Create)] + public void OpenZip_DisposesFileWhenArchiveCreationFails (ZipArchiveMode mode) + { + var archivePath = Path.Combine (TempDirectory, "archive.zip"); + using (ZipArchiveExtensions.CreateZip (archivePath)) { + } + + Assert.Throws (() => { + using var archive = mode switch { + ZipArchiveMode.Read => ZipArchiveExtensions.OpenZipRead (archivePath, Encoding.Unicode), + ZipArchiveMode.Update => ZipArchiveExtensions.OpenZipUpdate (archivePath, FileMode.Open, Encoding.Unicode), + ZipArchiveMode.Create => ZipArchiveExtensions.CreateZip (archivePath, FileMode.Create, Encoding.Unicode), + _ => throw new ArgumentOutOfRangeException (nameof (mode)), + }; + }); + + using var exclusive = new FileStream (archivePath, FileMode.Open, FileAccess.ReadWrite, FileShare.None); + } + + [Test] + public void ReadZip64Metadata () + { + var archivePath = Path.Combine (TempDirectory, "archive.zip"); + CreateZip64Archive (archivePath); + + var metadata = ZipArchiveMetadataReader.ReadEntries (archivePath); + + Assert.AreEqual (1, metadata.Count); + Assert.AreEqual ("entry.txt", metadata [0].FullName); + Assert.AreEqual (3, metadata [0].CompressedSize); + Assert.AreEqual (3, metadata [0].UncompressedSize); + Assert.AreEqual (ZipEntryCompressionMethod.Store, metadata [0].CompressionMethod); + } + static void CreateMalformedEmptyStoredEntryArchive (string path) { const string entryName = "R.txt"; @@ -129,5 +165,80 @@ static void CreateMalformedEmptyStoredEntryArchive (string path) writer.Write ((int) centralDirectoryOffset); writer.Write ((short) 0); } + + static void CreateZip64Archive (string path) + { + const string entryName = "entry.txt"; + var entryNameBytes = Encoding.ASCII.GetBytes (entryName); + var contents = Encoding.ASCII.GetBytes ("zip"); + + using var stream = File.Create (path); + using var writer = new BinaryWriter (stream, Encoding.UTF8, leaveOpen: false); + + writer.Write (0x04034b50); + writer.Write ((short) 45); + writer.Write ((short) 0); + writer.Write ((short) 0); + writer.Write (0); + writer.Write (0); + writer.Write (contents.Length); + writer.Write (contents.Length); + writer.Write ((short) entryNameBytes.Length); + writer.Write ((short) 0); + writer.Write (entryNameBytes); + writer.Write (contents); + + long centralDirectoryOffset = stream.Position; + writer.Write (0x02014b50); + writer.Write ((short) 45); + writer.Write ((short) 45); + writer.Write ((short) 0); + writer.Write ((short) 0); + writer.Write (0); + writer.Write (0); + writer.Write (uint.MaxValue); + writer.Write (uint.MaxValue); + writer.Write ((short) entryNameBytes.Length); + writer.Write ((short) 20); + writer.Write ((short) 0); + writer.Write ((short) 0); + writer.Write ((short) 0); + writer.Write (0); + writer.Write (0); + writer.Write (entryNameBytes); + writer.Write (Zip64ExtraFieldId); + writer.Write ((short) 16); + writer.Write ((long) contents.Length); + writer.Write ((long) contents.Length); + + long centralDirectorySize = stream.Position - centralDirectoryOffset; + long zip64DirectoryOffset = stream.Position; + writer.Write (0x06064b50); + writer.Write ((long) 44); + writer.Write ((short) 45); + writer.Write ((short) 45); + writer.Write (0); + writer.Write (0); + writer.Write ((long) 1); + writer.Write ((long) 1); + writer.Write (centralDirectorySize); + writer.Write (centralDirectoryOffset); + + writer.Write (0x07064b50); + writer.Write (0); + writer.Write (zip64DirectoryOffset); + writer.Write (1); + + writer.Write (0x06054b50); + writer.Write ((short) 0); + writer.Write ((short) 0); + writer.Write (ushort.MaxValue); + writer.Write (ushort.MaxValue); + writer.Write (uint.MaxValue); + writer.Write (uint.MaxValue); + writer.Write ((short) 0); + } + + const short Zip64ExtraFieldId = 0x0001; } } From c9dc0fb3db8d5178e49187fce0cd18951d6d8b5f Mon Sep 17 00:00:00 2001 From: Simon Rozsival Date: Thu, 20 Aug 2026 17:27:51 +0200 Subject: [PATCH 5/5] [build] Simplify ZIP metadata handling Use modern ZipArchiveEntry metadata in the out-of-process task and reduce the netstandard ZIP parser to the ordered name/CRC data required for incremental hashing. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: 4a826a98-ebc1-4ff9-9fa8-4a71854812f4 --- .../Files.cs | 10 +- .../ZipArchiveMetadataReader.cs | 189 ++---------------- .../BuildArchive.cs | 25 +-- .../Xamarin.Android.Build.Tests/BuildTest2.cs | 7 +- .../PackagingTest.cs | 20 +- .../Tasks/BuildArchiveTests.cs | 18 +- .../Tests/BundleToolTests.cs | 8 +- .../ZipArchiveExtensionsTests.cs | 34 ++-- 8 files changed, 74 insertions(+), 237 deletions(-) diff --git a/src/Microsoft.Android.Build.BaseTasks/Files.cs b/src/Microsoft.Android.Build.BaseTasks/Files.cs index c397733e447..d533026e6b2 100644 --- a/src/Microsoft.Android.Build.BaseTasks/Files.cs +++ b/src/Microsoft.Android.Build.BaseTasks/Files.cs @@ -3,7 +3,6 @@ using System; using System.Collections.Generic; -using System.Globalization; using System.IO; using System.IO.Compression; using System.IO.Hashing; @@ -405,9 +404,7 @@ public static bool HasBytesChanged (byte [] bytes, string destination) var hashes = new StringBuilder (); try { - foreach (var item in ZipArchiveMetadataReader.ReadEntries (stream)) { - hashes.AppendFormat (CultureInfo.InvariantCulture, "{0}{1}", item.FullName, item.Crc32); - } + ZipArchiveMetadataReader.AppendHashInput (stream, hashes); } catch { return null; } @@ -423,9 +420,8 @@ public static bool HasBytesChanged (byte [] bytes, string destination) if (File.Exists (filename + ".hash")) return File.ReadAllText (filename + ".hash"); - foreach (var item in ZipArchiveMetadataReader.ReadEntries (filename)) { - hashes.AppendFormat (CultureInfo.InvariantCulture, "{0}{1}", item.FullName, item.Crc32); - } + using var stream = File.OpenRead (filename); + ZipArchiveMetadataReader.AppendHashInput (stream, hashes); } catch { return null; } diff --git a/src/Microsoft.Android.Build.BaseTasks/ZipArchiveMetadataReader.cs b/src/Microsoft.Android.Build.BaseTasks/ZipArchiveMetadataReader.cs index e3cf9abc424..b25b4da9e60 100644 --- a/src/Microsoft.Android.Build.BaseTasks/ZipArchiveMetadataReader.cs +++ b/src/Microsoft.Android.Build.BaseTasks/ZipArchiveMetadataReader.cs @@ -1,91 +1,40 @@ using System; -using System.Collections.Generic; +using System.Globalization; using System.IO; using System.Text; namespace Microsoft.Android.Build.Tasks { - public enum ZipEntryCompressionMethod : ushort - { - Store = 0, - Deflate = 8, - } - - public readonly struct ZipEntryMetadata - { - public string FullName { get; } - public uint Crc32 { get; } - public long CompressedSize { get; } - public long UncompressedSize { get; } - public ZipEntryCompressionMethod CompressionMethod { get; } - - public ZipEntryMetadata (string fullName, uint crc32, long compressedSize, long uncompressedSize, ZipEntryCompressionMethod compressionMethod) - { - FullName = fullName; - Crc32 = crc32; - CompressedSize = compressedSize; - UncompressedSize = uncompressedSize; - CompressionMethod = compressionMethod; - } - } - - public static class ZipArchiveMetadataReader + static class ZipArchiveMetadataReader { const uint EndOfCentralDirectorySignature = 0x06054b50; const uint Zip64EndOfCentralDirectorySignature = 0x06064b50; const uint Zip64EndOfCentralDirectoryLocatorSignature = 0x07064b50; const uint CentralDirectoryFileHeaderSignature = 0x02014b50; - const ushort Zip64ExtraFieldId = 0x0001; const int EndOfCentralDirectoryMinimumSize = 22; const int Zip64EndOfCentralDirectoryLocatorSize = 20; const int CentralDirectoryFileHeaderSize = 46; const int EndOfCentralDirectorySearchWindow = ushort.MaxValue + EndOfCentralDirectoryMinimumSize; static readonly Encoding Cp437 = CreateCp437Encoding (); - public static IReadOnlyDictionary Read (string archivePath) - { - if (archivePath == null) - throw new ArgumentNullException (nameof (archivePath)); - - using var stream = File.OpenRead (archivePath); - return Read (stream); - } - - public static IReadOnlyDictionary Read (Stream stream) - { - var entries = ReadEntries (stream); - var metadata = new Dictionary (entries.Count, StringComparer.Ordinal); - foreach (var entry in entries) { - metadata [entry.FullName] = entry; - } - return metadata; - } - - public static IReadOnlyList ReadEntries (string archivePath) - { - if (archivePath == null) - throw new ArgumentNullException (nameof (archivePath)); - - using var stream = File.OpenRead (archivePath); - return ReadEntries (stream); - } - - public static IReadOnlyList ReadEntries (Stream stream) + public static void AppendHashInput (Stream stream, StringBuilder hashInput) { if (stream == null) throw new ArgumentNullException (nameof (stream)); + if (hashInput == null) + throw new ArgumentNullException (nameof (hashInput)); if (!stream.CanSeek) throw new NotSupportedException ("ZIP metadata requires a seekable stream."); long originalPosition = stream.Position; try { - return ReadEntriesCore (stream); + AppendHashInputCore (stream, hashInput); } finally { stream.Seek (originalPosition, SeekOrigin.Begin); } } - static IReadOnlyList ReadEntriesCore (Stream stream) + static void AppendHashInputCore (Stream stream, StringBuilder hashInput) { long endOfCentralDirectoryOffset = FindEndOfCentralDirectory (stream); using var reader = new BinaryReader (stream, Encoding.UTF8, leaveOpen: true); @@ -124,9 +73,9 @@ static IReadOnlyList ReadEntriesCore (Stream stream) throw new InvalidDataException ("ZIP central directory exceeds the available data."); stream.Seek (centralDirectoryStart, SeekOrigin.Begin); - var entries = new List ((int) entryCount); long centralDirectoryEnd = centralDirectoryStart + centralDirectoryLength; - while ((ulong) entries.Count < entryCount) { + ulong entriesRead = 0; + while (entriesRead < entryCount) { if (stream.Position > centralDirectoryEnd - CentralDirectoryFileHeaderSize) throw new InvalidDataException ("ZIP central directory contains fewer entries than expected."); if (reader.ReadUInt32 () != CentralDirectoryFileHeaderSignature) @@ -135,55 +84,30 @@ static IReadOnlyList ReadEntriesCore (Stream stream) reader.ReadUInt16 (); // version made by reader.ReadUInt16 (); // version needed to extract ushort flags = reader.ReadUInt16 (); - ushort compressionMethod = reader.ReadUInt16 (); + reader.ReadUInt16 (); // compression method reader.ReadUInt16 (); // last mod file time reader.ReadUInt16 (); // last mod file date uint crc32 = reader.ReadUInt32 (); - uint compressedSize = reader.ReadUInt32 (); - uint uncompressedSize = reader.ReadUInt32 (); + reader.ReadUInt32 (); // compressed size + reader.ReadUInt32 (); // uncompressed size ushort fileNameLength = reader.ReadUInt16 (); ushort extraFieldLength = reader.ReadUInt16 (); ushort fileCommentLength = reader.ReadUInt16 (); - ushort diskNumberStart = reader.ReadUInt16 (); + reader.ReadUInt16 (); // disk number start reader.ReadUInt16 (); // internal file attributes reader.ReadUInt32 (); // external file attributes - uint localHeaderOffset = reader.ReadUInt32 (); + reader.ReadUInt32 (); // relative offset of local header var fileNameBytes = reader.ReadBytes (fileNameLength); if (fileNameBytes.Length != fileNameLength) throw new InvalidDataException ("ZIP central directory entry name exceeds the available data."); - var encoding = (flags & (1 << 11)) != 0 ? Encoding.UTF8 : Cp437; - var fullName = encoding.GetString (fileNameBytes); - if (stream.Position > centralDirectoryEnd - extraFieldLength - fileCommentLength) throw new InvalidDataException ("ZIP central directory entry exceeds the available data."); - - var extraField = reader.ReadBytes (extraFieldLength); - if (extraField.Length != extraFieldLength) - throw new InvalidDataException ("ZIP central directory extra field exceeds the available data."); - ReadZip64Sizes ( - extraField, - compressedSize, - uncompressedSize, - localHeaderOffset, - diskNumberStart, - out ulong zip64CompressedSize, - out ulong zip64UncompressedSize - ); - stream.Seek (fileCommentLength, SeekOrigin.Current); - - if (zip64CompressedSize > long.MaxValue || zip64UncompressedSize > long.MaxValue) - throw new InvalidDataException ("ZIP entry is too large."); - entries.Add (new ZipEntryMetadata ( - fullName, - crc32, - (long) zip64CompressedSize, - (long) zip64UncompressedSize, - (ZipEntryCompressionMethod) compressionMethod - )); + var encoding = (flags & (1 << 11)) != 0 ? Encoding.UTF8 : Cp437; + hashInput.AppendFormat (CultureInfo.InvariantCulture, "{0}{1}", encoding.GetString (fileNameBytes), crc32); + stream.Seek (extraFieldLength + fileCommentLength, SeekOrigin.Current); + entriesRead++; } - - return entries; } static bool TryReadZip64EndOfCentralDirectory (Stream stream, BinaryReader reader, long endOfCentralDirectoryOffset, out Zip64DirectoryInfo info) @@ -228,83 +152,6 @@ static bool TryReadZip64EndOfCentralDirectory (Stream stream, BinaryReader reade return true; } - static void ReadZip64Sizes ( - byte [] extraField, - uint compressedSize, - uint uncompressedSize, - uint localHeaderOffset, - ushort diskNumberStart, - out ulong actualCompressedSize, - out ulong actualUncompressedSize) - { - actualCompressedSize = compressedSize; - actualUncompressedSize = uncompressedSize; - bool needsCompressedSize = compressedSize == uint.MaxValue; - bool needsUncompressedSize = uncompressedSize == uint.MaxValue; - bool needsLocalHeaderOffset = localHeaderOffset == uint.MaxValue; - bool needsDiskNumberStart = diskNumberStart == ushort.MaxValue; - if (!needsCompressedSize && !needsUncompressedSize) - return; - - int offset = 0; - while (offset <= extraField.Length - 4) { - ushort headerId = ReadUInt16 (extraField, offset); - ushort dataSize = ReadUInt16 (extraField, offset + 2); - offset += 4; - if (dataSize > extraField.Length - offset) - throw new InvalidDataException ("ZIP central directory extra field is truncated."); - - if (headerId == Zip64ExtraFieldId) { - int end = offset + dataSize; - if (needsUncompressedSize) - actualUncompressedSize = ReadZip64UInt64 (extraField, ref offset, end); - if (needsCompressedSize) - actualCompressedSize = ReadZip64UInt64 (extraField, ref offset, end); - if (needsLocalHeaderOffset) - ReadZip64UInt64 (extraField, ref offset, end); - if (needsDiskNumberStart) - ReadZip64UInt32 (extraField, ref offset, end); - return; - } - - offset += dataSize; - } - - throw new InvalidDataException ("ZIP64 entry size is missing from the central directory extra field."); - } - - static ulong ReadZip64UInt64 (byte [] buffer, ref int offset, int end) - { - if (offset > end - sizeof (long)) - throw new InvalidDataException ("ZIP64 central directory extra field is truncated."); - - ulong value = - buffer [offset] | - ((ulong) buffer [offset + 1] << 8) | - ((ulong) buffer [offset + 2] << 16) | - ((ulong) buffer [offset + 3] << 24) | - ((ulong) buffer [offset + 4] << 32) | - ((ulong) buffer [offset + 5] << 40) | - ((ulong) buffer [offset + 6] << 48) | - ((ulong) buffer [offset + 7] << 56); - offset += sizeof (long); - return value; - } - - static uint ReadZip64UInt32 (byte [] buffer, ref int offset, int end) - { - if (offset > end - sizeof (int)) - throw new InvalidDataException ("ZIP64 central directory extra field is truncated."); - - uint value = - buffer [offset] | - ((uint) buffer [offset + 1] << 8) | - ((uint) buffer [offset + 2] << 16) | - ((uint) buffer [offset + 3] << 24); - offset += sizeof (int); - return value; - } - static long FindEndOfCentralDirectory (Stream stream) { long searchLength = Math.Min (stream.Length, EndOfCentralDirectorySearchWindow); diff --git a/src/Microsoft.Android.Build.Tasks/BuildArchive.cs b/src/Microsoft.Android.Build.Tasks/BuildArchive.cs index 2e41b7378cf..d36f54a92a3 100644 --- a/src/Microsoft.Android.Build.Tasks/BuildArchive.cs +++ b/src/Microsoft.Android.Build.Tasks/BuildArchive.cs @@ -119,7 +119,6 @@ void RefreshEntriesFromInputArchive (ArchiveUpdateSession apk, List exis DateTime lastWriteOutput = File.Exists (ApkOutputPath) ? File.GetLastWriteTimeUtc (ApkOutputPath) : DateTime.MinValue; DateTime lastWriteInput = File.GetLastWriteTimeUtc (ApkInputPath); - var inputMetadata = ZipArchiveMetadataReader.Read (ApkInputPath); using var packaged = ZipArchiveExtensions.OpenZipRead (ApkInputPath); foreach (var entry in packaged.Entries) { @@ -146,12 +145,8 @@ void RefreshEntriesFromInputArchive (ArchiveUpdateSession apk, List exis continue; } - if (!inputMetadata.TryGetValue (entry.FullName, out ZipEntryMetadata metadata)) { - throw new InvalidDataException ($"Unable to read ZIP metadata for '{entry.FullName}' in '{ApkInputPath}'."); - } - var currentEntry = apk.Archive.ReadEntry (entryName, StringComparison.Ordinal); - if (currentEntry != null && metadata.Crc32 == GetEntryCrc32 (currentEntry) && metadata.CompressedSize == currentEntry.CompressedLength) { + if (currentEntry != null && entry.Crc32 == currentEntry.Crc32 && entry.CompressedLength == currentEntry.CompressedLength) { Log.LogDebugMessage ($"Skipping {entryName} from {ApkInputPath} as its up to date."); continue; } @@ -161,7 +156,7 @@ void RefreshEntriesFromInputArchive (ArchiveUpdateSession apk, List exis } Log.LogDebugMessage ($"Refreshing {entryName} from {ApkInputPath}"); - CopyEntryToArchive (apk.Archive, entryName, entry, ToCompressionLevel (metadata.CompressionMethod)); + CopyEntryToArchive (apk.Archive, entryName, entry, ToCompressionLevel (entry.CompressionMethod)); apk.RecordWrite (entry.Length); } } @@ -210,7 +205,7 @@ void AddJarEntryToArchive (ArchiveUpdateSession apk, string diskPath, string arc return; } - if (currentEntry != null && GetEntryCrc32 (currentEntry) == GetEntryCrc32 (jarEntry)) { + if (currentEntry != null && currentEntry.Crc32 == jarEntry.Crc32) { Log.LogDebugMessage ("Skipping {0} from {1} as it is up to date.", jarEntryName, jarFilePath); return; } @@ -299,15 +294,6 @@ static CompressionLevel ToCompressionLevel (ZipCompressionMethod compressionMeth }; } - static CompressionLevel ToCompressionLevel (ZipEntryCompressionMethod compressionMethod) - { - return compressionMethod switch { - ZipEntryCompressionMethod.Store => CompressionLevel.NoCompression, - ZipEntryCompressionMethod.Deflate => CompressionLevel.Optimal, - _ => throw new NotSupportedException ($"Unsupported ZIP compression method: {(ushort) compressionMethod}"), - }; - } - static ZipCompressionMethod GetExistingCompressionMethod (ZipArchiveEntry entry) { return entry.CompressionMethod switch { @@ -317,11 +303,6 @@ static ZipCompressionMethod GetExistingCompressionMethod (ZipArchiveEntry entry) }; } - static uint GetEntryCrc32 (ZipArchiveEntry entry) - { - return entry.Crc32; - } - HashSet ParseUncompressedFileExtensions () { var parsedExtensions = new HashSet (StringComparer.OrdinalIgnoreCase); diff --git a/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/BuildTest2.cs b/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/BuildTest2.cs index 1ab241420c6..3efe873dcb6 100644 --- a/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/BuildTest2.cs +++ b/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/BuildTest2.cs @@ -2,6 +2,7 @@ using System.Collections.Generic; using System.Diagnostics; using System.IO; +using System.IO.Compression; using System.Linq; using System.Reflection; using System.Runtime.InteropServices; @@ -1458,8 +1459,10 @@ public void BuildIncrementingClassName ([Values (AndroidRuntime.CoreCLR, Android var classesZipPath = Path.Combine (Root, b.ProjectDirectory, proj.IntermediateOutputPath, "android", "bin", "classes.zip"); FileAssert.Exists (classesZipPath); - foreach (var entry in ZipArchiveMetadataReader.ReadEntries (classesZipPath)) { - Assert.AreEqual (ZipEntryCompressionMethod.Store, entry.CompressionMethod, $"{entry.FullName} should be stored."); + using (var classesZip = ZipFile.OpenRead (classesZipPath)) { + foreach (var entry in classesZip.Entries) { + Assert.AreEqual (ZipCompressionMethod.Stored, entry.CompressionMethod, $"{entry.FullName} should be stored."); + } } var expectedBuilder = new StringBuilder (); using (var zip = ZipHelper.OpenZip (classesZipPath)) { diff --git a/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/PackagingTest.cs b/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/PackagingTest.cs index e40b592d286..95775c58ab5 100644 --- a/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/PackagingTest.cs +++ b/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/PackagingTest.cs @@ -260,14 +260,13 @@ public void CheckIncludedNativeLibraries ([Values] bool compressNativeLibraries, Assert.IsTrue (b.Build (proj), "build failed"); var apk = Path.Combine (Root, b.ProjectDirectory, proj.OutputPath, $"{proj.PackageName}-Signed.apk"); - var metadata = ZipArchiveMetadataReader.Read (apk); - var expectedCompression = compressNativeLibraries ? ZipEntryCompressionMethod.Deflate : ZipEntryCompressionMethod.Store; + var expectedCompression = compressNativeLibraries ? ZipCompressionMethod.Deflate : ZipCompressionMethod.Stored; using (var zip = ZipHelper.OpenZip (apk)) { var libFiles = zip.Entries.Where (x => x.FullName.StartsWith("lib/", StringComparison.Ordinal) && !x.FullName.Equals("lib/", StringComparison.InvariantCultureIgnoreCase)); var abiPaths = new string[] { "lib/x86_64/" }; foreach (var file in libFiles) { Assert.IsTrue (abiPaths.Any (x => file.FullName.Contains (x)), $"Apk contains an unnesscary lib file: {file.FullName}"); - Assert.AreEqual (expectedCompression, metadata [file.FullName].CompressionMethod, $"{file.FullName} should have been {expectedCompression} in the apk."); + Assert.AreEqual (expectedCompression, file.CompressionMethod, $"{file.FullName} should have been {expectedCompression} in the apk."); } } } @@ -320,20 +319,19 @@ void AssertEmbeddedDSOs (string apk) using (var zip = ZipHelper.OpenZip (apk)) { foreach (var entry in zip.Entries) { if (entry.FullName.EndsWith (".so", StringComparison.Ordinal)) { - AssertCompression (apk, entry, compressed: false); + AssertCompression (entry, compressed: false); } } } } - void AssertCompression (string archivePath, ZipArchiveEntry entry, bool compressed) + void AssertCompression (ZipArchiveEntry entry, bool compressed) { - var compressionMethod = ZipArchiveMetadataReader.Read (archivePath) [entry.FullName].CompressionMethod; if (compressed) { - Assert.AreNotEqual (ZipEntryCompressionMethod.Store, compressionMethod, $"`{entry.FullName}` should be compressed!"); + Assert.AreNotEqual (ZipCompressionMethod.Stored, entry.CompressionMethod, $"`{entry.FullName}` should be compressed!"); Assert.AreNotEqual (entry.Length, entry.CompressedLength, $"`{entry.FullName}` should be compressed!"); } else { - Assert.AreEqual (ZipEntryCompressionMethod.Store, compressionMethod, $"`{entry.FullName}` should be uncompressed!"); + Assert.AreEqual (ZipCompressionMethod.Stored, entry.CompressionMethod, $"`{entry.FullName}` should be uncompressed!"); Assert.AreEqual (entry.Length, entry.CompressedLength, $"`{entry.FullName}` should be uncompressed!"); } } @@ -365,7 +363,7 @@ public void IncrementalCompression ([Values (AndroidRuntime.CoreCLR, AndroidRunt using (var zip = ZipHelper.OpenZip (apk)) { foreach (var entry in zip) { if (entry.FullName.EndsWith (".so", StringComparison.Ordinal) || entry.FullName.EndsWith (".bar", StringComparison.Ordinal)) { - AssertCompression (apk, entry, compressed: true); + AssertCompression (entry, compressed: true); } } } @@ -382,7 +380,7 @@ public void IncrementalCompression ([Values (AndroidRuntime.CoreCLR, AndroidRunt using (var zip = ZipHelper.OpenZip (apk)) { foreach (var entry in zip) { if (entry.FullName.EndsWith (".so", StringComparison.Ordinal) || entry.FullName.EndsWith (".bar", StringComparison.Ordinal)) { - AssertCompression (apk, entry, compressed: false); + AssertCompression (entry, compressed: false); } } } @@ -933,7 +931,7 @@ public void ExtractNativeLibsTrue ([Values (AndroidRuntime.CoreCLR, AndroidRunti using (var zip = ZipHelper.OpenZip (apk)) { foreach (var entry in zip) { if (entry.FullName.EndsWith (".so", StringComparison.Ordinal)) { - AssertCompression (apk, entry, compressed: true); + AssertCompression (entry, compressed: true); } } } diff --git a/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/Tasks/BuildArchiveTests.cs b/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/Tasks/BuildArchiveTests.cs index 68d149fc20c..f1e394ae2d1 100644 --- a/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/Tasks/BuildArchiveTests.cs +++ b/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/Tasks/BuildArchiveTests.cs @@ -181,9 +181,10 @@ public void StoredBundleManifestRelocationPreservesCompressionMethod () Assert.IsTrue (task.RunTask (), "task should have succeeded"); - var metadata = ZipArchiveMetadataReader.Read (bundle); - Assert.IsFalse (metadata.ContainsKey ("AndroidManifest.xml"), "Original manifest entry should be moved."); - Assert.AreEqual (ZipEntryCompressionMethod.Store, metadata ["manifest/AndroidManifest.xml"].CompressionMethod, "Moved manifest should stay stored."); + using var archive = ZipFile.OpenRead (bundle); + Assert.IsNull (archive.GetEntry ("AndroidManifest.xml"), "Original manifest entry should be moved."); + var manifest = archive.GetEntry ("manifest/AndroidManifest.xml") ?? throw new InvalidDataException ("Moved manifest entry is missing."); + Assert.AreEqual (ZipCompressionMethod.Stored, manifest.CompressionMethod, "Moved manifest should stay stored."); } [Test] @@ -206,7 +207,7 @@ public void ZeroByteStoredFileStabilizesAcrossBuilds () Assert.IsTrue (firstRun.RunTask (), "first build should have succeeded"); var firstSnapshot = GetArchiveSnapshot (apk); - Assert.AreEqual (ZipEntryCompressionMethod.Store, ZipArchiveMetadataReader.Read (apk) ["empty.dat"].CompressionMethod, "Entry should be stored."); + Assert.AreEqual (ZipCompressionMethod.Stored, GetCompressionMethod (apk, "empty.dat"), "Entry should be stored."); var secondRun = new BuildArchive { BuildEngine = new MockBuildEngine (TestContext.Out, messages: secondRunMessages), @@ -256,7 +257,7 @@ public void ArchiveRootDirectoryProvidesRelativeEntryPath () Assert.IsTrue (task.RunTask (), "task should have succeeded"); using var archive = ZipFile.OpenRead (apk); archive.AssertEntryContents (apk, "com/example/Main.class", "class"); - Assert.AreEqual (ZipEntryCompressionMethod.Store, ZipArchiveMetadataReader.Read (apk) ["com/example/Main.class"].CompressionMethod); + Assert.AreEqual (ZipCompressionMethod.Stored, GetCompressionMethod (apk, "com/example/Main.class")); } [Test] @@ -317,4 +318,11 @@ static string GetArchiveSnapshot (string path) return $"{entry.FullName}:{Convert.ToBase64String (stream.ToArray ())}"; })); } + + static ZipCompressionMethod GetCompressionMethod (string archivePath, string entryName) + { + using var archive = ZipFile.OpenRead (archivePath); + var entry = archive.GetEntry (entryName) ?? throw new InvalidDataException ($"Entry '{entryName}' is missing from '{archivePath}'."); + return entry.CompressionMethod; + } } diff --git a/tests/MSBuildDeviceIntegration/Tests/BundleToolTests.cs b/tests/MSBuildDeviceIntegration/Tests/BundleToolTests.cs index af90cefa17c..9eb793fc7ea 100644 --- a/tests/MSBuildDeviceIntegration/Tests/BundleToolTests.cs +++ b/tests/MSBuildDeviceIntegration/Tests/BundleToolTests.cs @@ -342,13 +342,15 @@ public void ApkSet () } else { uncompressed.Add (".dll"); } - var metadata = ZipArchiveMetadataReader.Read (stream); - stream.Position = 0; using (var baseApk = new ZipArchive (stream, ZipArchiveMode.Read, leaveOpen: true)) { foreach (var file in baseApk.Entries) { foreach (var ext in uncompressed) { if (file.FullName.EndsWith (ext, StringComparison.OrdinalIgnoreCase)) { - Assert.AreEqual (ZipEntryCompressionMethod.Store, metadata [file.FullName].CompressionMethod, $"{file.FullName} should be uncompressed!"); +#if NET11_0_OR_GREATER + Assert.AreEqual (ZipCompressionMethod.Stored, file.CompressionMethod, $"{file.FullName} should be uncompressed!"); +#else + Assert.AreEqual (file.Length, file.CompressedLength, $"{file.FullName} should be uncompressed!"); +#endif } } } diff --git a/tests/Microsoft.Android.Build.BaseTasks-Tests/ZipArchiveExtensionsTests.cs b/tests/Microsoft.Android.Build.BaseTasks-Tests/ZipArchiveExtensionsTests.cs index 6e28815fab2..974f8ac4e08 100644 --- a/tests/Microsoft.Android.Build.BaseTasks-Tests/ZipArchiveExtensionsTests.cs +++ b/tests/Microsoft.Android.Build.BaseTasks-Tests/ZipArchiveExtensionsTests.cs @@ -41,9 +41,14 @@ public void FixupWindowsPathSeparators_PreservesStoredCompression () archive.FixupWindowsPathSeparators (_ => CompressionLevel.NoCompression); } - var updatedMetadata = ZipArchiveMetadataReader.Read (archivePath); - Assert.IsFalse (updatedMetadata.ContainsKey ("assets\\foo.txt"), "Malformed entry should be removed."); - Assert.AreEqual (ZipEntryCompressionMethod.Store, updatedMetadata ["assets/foo.txt"].CompressionMethod, "Normalized entry should stay stored."); + using var updatedArchive = ZipFile.OpenRead (archivePath); + Assert.IsNull (updatedArchive.GetEntry ("assets\\foo.txt"), "Malformed entry should be removed."); + var updatedEntry = updatedArchive.GetEntry ("assets/foo.txt") ?? throw new InvalidDataException ("Normalized entry is missing."); +#if NET11_0_OR_GREATER + Assert.AreEqual (ZipCompressionMethod.Stored, updatedEntry.CompressionMethod, "Normalized entry should stay stored."); +#else + Assert.AreEqual (updatedEntry.Length, updatedEntry.CompressedLength, "Normalized entry should stay stored."); +#endif } [Test] @@ -100,18 +105,15 @@ public void OpenZip_DisposesFileWhenArchiveCreationFails (ZipArchiveMode mode) } [Test] - public void ReadZip64Metadata () + public void CopyIfZipChanged_Zip64 () { - var archivePath = Path.Combine (TempDirectory, "archive.zip"); - CreateZip64Archive (archivePath); - - var metadata = ZipArchiveMetadataReader.ReadEntries (archivePath); + var source = Path.Combine (TempDirectory, "source.zip"); + var destination = Path.Combine (TempDirectory, "destination.zip"); + CreateZip64Archive (source, crc32: 1); + CreateZip64Archive (destination, crc32: 2); - Assert.AreEqual (1, metadata.Count); - Assert.AreEqual ("entry.txt", metadata [0].FullName); - Assert.AreEqual (3, metadata [0].CompressedSize); - Assert.AreEqual (3, metadata [0].UncompressedSize); - Assert.AreEqual (ZipEntryCompressionMethod.Store, metadata [0].CompressionMethod); + Assert.IsTrue (Files.CopyIfZipChanged (source, destination), "Different ZIP64 entry CRCs should produce different content hashes."); + Assert.IsFalse (Files.CopyIfZipChanged (source, destination), "Identical ZIP64 archives should have matching content hashes."); } static void CreateMalformedEmptyStoredEntryArchive (string path) @@ -166,7 +168,7 @@ static void CreateMalformedEmptyStoredEntryArchive (string path) writer.Write ((short) 0); } - static void CreateZip64Archive (string path) + static void CreateZip64Archive (string path, uint crc32) { const string entryName = "entry.txt"; var entryNameBytes = Encoding.ASCII.GetBytes (entryName); @@ -180,7 +182,7 @@ static void CreateZip64Archive (string path) writer.Write ((short) 0); writer.Write ((short) 0); writer.Write (0); - writer.Write (0); + writer.Write (crc32); writer.Write (contents.Length); writer.Write (contents.Length); writer.Write ((short) entryNameBytes.Length); @@ -195,7 +197,7 @@ static void CreateZip64Archive (string path) writer.Write ((short) 0); writer.Write ((short) 0); writer.Write (0); - writer.Write (0); + writer.Write (crc32); writer.Write (uint.MaxValue); writer.Write (uint.MaxValue); writer.Write ((short) entryNameBytes.Length);