diff --git a/.github/skills/extract-android-assemblies/scripts/extract-android-assemblies.cs b/.github/skills/extract-android-assemblies/scripts/extract-android-assemblies.cs index 7d62036449a..d5f0944ce1d 100644 --- a/.github/skills/extract-android-assemblies/scripts/extract-android-assemblies.cs +++ b/.github/skills/extract-android-assemblies/scripts/extract-android-assemblies.cs @@ -5,10 +5,10 @@ using System; using System.Collections.Generic; using System.IO; +using System.IO.Compression; using Xamarin.Android.AssemblyStore; using Xamarin.Android.Tools; -using Xamarin.Tools.Zip; namespace Xamarin.Android.Tools.DecompressAssemblies { @@ -79,7 +79,7 @@ static bool ExtractIndividualEntries (ZipArchive apk, string filePath, string as { bool retVal = true; int assemblyCount = 0; - foreach (ZipEntry entry in apk) { + foreach (var entry in apk.Entries) { if (!TryGetAssemblyOutputPath (entry.FullName, assembliesPath, nativeLibrariesPath, out string assemblyName)) { continue; } @@ -87,7 +87,7 @@ static bool ExtractIndividualEntries (ZipArchive apk, string filePath, string as assemblyCount++; using (var stream = new MemoryStream ()) { - entry.Extract (stream); + Utils.Extract (entry, stream); stream.Seek (0, SeekOrigin.Begin); string outputFile = GetSafeOutputFile (outputDirectory, assemblyName); using var payload = new MemoryStream (); @@ -166,15 +166,15 @@ static string GetAndroidAbi (AndroidTargetArch arch) static bool HasAssemblyStore (ZipArchive apk, string assembliesPath, string nativeLibrariesPath) { - if (apk.ContainsEntry ($"{assembliesPath}assemblies.blob")) { + if (Utils.ContainsEntry (apk, $"{assembliesPath}assemblies.blob", caseSensitive: true)) { return true; } foreach (AndroidTargetArch arch in targetArchitectures) { string abi = GetAndroidAbi (arch); if ( - apk.ContainsEntry ($"{nativeLibrariesPath}{abi}/libassembly-store.so") || - apk.ContainsEntry ($"{nativeLibrariesPath}{abi}/libassemblies.{abi}.blob.so") + Utils.ContainsEntry (apk, $"{nativeLibrariesPath}{abi}/libassembly-store.so", caseSensitive: true) || + Utils.ContainsEntry (apk, $"{nativeLibrariesPath}{abi}/libassemblies.{abi}.blob.so", caseSensitive: true) ) { return true; } @@ -185,7 +185,7 @@ static bool HasAssemblyStore (ZipArchive apk, string assembliesPath, string nati static bool ExtractFromArchive (string filePath, string assembliesPath, string nativeLibrariesPath, string outputDirectory) { - using (ZipArchive apk = ZipArchive.Open (filePath, FileMode.Open)) { + using (ZipArchive apk = Utils.OpenZip (filePath)) { if (HasAssemblyStore (apk, assembliesPath, nativeLibrariesPath)) { return ExtractAssemblyStores (filePath, outputDirectory); } diff --git a/.github/skills/read-assembly-store/src/AssemblyStore/AssemblyStore.csproj b/.github/skills/read-assembly-store/src/AssemblyStore/AssemblyStore.csproj index d9863dffdbd..eb095fbaf9b 100644 --- a/.github/skills/read-assembly-store/src/AssemblyStore/AssemblyStore.csproj +++ b/.github/skills/read-assembly-store/src/AssemblyStore/AssemblyStore.csproj @@ -12,7 +12,6 @@ - diff --git a/.github/skills/read-assembly-store/src/AssemblyStore/AssemblyStoreExplorer.cs b/.github/skills/read-assembly-store/src/AssemblyStore/AssemblyStoreExplorer.cs index 1d6cc65c7ec..45c69adfbb9 100644 --- a/.github/skills/read-assembly-store/src/AssemblyStore/AssemblyStoreExplorer.cs +++ b/.github/skills/read-assembly-store/src/AssemblyStore/AssemblyStoreExplorer.cs @@ -1,9 +1,9 @@ using System; using System.Collections.Generic; using System.IO; +using System.IO.Compression; using Xamarin.Android.Tools; -using Xamarin.Tools.Zip; namespace Xamarin.Android.AssemblyStore; @@ -127,7 +127,7 @@ public static (IList? explorers, string? errorMessage) Op static (IList? explorers, string? errorMessage) OpenArchive (FileInfo fi, IList paths) { string? errorMessage; - using (var zip = ZipArchive.Open (fi.FullName, FileMode.Open)) { + using (var zip = Utils.OpenZip (fi.FullName)) { (IList? explorers, string? loadError, bool pathsFound) = TryLoad (fi, zip, paths); if (pathsFound) { return (explorers, loadError); @@ -172,13 +172,15 @@ static bool IsV1Store (FileInfo info) var ret = new List (); foreach (string path in paths) { - if (!zip.ContainsEntry (path)) { + if (!Utils.ContainsEntry (zip, path, caseSensitive: true)) { continue; } - ZipEntry entry = zip.ReadEntry (path); + var entry = Utils.ReadEntry (zip, path, caseSensitive: true); + if (entry == null) + continue; var stream = new MemoryStream (); - entry.Extract (stream); + Utils.Extract (entry, stream); ret.Add (new AssemblyStoreExplorer (stream, $"{fi.FullName}!{path}")); } diff --git a/.github/skills/read-assembly-store/src/AssemblyStore/Utils.cs b/.github/skills/read-assembly-store/src/AssemblyStore/Utils.cs index 3b502ebaf62..f8decd14e3f 100644 --- a/.github/skills/read-assembly-store/src/AssemblyStore/Utils.cs +++ b/.github/skills/read-assembly-store/src/AssemblyStore/Utils.cs @@ -1,10 +1,11 @@ using System; using System.IO; using System.Buffers; +using System.IO.Compression; +using System.Linq; using ELFSharp.ELF; using ELFSharp.ELF.Sections; -using Xamarin.Tools.Zip; namespace Xamarin.Android.AssemblyStore; @@ -224,7 +225,7 @@ public static (FileFormat format, FileInfo? info) DetectFileFormat (string path) static FileFormat DetectAndroidArchive (FileInfo info, FileFormat defaultFormat) { - using var zip = ZipArchive.Open (info.FullName, FileMode.Open); + using var zip = OpenZip (info.FullName); if (HasAllEntries (zip, aabZipEntries)) { return FileFormat.Aab; @@ -244,11 +245,38 @@ static FileFormat DetectAndroidArchive (FileInfo info, FileFormat defaultFormat) static bool HasAllEntries (ZipArchive zip, string[] entries) { foreach (string entry in entries) { - if (!zip.ContainsEntry (entry, caseSensitive: true)) { + if (!ContainsEntry (zip, entry, caseSensitive: true)) { return false; } } return true; } + + public static ZipArchive OpenZip (string path) + { + return ZipFile.OpenRead (path); + } + + public static ZipArchive OpenZip (Stream stream, bool leaveOpen = false) + { + return new ZipArchive (stream, ZipArchiveMode.Read, leaveOpen); + } + + public static bool ContainsEntry (ZipArchive archive, string entryName, bool caseSensitive = false) + { + return ReadEntry (archive, entryName, caseSensitive) != null; + } + + public static ZipArchiveEntry? ReadEntry (ZipArchive archive, string entryName, bool caseSensitive = false) + { + var comparison = caseSensitive ? StringComparison.Ordinal : StringComparison.OrdinalIgnoreCase; + return archive.Entries.FirstOrDefault (entry => string.Equals (entry.FullName, entryName, comparison)); + } + + public static void Extract (ZipArchiveEntry entry, Stream destination) + { + using var stream = entry.Open (); + stream.CopyTo (destination); + } } diff --git a/.github/skills/read-assembly-store/src/AssemblyStore/V1/AssemblyStoreExplorer.cs b/.github/skills/read-assembly-store/src/AssemblyStore/V1/AssemblyStoreExplorer.cs index 85ebcd22264..94e9cd854f8 100644 --- a/.github/skills/read-assembly-store/src/AssemblyStore/V1/AssemblyStoreExplorer.cs +++ b/.github/skills/read-assembly-store/src/AssemblyStore/V1/AssemblyStoreExplorer.cs @@ -1,8 +1,8 @@ using System; using System.Collections.Generic; using System.IO; - -using Xamarin.Tools.Zip; +using System.IO.Compression; +using Xamarin.Android.AssemblyStore; namespace Xamarin.Android.AssemblyStore.V1 { @@ -215,20 +215,20 @@ void ReadStoreSetFromArchive (string baseName, string archivePath, string extens } basePathInArchive = $"{basePathInArchive}/{baseName}."; - using (ZipArchive archive = ZipArchive.Open (archivePath, FileMode.Open)) { + using (ZipArchive archive = Utils.OpenZip (archivePath)) { ReadStoreSetFromArchive (archive, basePathInArchive); } } void ReadStoreSetFromArchive (ZipArchive archive, string basePathInArchive) { - foreach (ZipEntry entry in archive) { + foreach (var entry in archive.Entries) { if (!entry.FullName.StartsWith (basePathInArchive, StringComparison.Ordinal)) { continue; } using (var stream = new MemoryStream ()) { - entry.Extract (stream); + Utils.Extract (entry, stream); if (entry.FullName.EndsWith (".blob", StringComparison.Ordinal)) { AddStore (new AssemblyStoreReader (stream, GetStoreArch (entry.FullName), keepStoreInMemory)); diff --git a/.github/skills/update-tpn/SKILL.md b/.github/skills/update-tpn/SKILL.md index e87dde58734..430d32bb47e 100644 --- a/.github/skills/update-tpn/SKILL.md +++ b/.github/skills/update-tpn/SKILL.md @@ -82,7 +82,6 @@ Search `.csproj` files for `` elements. Current third-party Nu |---------|------------|-------------| | ELFSharp | KonradKuczynski/ELFSharp | https://elfsharp.it/ (MIT + LLVM) | | K4os.Compression.LZ4 | MiloszKrajewski/K4os.Compression.LZ4 | https://github.com/MiloszKrajewski/K4os.Compression.LZ4/ (MIT) | -| Xamarin.LibZipSharp | xamarin/LibZipSharp | https://github.com/xamarin/LibZipSharp/ (MIT) | | Irony | IronyProject/Irony | https://github.com/IronyProject/Irony (MIT) | | Newtonsoft.Json | JamesNK/Newtonsoft.Json | https://github.com/JamesNK/Newtonsoft.Json (MIT) | | NuGet.ProjectModel | NuGet/NuGet.Client | https://github.com/NuGet/NuGet.Client (Apache 2.0) | @@ -111,13 +110,6 @@ These are downloaded and shipped with the SDK: | r8 | google/r8 | https://r8.googlesource.com/r8/ (BSD-3-Clause) | | binutils | gnu/binutils | https://sourceware.org/git/?p=binutils-gdb.git;a=tree;hb=HEAD (GPLv3) | -#### libzip (via LibZipSharp NuGet) -LibZipSharp bundles libzip internally: - -| Source | Name in TPN | License Location | -|--------|------------|-----------------| -| libzip (in LibZipSharp NuGet) | nih-at/libzip | LibZipSharp NuGet `Licences/libzip/LICENSE` or https://github.com/nih-at/libzip/ (BSD-3-Clause) | - ### Step 2 — Cross-reference Compare the inventory against the current entries in `THIRD-PARTY-NOTICES.TXT`: diff --git a/Directory.Build.props b/Directory.Build.props index baf6a832968..60beac9242b 100644 --- a/Directory.Build.props +++ b/Directory.Build.props @@ -39,7 +39,6 @@ - 3.3.0 1.0.0 13.0.3 5.4.0 diff --git a/THIRD-PARTY-NOTICES.TXT b/THIRD-PARTY-NOTICES.TXT index 5759a3f7d26..533bcb53f48 100644 --- a/THIRD-PARTY-NOTICES.TXT +++ b/THIRD-PARTY-NOTICES.TXT @@ -30,10 +30,8 @@ implication, estoppel or otherwise. 19. MiloszKrajewski/K4os.Compression.LZ4 (https://github.com/MiloszKrajewski/K4os.Compression.LZ4/) 20. mono/cecil (https://github.com/mono/cecil/) 21. mono/linker (https://github.com/mono/linker/) -22. nih-at/libzip (https://github.com/nih-at/libzip/) -23. NuGet/NuGet.Client (https://github.com/NuGet/NuGet.Client) -24. tessil/robin-map (https://github.com/Tessil/robin-map) -25. xamarin/LibZipSharp (https://github.com/xamarin/LibZipSharp/) +22. NuGet/NuGet.Client (https://github.com/NuGet/NuGet.Client) +23. tessil/robin-map (https://github.com/Tessil/robin-map) %% Android API documentation NOTICES AND INFORMATION BEGIN HERE ================================================================ @@ -2401,45 +2399,6 @@ SOFTWARE. END OF mono/linker NOTICES AND INFORMATION -%% nih-at/libzip NOTICES AND INFORMATION BEGIN HERE -=================================================== -Copyright (C) 1999-2020 Dieter Baron and Thomas Klausner - -The authors can be contacted at - -Redistribution and use in source and binary forms, with or without -modification, are permitted provided that the following conditions -are met: - -1. Redistributions of source code must retain the above copyright - notice, this list of conditions and the following disclaimer. - -2. Redistributions in binary form must reproduce the above copyright - notice, this list of conditions and the following disclaimer in - the documentation and/or other materials provided with the - distribution. - -3. The names of the authors may not be used to endorse or promote - products derived from this software without specific prior - written permission. - -THIS SOFTWARE IS PROVIDED BY THE AUTHORS ``AS IS'' AND ANY EXPRESS -OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED -WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE -ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY -DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL -DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE -GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS -INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER -IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR -OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN -IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - - -=================================================== -END OF nih-at/libzip NOTICES AND INFORMATION - - %% NuGet/NuGet.Client NOTICES AND INFORMATION BEGIN HERE ======================================================== Copyright (c) .NET Foundation and Contributors. @@ -2491,33 +2450,3 @@ SOFTWARE. ====================================================== END OF tessil/robin-map NOTICES AND INFORMATION - -%% xamarin/LibZipSharp NOTICES AND INFORMATION BEGIN HERE -========================================================= -The MIT License (MIT) - -Copyright (c) 2016 Marek Habersack - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all -copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -SOFTWARE. - - -========================================================= -END OF xamarin/LibZipSharp NOTICES AND INFORMATION - - diff --git a/build-tools/Xamarin.Android.Tools.BootstrapTasks/Xamarin.Android.Tools.BootstrapTasks.csproj b/build-tools/Xamarin.Android.Tools.BootstrapTasks/Xamarin.Android.Tools.BootstrapTasks.csproj index b84f0c3d1de..bce2837a4af 100644 --- a/build-tools/Xamarin.Android.Tools.BootstrapTasks/Xamarin.Android.Tools.BootstrapTasks.csproj +++ b/build-tools/Xamarin.Android.Tools.BootstrapTasks/Xamarin.Android.Tools.BootstrapTasks.csproj @@ -4,7 +4,6 @@ $(DotNetStableTargetFramework) - true $(BootstrapOutputDirectory) diff --git a/build-tools/Xamarin.Android.Tools.BootstrapTasks/Xamarin.Android.Tools.BootstrapTasks/CheckApiCompatibility.cs b/build-tools/Xamarin.Android.Tools.BootstrapTasks/Xamarin.Android.Tools.BootstrapTasks/CheckApiCompatibility.cs index 44e726086be..96b1a3f3880 100644 --- a/build-tools/Xamarin.Android.Tools.BootstrapTasks/Xamarin.Android.Tools.BootstrapTasks/CheckApiCompatibility.cs +++ b/build-tools/Xamarin.Android.Tools.BootstrapTasks/Xamarin.Android.Tools.BootstrapTasks/CheckApiCompatibility.cs @@ -2,10 +2,11 @@ using System.Collections.Generic; using System.Diagnostics; using System.IO; +using System.IO.Compression; using System.Linq; using Microsoft.Build.Framework; using Microsoft.Build.Utilities; -using Xamarin.Tools.Zip; +using Microsoft.Android.Build.Tasks; namespace Xamarin.Android.Tools.BootstrapTasks { @@ -133,11 +134,12 @@ public override bool Execute () var zipFiles = Directory.GetFiles (referenceContractPath.Parent.FullName, "*.zip"); foreach (var zipFile in zipFiles) { var zipDateTime = File.GetLastWriteTimeUtc (zipFile); - using (var zip = ZipArchive.Open (zipFile, FileMode.Open)) { - foreach (var entry in zip) { - var path = Path.Combine (referenceContractPath.FullName, entry.NativeFullName); + using (var zip = ZipArchiveExtensions.OpenZipRead (zipFile)) { + foreach (var entry in zip.Entries) { + var path = Path.Combine (referenceContractPath.FullName, entry.FullName.Replace ('/', Path.DirectorySeparatorChar)); if (!File.Exists (path) || File.GetLastWriteTimeUtc (path) < zipDateTime) { Log.LogMessage ($"Extracting: {path}"); + Directory.CreateDirectory (Path.GetDirectoryName (path)); using (var fileStream = File.Create (path)) { entry.Extract (fileStream); } diff --git a/build-tools/Xamarin.Android.Tools.BootstrapTasks/Xamarin.Android.Tools.BootstrapTasks/UnzipDirectoryChildren.cs b/build-tools/Xamarin.Android.Tools.BootstrapTasks/Xamarin.Android.Tools.BootstrapTasks/UnzipDirectoryChildren.cs index ad0d915b7be..8e69f1db850 100644 --- a/build-tools/Xamarin.Android.Tools.BootstrapTasks/Xamarin.Android.Tools.BootstrapTasks/UnzipDirectoryChildren.cs +++ b/build-tools/Xamarin.Android.Tools.BootstrapTasks/Xamarin.Android.Tools.BootstrapTasks/UnzipDirectoryChildren.cs @@ -1,8 +1,9 @@ using Microsoft.Build.Framework; using System.IO; +using System.IO.Compression; using System.Text; using System.Collections.Generic; -using Xamarin.Tools.Zip; +using Microsoft.Android.Build.Tasks; using MTask = Microsoft.Build.Utilities.Task; using TTask = System.Threading.Tasks.Task; @@ -70,17 +71,22 @@ void ExtractFile (string sourceFile, string relativeDestDir, string destinationF { relativeDestDir = relativeDestDir?.Replace ('\\', Path.DirectorySeparatorChar); - using (var zip = ZipArchive.Open (sourceFile, FileMode.Open)) { - foreach (var entry in zip) { - if (!entry.IsDirectory) { + using (var zip = ZipArchiveExtensions.OpenZipRead (sourceFile, encoding)) { + foreach (var entry in zip.Entries) { + if (!entry.IsDirectory ()) { if (filesToExtract.Count > 0 && !filesToExtract.Contains (Path.GetFileName (entry.FullName))) continue; - var entryPath = entry.NativeFullName; + var entryPath = entry.FullName.Replace ('/', Path.DirectorySeparatorChar); if (!NoSubdirectory) { - entryPath = entryPath.Substring (entryPath.IndexOf (Path.DirectorySeparatorChar) + 1); + int separatorIndex = entryPath.IndexOf (Path.DirectorySeparatorChar); + if (separatorIndex < 0) + continue; + entryPath = entryPath.Substring (separatorIndex + 1); + if (entryPath.Length == 0) + continue; } var destinationPath = Path.Combine (destinationFolder, relativeDestDir, entryPath); - Log.LogMessage (MessageImportance.Low, $"Extracting {entry.NativeFullName} to {destinationPath}"); + Log.LogMessage (MessageImportance.Low, $"Extracting {entry.FullName} to {destinationPath}"); entry.Extract (Path.GetDirectoryName (destinationPath), Path.GetFileName (destinationPath)); } } @@ -88,4 +94,3 @@ void ExtractFile (string sourceFile, string relativeDestDir, string destinationF } } } - diff --git a/build-tools/Xamarin.Android.Tools.BootstrapTasks/Xamarin.Android.Tools.BootstrapTasks/Zip.cs b/build-tools/Xamarin.Android.Tools.BootstrapTasks/Xamarin.Android.Tools.BootstrapTasks/Zip.cs index a2c1cbdca69..f93daf4c523 100644 --- a/build-tools/Xamarin.Android.Tools.BootstrapTasks/Xamarin.Android.Tools.BootstrapTasks/Zip.cs +++ b/build-tools/Xamarin.Android.Tools.BootstrapTasks/Xamarin.Android.Tools.BootstrapTasks/Zip.cs @@ -5,8 +5,7 @@ using Microsoft.Build.Framework; using Microsoft.Build.Utilities; - -using Xamarin.Tools.Zip; +using Microsoft.Android.Build.Tasks; using IOFile = System.IO.File; @@ -43,7 +42,7 @@ public override bool Execute () prefix += Path.DirectorySeparatorChar; } - using (var zip = ZipArchive.Open (File.ItemSpec, FileMode.OpenOrCreate)) { + using (var zip = ZipArchiveExtensions.OpenZipUpdate (File.ItemSpec)) { if (Entries == null) return !Log.HasLoggedErrors; foreach (var entry in Entries) { @@ -57,17 +56,13 @@ public override bool Execute () if (prefix != null && entryDir.StartsWith (prefix, StringComparison.OrdinalIgnoreCase)) { zipDir = entryDir.Substring (prefix.Length); } - if (string.IsNullOrEmpty (zipDir)) { - // JonP can't figure out how to actually clear the archive directory name - // using AddFileToDirectory(). This works as desired. - zip.AddFile (entryPath, Path.GetFileName (entryPath)); - } else { - zip.AddFileToDirectory (entryPath, zipDir, useFileDirectory: false); - } + var archivePath = string.IsNullOrEmpty (zipDir) + ? Path.GetFileName (entryPath) + : Path.Combine (zipDir, Path.GetFileName (entryPath)); + zip.AddFile (entryPath, archivePath.Replace ('\\', '/')); } } return !Log.HasLoggedErrors; } } } - diff --git a/build-tools/create-packs/SignList.xml b/build-tools/create-packs/SignList.xml index a71ea2c7fcf..352da582aa5 100644 --- a/build-tools/create-packs/SignList.xml +++ b/build-tools/create-packs/SignList.xml @@ -1,7 +1,6 @@ - diff --git a/build-tools/debian-metadata/rules b/build-tools/debian-metadata/rules index 0db5a1752d6..0a73bf108b2 100755 --- a/build-tools/debian-metadata/rules +++ b/build-tools/debian-metadata/rules @@ -21,7 +21,6 @@ override_dh_install: rm -f bin/*/lib/xamarin.android/xbuild/Xamarin/Android/opt.exe rm -f bin/*/lib/xamarin.android/xbuild/Xamarin/Android/aapt2.exe rm -f bin/*/lib/xamarin.android/xbuild/Xamarin/Android/libwinpthread-1.dll - rm -f bin/*/lib/xamarin.android/xbuild/Xamarin/Android/libZipSharpNative-*.dll rm -f bin/*/lib/xamarin.android/xbuild/Xamarin/Android/runtimes/*/libMono.Unix.so dh_install @@ -41,7 +40,6 @@ override_dh_clideps: --exclude-moduleref=libfam.so.0 \ --exclude-moduleref=libgamin-1.so.0 \ --exclude-moduleref=libmono-btls-shared \ - --exclude-moduleref=libZipSharpNative \ --exclude-moduleref=lzo.dll \ --exclude-moduleref=Microsoft.VisualStudio.Setup.Configuration.Native.dll \ --exclude-moduleref=mscoree.dll \ diff --git a/build-tools/installers/create-installers.targets b/build-tools/installers/create-installers.targets index b19283e27ce..df517785566 100644 --- a/build-tools/installers/create-installers.targets +++ b/build-tools/installers/create-installers.targets @@ -96,9 +96,6 @@ <_MSBuildFiles Include="$(MicrosoftAndroidSdkOutDir)java-interop.jar" /> <_MSBuildFiles Include="$(MicrosoftAndroidSdkOutDir)java-source-utils.jar" /> <_MSBuildFiles Include="$(MicrosoftAndroidSdkOutDir)LayoutBinding.cs" /> - <_MSBuildFiles Include="$(MicrosoftAndroidSdkOutDir)libZipSharp.dll" /> - <_MSBuildFiles Include="@(_LocalizationLanguages->'$(MicrosoftAndroidSdkOutDir)%(Identity)\libZipSharp.resources.dll')" /> - <_MSBuildFiles Include="$(MicrosoftAndroidSdkOutDir)libZipSharp.pdb" /> <_MSBuildFiles Include="$(MicrosoftAndroidSdkOutDir)Mono.Unix.dll" /> <_MSBuildFiles Include="$(MicrosoftAndroidSdkOutDir)Mono.Unix.pdb" /> <_MSBuildFiles Include="$(MicrosoftAndroidSdkOutDir)Microsoft.Android.Build.BaseTasks.dll" /> @@ -182,12 +179,6 @@ <_MSBuildTargetsSrcFiles Include="$(MSBuildTargetsSrcDir)\Xamarin.Android.AvailableItems.targets" /> - <_MSBuildFilesWin Include="$(MicrosoftAndroidSdkOutDir)x86\libZipSharpNative-*.dll" /> - <_MSBuildFilesWin Include="$(MicrosoftAndroidSdkOutDir)x86\libZipSharpNative-*.pdb" /> - <_MSBuildFilesWin Include="$(MicrosoftAndroidSdkOutDir)x64\libZipSharpNative-*.dll" /> - <_MSBuildFilesWin Include="$(MicrosoftAndroidSdkOutDir)x64\libZipSharpNative-*.pdb" /> - <_MSBuildFilesWin Include="$(MicrosoftAndroidSdkOutDir)arm64\libZipSharpNative-*.dll" /> - <_MSBuildFilesWin Include="$(MicrosoftAndroidSdkOutDir)arm64\libZipSharpNative-*.pdb" /> <_MSBuildFilesWin Include="$(MicrosoftAndroidSdkOutDir)aapt2.exe" /> <_MSBuildFilesWin Include="$(MicrosoftAndroidSdkOutDir)binutils\bin\as.exe" /> <_MSBuildFilesWin Include="$(MicrosoftAndroidSdkOutDir)binutils\bin\ld.exe" /> @@ -228,7 +219,6 @@ <_MSBuildFilesUnix Include="$(MicrosoftAndroidSdkOutDir)$(HostOS)\jit-times" Permission="755" /> <_MSBuildFilesUnix Include="$(MicrosoftAndroidSdkOutDir)$(HostOS)\mono.config" /> <_MSBuildFilesUnixSignAndHarden Include="$(MicrosoftAndroidSdkOutDir)$(HostOS)\aapt2" /> - <_MSBuildFilesUnixSign Include="$(MicrosoftAndroidSdkOutDir)libZipSharpNative-*.$(LibExtension)" /> <_MSBuildFilesUnixSign Include="$(MicrosoftAndroidSdkOutDir)libMono.Unix.$(LibExtension)" /> diff --git a/src-ThirdParty/android-platform-tools-base/PackagingUtils.cs b/src-ThirdParty/android-platform-tools-base/PackagingUtils.cs index ee48570cc02..1f4efa60212 100644 --- a/src-ThirdParty/android-platform-tools-base/PackagingUtils.cs +++ b/src-ThirdParty/android-platform-tools-base/PackagingUtils.cs @@ -33,7 +33,7 @@ internal class PackagingUtils /// /// Checks if a zip entry is valid for packaging into the .apk as standard Java resource. /// - /// the name of the zip entry from Xamarin.Tools.Zip.ZipEntry.FullName. + /// the full name of the zip entry. /// true if the entry is valid for packaging. public static bool CheckEntryForPackaging (string entryName) { diff --git a/src/Microsoft.Android.Build.BaseTasks/Files.cs b/src/Microsoft.Android.Build.BaseTasks/Files.cs index 514448df475..d533026e6b2 100644 --- a/src/Microsoft.Android.Build.BaseTasks/Files.cs +++ b/src/Microsoft.Android.Build.BaseTasks/Files.cs @@ -3,13 +3,13 @@ using System; using System.Collections.Generic; -using System.Globalization; using System.IO; +using System.IO.Compression; +using System.IO.Hashing; using System.Linq; using System.Runtime.CompilerServices; using System.Security.Cryptography; using System.Text; -using Xamarin.Tools.Zip; using Microsoft.Build.Utilities; using System.Threading; using System.Runtime.InteropServices; @@ -401,49 +401,42 @@ public static bool HasBytesChanged (byte [] bytes, string destination) static string? HashZip (Stream stream) { - string hashes = String.Empty; + var hashes = new StringBuilder (); try { - using (var zip = ZipArchive.Open (stream)) { - foreach (var item in zip) { - hashes += String.Format (CultureInfo.InvariantCulture, "{0}{1}", item.FullName, item.CRC); - } - } + ZipArchiveMetadataReader.AppendHashInput (stream, hashes); } catch { return null; } - return hashes; + return hashes.ToString (); } static string? HashZip (string filename) { - string hashes = String.Empty; + var hashes = new StringBuilder (); try { // check cache if (File.Exists (filename + ".hash")) return File.ReadAllText (filename + ".hash"); - using (var zip = ReadZipFile (filename)) { - foreach (var item in zip) { - hashes += String.Format (CultureInfo.InvariantCulture, "{0}{1}", item.FullName, item.CRC); - } - } + using var stream = File.OpenRead (filename); + ZipArchiveMetadataReader.AppendHashInput (stream, hashes); } catch { return null; } - return hashes; + return hashes.ToString (); } public static ZipArchive ReadZipFile (string filename, bool strictConsistencyChecks = false) { - return ZipArchive.Open (filename, FileMode.Open, strictConsistencyChecks: strictConsistencyChecks); + return ZipArchiveExtensions.OpenZipRead (filename); } - public static bool ZipAny (string filename, Func filter) + public static bool ZipAny (string filename, Func filter) { using (var zip = ReadZipFile (filename)) { - return zip.Any (filter); + return zip.Entries.Any (filter); } } @@ -458,15 +451,15 @@ public static bool ExtractAll (ZipArchive zip, string destination, Action? deleteCallback = null, Func? skipCallback = null, TaskLoggingHelper? log = null) { int i = 0; - int total = (int)zip.EntryCount; + int total = zip.Entries.Count; bool updated = false; var files = new HashSet (); var memoryStream = MemoryStreamPool.Shared.Rent (); var fullDestination = Path.GetFullPath (destination + Path.DirectorySeparatorChar); try { - foreach (var entry in zip) { + foreach (var entry in zip.Entries) { progressCallback?.Invoke (i++, total); - if (entry.IsDirectory) + if (entry.IsDirectory ()) continue; if (entry.FullName.Contains ("/__MACOSX/") || entry.FullName.EndsWith ("/__MACOSX", StringComparison.OrdinalIgnoreCase) || diff --git a/src/Microsoft.Android.Build.BaseTasks/HexUtilities.cs b/src/Microsoft.Android.Build.BaseTasks/HexUtilities.cs index ea69f86dfeb..64b83fc9106 100644 --- a/src/Microsoft.Android.Build.BaseTasks/HexUtilities.cs +++ b/src/Microsoft.Android.Build.BaseTasks/HexUtilities.cs @@ -11,7 +11,7 @@ namespace Microsoft.Android.Build.Tasks /// /// This file is also linked into Microsoft.Android.Sdk.TrimmableTypeMap, which /// deliberately does not reference Microsoft.Android.Build.BaseTasks (that would drag - /// Microsoft.Build.*, LibZipSharp, K4os.LZ4 and Mono.Unix into it). Only the copy compiled + /// Microsoft.Build.*, System.IO.Hashing, K4os.LZ4 and Mono.Unix into it). Only the copy compiled /// into Microsoft.Android.Build.BaseTasks is public; the linked copy stays /// internal, otherwise Xamarin.Android.Build.Tasks — which references both /// assemblies — fails with CS0433. diff --git a/src/Microsoft.Android.Build.BaseTasks/MSBuildReferences.projitems b/src/Microsoft.Android.Build.BaseTasks/MSBuildReferences.projitems index e5ccd74facc..ed8c779f8a0 100644 --- a/src/Microsoft.Android.Build.BaseTasks/MSBuildReferences.projitems +++ b/src/Microsoft.Android.Build.BaseTasks/MSBuildReferences.projitems @@ -6,7 +6,6 @@ 18.7.1 10.0.4 - 3.3.0 7.1.0-final.1.21458.1 @@ -17,7 +16,6 @@ - diff --git a/src/Microsoft.Android.Build.BaseTasks/Microsoft.Android.Build.BaseTasks.csproj b/src/Microsoft.Android.Build.BaseTasks/Microsoft.Android.Build.BaseTasks.csproj index 8f3f3b7be8e..830a52662bd 100644 --- a/src/Microsoft.Android.Build.BaseTasks/Microsoft.Android.Build.BaseTasks.csproj +++ b/src/Microsoft.Android.Build.BaseTasks/Microsoft.Android.Build.BaseTasks.csproj @@ -34,6 +34,7 @@ + diff --git a/src/Microsoft.Android.Build.BaseTasks/ZipArchiveExtensions.cs b/src/Microsoft.Android.Build.BaseTasks/ZipArchiveExtensions.cs new file mode 100644 index 00000000000..6ff12acbf4f --- /dev/null +++ b/src/Microsoft.Android.Build.BaseTasks/ZipArchiveExtensions.cs @@ -0,0 +1,271 @@ +using System; +using System.IO; +using System.IO.Compression; +using System.Linq; +using System.Text; + +namespace Microsoft.Android.Build.Tasks +{ + public static class ZipArchiveExtensions + { + public static ZipArchive OpenZipRead (string archivePath, Encoding? entryNameEncoding = null) + { + if (archivePath == null) + throw new ArgumentNullException (nameof (archivePath)); + + return OpenZip (archivePath, FileMode.Open, FileAccess.Read, ZipArchiveMode.Read, entryNameEncoding); + } + + public static ZipArchive OpenZipUpdate (string archivePath, FileMode fileMode = FileMode.OpenOrCreate, Encoding? entryNameEncoding = null) + { + if (archivePath == null) + throw new ArgumentNullException (nameof (archivePath)); + + switch (fileMode) { + case FileMode.Open: + case FileMode.OpenOrCreate: + break; + default: + throw new ArgumentOutOfRangeException (nameof (fileMode), fileMode, null); + } + + return OpenZip (archivePath, fileMode, FileAccess.ReadWrite, ZipArchiveMode.Update, entryNameEncoding); + } + + public static ZipArchive CreateZip (string archivePath, FileMode fileMode = FileMode.Create, Encoding? entryNameEncoding = null) + { + if (archivePath == null) + throw new ArgumentNullException (nameof (archivePath)); + + switch (fileMode) { + case FileMode.Create: + case FileMode.CreateNew: + case FileMode.Truncate: + break; + default: + throw new ArgumentOutOfRangeException (nameof (fileMode), fileMode, null); + } + + return OpenZip (archivePath, fileMode, FileAccess.ReadWrite, ZipArchiveMode.Create, entryNameEncoding); + } + + public static ZipArchive OpenZip (Stream stream, ZipArchiveMode mode = ZipArchiveMode.Read, bool leaveOpen = false, Encoding? entryNameEncoding = null) + { + if (stream == null) + throw new ArgumentNullException (nameof (stream)); + + return new ZipArchive (stream, mode, leaveOpen, entryNameEncoding); + } + + public static bool ContainsEntry (this ZipArchive archive, string entryName, StringComparison comparison = StringComparison.Ordinal) + => archive.ReadEntry (entryName, comparison) != null; + + public static ZipArchiveEntry? ReadEntry (this ZipArchive archive, string entryName, StringComparison comparison = StringComparison.Ordinal) + { + if (archive == null) + throw new ArgumentNullException (nameof (archive)); + if (entryName == null) + throw new ArgumentNullException (nameof (entryName)); + + if (comparison == StringComparison.Ordinal) + return archive.GetEntry (entryName); + + return archive.Entries.FirstOrDefault (entry => string.Equals (entry.FullName, entryName, comparison)); + } + + public static bool IsDirectory (this ZipArchiveEntry entry) + { + if (entry == null) + throw new ArgumentNullException (nameof (entry)); + + return entry.FullName.EndsWith ("/", StringComparison.Ordinal) || entry.FullName.EndsWith ("\\", StringComparison.Ordinal); + } + + public static void Extract (this ZipArchiveEntry entry, Stream destination) + { + if (entry == null) + throw new ArgumentNullException (nameof (entry)); + if (destination == null) + throw new ArgumentNullException (nameof (destination)); + + // Some Android archives encode empty stored entries with non-zero compressed data. + // ZipArchive validates that data when opening the entry, even though there is + // nothing to extract. + if (entry.Length == 0) + return; + + using var source = entry.Open (); + source.CopyTo (destination); + } + + public static void Extract (this ZipArchiveEntry entry, string destinationDirectory, string? destinationFileName = null) + { + if (entry == null) + throw new ArgumentNullException (nameof (entry)); + if (destinationDirectory == null) + throw new ArgumentNullException (nameof (destinationDirectory)); + + var fileName = destinationFileName ?? entry.FullName.Replace ('/', Path.DirectorySeparatorChar); + var destinationPath = Path.Combine (destinationDirectory, fileName); + var destinationFolder = Path.GetDirectoryName (destinationPath); + if (!string.IsNullOrEmpty (destinationFolder)) + Directory.CreateDirectory (destinationFolder); + + using var output = File.Create (destinationPath); + entry.Extract (output); + } + + public static void AddEntry (this ZipArchive archive, string entryName, string contents, Encoding encoding, CompressionLevel compressionLevel = CompressionLevel.Optimal) + { + if (archive == null) + throw new ArgumentNullException (nameof (archive)); + if (entryName == null) + throw new ArgumentNullException (nameof (entryName)); + if (contents == null) + throw new ArgumentNullException (nameof (contents)); + if (encoding == null) + throw new ArgumentNullException (nameof (encoding)); + + DeleteEntry (archive, entryName); + var entry = archive.CreateEntry (entryName, compressionLevel); + using var writer = new StreamWriter (entry.Open (), encoding); + writer.Write (contents); + } + + public static void AddStream (this ZipArchive archive, Stream source, string entryName, CompressionLevel compressionLevel = CompressionLevel.Optimal) + { + if (archive == null) + throw new ArgumentNullException (nameof (archive)); + if (source == null) + throw new ArgumentNullException (nameof (source)); + if (entryName == null) + throw new ArgumentNullException (nameof (entryName)); + + DeleteEntry (archive, entryName); + var entry = archive.CreateEntry (entryName, compressionLevel); + using var destination = entry.Open (); + source.CopyTo (destination); + } + + public static void AddFile (this ZipArchive archive, string filePath, string entryName, CompressionLevel compressionLevel = CompressionLevel.Optimal) + { + if (archive == null) + throw new ArgumentNullException (nameof (archive)); + if (filePath == null) + throw new ArgumentNullException (nameof (filePath)); + if (entryName == null) + throw new ArgumentNullException (nameof (entryName)); + + DeleteEntry (archive, entryName); + ZipFileExtensions.CreateEntryFromFile (archive, filePath, entryName, compressionLevel); + } + + public static void AddDirectory (this ZipArchive archive, string directory, string directoryPathInArchive = "", CompressionLevel compressionLevel = CompressionLevel.Optimal) + { + if (archive == null) + throw new ArgumentNullException (nameof (archive)); + if (directory == null) + throw new ArgumentNullException (nameof (directory)); + + directory = directory.Replace ('/', Path.DirectorySeparatorChar).Replace ('\\', Path.DirectorySeparatorChar); + directory = Path.GetFullPath (directory); + if (directory [directory.Length - 1] == Path.DirectorySeparatorChar) + directory = directory.Substring (0, directory.Length - 1); + + AddDirectoryContents (directory); + + void AddDirectoryContents (string currentDirectory) + { + foreach (var filePath in Directory.GetFiles (currentDirectory, "*.*", SearchOption.TopDirectoryOnly).OrderBy (path => path, StringComparer.Ordinal)) { + var fileInfo = new FileInfo (filePath); + if ((fileInfo.Attributes & FileAttributes.Hidden) != 0) + continue; + + var relativePath = filePath.Substring (directory.Length).TrimStart (Path.DirectorySeparatorChar, Path.AltDirectorySeparatorChar).Replace ('\\', '/'); + var entryName = string.IsNullOrEmpty (directoryPathInArchive) ? relativePath : $"{directoryPathInArchive.TrimEnd ('/')}/{relativePath}"; + archive.AddFile (filePath, entryName, compressionLevel); + } + + foreach (var childDirectory in Directory.GetDirectories (currentDirectory, "*", SearchOption.TopDirectoryOnly).OrderBy (path => path, StringComparer.Ordinal)) { + var directoryInfo = new DirectoryInfo (childDirectory); + if ((directoryInfo.Attributes & FileAttributes.Hidden) != 0) + continue; + + AddDirectoryContents (childDirectory); + } + } + } + + public static bool MoveEntry (this ZipArchive archive, string oldEntryName, string newEntryName, CompressionLevel compressionLevel) + { + if (archive == null) + throw new ArgumentNullException (nameof (archive)); + if (oldEntryName == null) + throw new ArgumentNullException (nameof (oldEntryName)); + if (newEntryName == null) + throw new ArgumentNullException (nameof (newEntryName)); + + var source = archive.ReadEntry (oldEntryName, StringComparison.Ordinal); + if (source == null) + return false; + + using var buffer = MemoryStreamPool.Shared.Rent (); + source.Extract (buffer); + buffer.Position = 0; + + DeleteEntry (archive, newEntryName); + var destination = archive.CreateEntry (newEntryName, compressionLevel); + destination.LastWriteTime = source.LastWriteTime; + using (var destinationStream = destination.Open ()) { + buffer.CopyTo (destinationStream); + } + + source.Delete (); + return true; + } + + public static void FixupWindowsPathSeparators (this ZipArchive archive, Func compressionLevelSelector, Action? onRename = null) + { + if (archive == null) + throw new ArgumentNullException (nameof (archive)); + if (compressionLevelSelector == null) + throw new ArgumentNullException (nameof (compressionLevelSelector)); + + foreach (var entryName in archive.Entries + .Where (entry => entry.FullName.Contains ('\\')) + .Select (entry => entry.FullName) + .ToArray ()) { + var entry = archive.ReadEntry (entryName, StringComparison.Ordinal); + if (entry == null) + continue; + + var normalizedName = entryName.Replace ('\\', '/'); + if (normalizedName == entryName) + continue; + + var compressionLevel = compressionLevelSelector (entry); + onRename?.Invoke (entryName, normalizedName); + archive.MoveEntry (entryName, normalizedName, compressionLevel); + } + } + + static void DeleteEntry (ZipArchive archive, string entryName) + { + if (archive.Mode == ZipArchiveMode.Create) + return; + + archive.ReadEntry (entryName, StringComparison.Ordinal)?.Delete (); + } + + static ZipArchive OpenZip (string archivePath, FileMode fileMode, FileAccess fileAccess, ZipArchiveMode archiveMode, Encoding? entryNameEncoding) + { + var stream = new FileStream (archivePath, fileMode, fileAccess, FileShare.Read); + try { + return OpenZip (stream, archiveMode, leaveOpen: false, entryNameEncoding); + } catch { + stream.Dispose (); + throw; + } + } + } +} diff --git a/src/Microsoft.Android.Build.BaseTasks/ZipArchiveMetadataReader.cs b/src/Microsoft.Android.Build.BaseTasks/ZipArchiveMetadataReader.cs new file mode 100644 index 00000000000..b25b4da9e60 --- /dev/null +++ b/src/Microsoft.Android.Build.BaseTasks/ZipArchiveMetadataReader.cs @@ -0,0 +1,216 @@ +using System; +using System.Globalization; +using System.IO; +using System.Text; + +namespace Microsoft.Android.Build.Tasks +{ + static class ZipArchiveMetadataReader + { + const uint EndOfCentralDirectorySignature = 0x06054b50; + const uint Zip64EndOfCentralDirectorySignature = 0x06064b50; + const uint Zip64EndOfCentralDirectoryLocatorSignature = 0x07064b50; + const uint CentralDirectoryFileHeaderSignature = 0x02014b50; + const int EndOfCentralDirectoryMinimumSize = 22; + const int Zip64EndOfCentralDirectoryLocatorSize = 20; + const int CentralDirectoryFileHeaderSize = 46; + const int EndOfCentralDirectorySearchWindow = ushort.MaxValue + EndOfCentralDirectoryMinimumSize; + static readonly Encoding Cp437 = CreateCp437Encoding (); + + public static void AppendHashInput (Stream stream, StringBuilder hashInput) + { + if (stream == null) + throw new ArgumentNullException (nameof (stream)); + if (hashInput == null) + throw new ArgumentNullException (nameof (hashInput)); + if (!stream.CanSeek) + throw new NotSupportedException ("ZIP metadata requires a seekable stream."); + + long originalPosition = stream.Position; + try { + AppendHashInputCore (stream, hashInput); + } finally { + stream.Seek (originalPosition, SeekOrigin.Begin); + } + } + + static void AppendHashInputCore (Stream stream, StringBuilder hashInput) + { + long endOfCentralDirectoryOffset = FindEndOfCentralDirectory (stream); + using var reader = new BinaryReader (stream, Encoding.UTF8, leaveOpen: true); + stream.Seek (endOfCentralDirectoryOffset + 4, SeekOrigin.Begin); + ushort diskNumber = reader.ReadUInt16 (); + ushort centralDirectoryDisk = reader.ReadUInt16 (); + ulong entriesOnDisk = reader.ReadUInt16 (); + ulong entryCount = reader.ReadUInt16 (); + ulong centralDirectorySize = reader.ReadUInt32 (); + ulong centralDirectoryOffset = reader.ReadUInt32 (); + + bool requiresZip64 = + entriesOnDisk == ushort.MaxValue || + entryCount == ushort.MaxValue || + centralDirectorySize == uint.MaxValue || + centralDirectoryOffset == uint.MaxValue; + if (requiresZip64 && TryReadZip64EndOfCentralDirectory (stream, reader, endOfCentralDirectoryOffset, out var zip64)) { + entriesOnDisk = zip64.EntriesOnDisk; + entryCount = zip64.EntryCount; + centralDirectorySize = zip64.CentralDirectorySize; + centralDirectoryOffset = zip64.CentralDirectoryOffset; + } else if (centralDirectorySize == uint.MaxValue || centralDirectoryOffset == uint.MaxValue) { + throw new InvalidDataException ("ZIP64 end of central directory record is missing."); + } + + if (diskNumber != 0 || centralDirectoryDisk != 0 || entriesOnDisk != entryCount) + throw new NotSupportedException ("Multi-disk ZIP archives are not supported."); + if (entryCount > int.MaxValue) + throw new InvalidDataException ("ZIP archive contains too many entries."); + if (centralDirectoryOffset > long.MaxValue || centralDirectorySize > long.MaxValue) + throw new InvalidDataException ("ZIP central directory is too large."); + + long centralDirectoryStart = (long) centralDirectoryOffset; + long centralDirectoryLength = (long) centralDirectorySize; + if (centralDirectoryStart > stream.Length || centralDirectoryLength > stream.Length - centralDirectoryStart) + throw new InvalidDataException ("ZIP central directory exceeds the available data."); + + stream.Seek (centralDirectoryStart, SeekOrigin.Begin); + long centralDirectoryEnd = centralDirectoryStart + centralDirectoryLength; + ulong entriesRead = 0; + while (entriesRead < entryCount) { + if (stream.Position > centralDirectoryEnd - CentralDirectoryFileHeaderSize) + throw new InvalidDataException ("ZIP central directory contains fewer entries than expected."); + if (reader.ReadUInt32 () != CentralDirectoryFileHeaderSignature) + throw new InvalidDataException ("Invalid ZIP central directory header."); + + reader.ReadUInt16 (); // version made by + reader.ReadUInt16 (); // version needed to extract + ushort flags = reader.ReadUInt16 (); + reader.ReadUInt16 (); // compression method + reader.ReadUInt16 (); // last mod file time + reader.ReadUInt16 (); // last mod file date + uint crc32 = reader.ReadUInt32 (); + reader.ReadUInt32 (); // compressed size + reader.ReadUInt32 (); // uncompressed size + ushort fileNameLength = reader.ReadUInt16 (); + ushort extraFieldLength = reader.ReadUInt16 (); + ushort fileCommentLength = reader.ReadUInt16 (); + reader.ReadUInt16 (); // disk number start + reader.ReadUInt16 (); // internal file attributes + reader.ReadUInt32 (); // external file attributes + reader.ReadUInt32 (); // relative offset of local header + + var fileNameBytes = reader.ReadBytes (fileNameLength); + if (fileNameBytes.Length != fileNameLength) + throw new InvalidDataException ("ZIP central directory entry name exceeds the available data."); + if (stream.Position > centralDirectoryEnd - extraFieldLength - fileCommentLength) + throw new InvalidDataException ("ZIP central directory entry exceeds the available data."); + var encoding = (flags & (1 << 11)) != 0 ? Encoding.UTF8 : Cp437; + hashInput.AppendFormat (CultureInfo.InvariantCulture, "{0}{1}", encoding.GetString (fileNameBytes), crc32); + stream.Seek (extraFieldLength + fileCommentLength, SeekOrigin.Current); + entriesRead++; + } + } + + static bool TryReadZip64EndOfCentralDirectory (Stream stream, BinaryReader reader, long endOfCentralDirectoryOffset, out Zip64DirectoryInfo info) + { + info = default; + long locatorOffset = endOfCentralDirectoryOffset - Zip64EndOfCentralDirectoryLocatorSize; + if (locatorOffset < 0) + return false; + + stream.Seek (locatorOffset, SeekOrigin.Begin); + if (reader.ReadUInt32 () != Zip64EndOfCentralDirectoryLocatorSignature) + return false; + + uint zip64DirectoryDisk = reader.ReadUInt32 (); + ulong zip64DirectoryOffset = reader.ReadUInt64 (); + uint diskCount = reader.ReadUInt32 (); + if (zip64DirectoryDisk != 0 || diskCount != 1) + throw new NotSupportedException ("Multi-disk ZIP archives are not supported."); + if (locatorOffset < 56 || zip64DirectoryOffset > long.MaxValue || zip64DirectoryOffset > (ulong) (locatorOffset - 56)) + throw new InvalidDataException ("ZIP64 end of central directory record exceeds the available data."); + + stream.Seek ((long) zip64DirectoryOffset, SeekOrigin.Begin); + if (reader.ReadUInt32 () != Zip64EndOfCentralDirectorySignature) + throw new InvalidDataException ("Invalid ZIP64 end of central directory record."); + + ulong recordSize = reader.ReadUInt64 (); + if (recordSize < 44 || recordSize > (ulong) (locatorOffset - stream.Position)) + throw new InvalidDataException ("ZIP64 end of central directory record exceeds the available data."); + + reader.ReadUInt16 (); // version made by + reader.ReadUInt16 (); // version needed to extract + uint diskNumber = reader.ReadUInt32 (); + uint centralDirectoryDisk = reader.ReadUInt32 (); + ulong entriesOnDisk = reader.ReadUInt64 (); + ulong entryCount = reader.ReadUInt64 (); + ulong centralDirectorySize = reader.ReadUInt64 (); + ulong centralDirectoryOffset = reader.ReadUInt64 (); + if (diskNumber != 0 || centralDirectoryDisk != 0 || entriesOnDisk != entryCount) + throw new NotSupportedException ("Multi-disk ZIP archives are not supported."); + + info = new Zip64DirectoryInfo (entriesOnDisk, entryCount, centralDirectorySize, centralDirectoryOffset); + return true; + } + + static long FindEndOfCentralDirectory (Stream stream) + { + long searchLength = Math.Min (stream.Length, EndOfCentralDirectorySearchWindow); + var buffer = new byte [searchLength]; + stream.Seek (-searchLength, SeekOrigin.End); + ReadExactly (stream, buffer, 0, buffer.Length); + + for (int index = buffer.Length - EndOfCentralDirectoryMinimumSize; index >= 0; index--) { + if ( + buffer [index] == 0x50 && + buffer [index + 1] == 0x4b && + buffer [index + 2] == 0x05 && + buffer [index + 3] == 0x06 && + index + EndOfCentralDirectoryMinimumSize + ReadUInt16 (buffer, index + 20) == buffer.Length + ) { + return stream.Length - searchLength + index; + } + } + + throw new InvalidDataException ("Could not locate the ZIP end of central directory record."); + } + + static ushort ReadUInt16 (byte [] buffer, int offset) + { + return (ushort) (buffer [offset] | (buffer [offset + 1] << 8)); + } + + static void ReadExactly (Stream stream, byte [] buffer, int offset, int count) + { + while (count > 0) { + int bytesRead = stream.Read (buffer, offset, count); + if (bytesRead == 0) + throw new EndOfStreamException (); + + offset += bytesRead; + count -= bytesRead; + } + } + + static Encoding CreateCp437Encoding () + { + Encoding.RegisterProvider (CodePagesEncodingProvider.Instance); + return Encoding.GetEncoding (437); + } + + readonly struct Zip64DirectoryInfo + { + public ulong EntriesOnDisk { get; } + public ulong EntryCount { get; } + public ulong CentralDirectorySize { get; } + public ulong CentralDirectoryOffset { get; } + + public Zip64DirectoryInfo (ulong entriesOnDisk, ulong entryCount, ulong centralDirectorySize, ulong centralDirectoryOffset) + { + EntriesOnDisk = entriesOnDisk; + EntryCount = entryCount; + CentralDirectorySize = centralDirectorySize; + CentralDirectoryOffset = centralDirectoryOffset; + } + } + } +} diff --git a/src/Microsoft.Android.Build.Tasks/BuildArchive.cs b/src/Microsoft.Android.Build.Tasks/BuildArchive.cs new file mode 100644 index 00000000000..d36f54a92a3 --- /dev/null +++ b/src/Microsoft.Android.Build.Tasks/BuildArchive.cs @@ -0,0 +1,396 @@ +#nullable enable +using System; +using System.Collections.Generic; +using System.IO; +using System.IO.Compression; +using Microsoft.Android.Build.Tasks; +using Microsoft.Build.Framework; + +namespace Microsoft.Android.Tasks; + +/// +/// Takes a list of files and adds them to an APK archive. If the APK archive already +/// exists, files are only added if they were changed. Note *ALL* files to be in the final +/// APK must be passed in via @(FilesToAddToArchive). This task will determine any unchanged files +/// and skip them, as well as remove any existing files in the APK that are no longer required. +/// +public class BuildArchive : AndroidTask +{ + public override string TaskPrefix => "BAA"; + + public string? AndroidPackageFormat { get; set; } + + public string? ApkInputPath { get; set; } + + [Required] + public string ApkOutputPath { get; set; } = ""; + + [Required] + public ITaskItem [] FilesToAddToArchive { get; set; } = []; + + public string? ArchiveRootDirectory { get; set; } + + public string? UncompressedFileExtensions { get; set; } + + public string? ZipFlushFilesLimit { get; set; } + + public string? ZipFlushSizeLimit { get; set; } + + HashSet? uncompressedFileExtensions; + + HashSet UncompressedFileExtensionsSet => uncompressedFileExtensions ??= ParseUncompressedFileExtensions (); + + CompressionLevel uncompressedFileCompression = CompressionLevel.NoCompression; + + const int DefaultFlushFilesLimit = 512; + const long DefaultFlushSizeLimit = 100 * 1024 * 1024; + + public override bool RunTask () + { + bool isAab = string.Equals (AndroidPackageFormat, "aab", StringComparison.OrdinalIgnoreCase); + if (isAab) { + uncompressedFileCompression = CompressionLevel.Optimal; + } + + Directory.CreateDirectory (Path.GetDirectoryName (ApkOutputPath) ?? "."); + + bool refreshExistingOutput = true; + if (!string.IsNullOrEmpty (ApkInputPath) && File.Exists (ApkInputPath) && !File.Exists (ApkOutputPath)) { + Log.LogDebugMessage ($"Copying {ApkInputPath} to {ApkOutputPath}"); + File.Copy (ApkInputPath, ApkOutputPath, overwrite: true); + refreshExistingOutput = false; + } + + using var apk = new ArchiveUpdateSession ( + ApkOutputPath, + ParseFlushLimit (ZipFlushFilesLimit, DefaultFlushFilesLimit), + ParseFlushLimit (ZipFlushSizeLimit, DefaultFlushSizeLimit) + ); + var existingEntries = new List (); + + if (refreshExistingOutput) { + foreach (var entry in apk.Archive.Entries) { + Log.LogDebugMessage ($"Registering item {entry.FullName}"); + existingEntries.Add (entry.FullName); + } + } + + if (!string.IsNullOrEmpty (ApkInputPath) && File.Exists (ApkInputPath) && refreshExistingOutput) { + RefreshEntriesFromInputArchive (apk, existingEntries, isAab); + } + + bool fixedPathSeparators = false; + apk.Archive.FixupWindowsPathSeparators ( + entry => ToCompressionLevel (entry.CompressionMethod), + (source, destination) => { + fixedPathSeparators = true; + Log.LogDebugMessage ($"Fixing up malformed entry `{source}` -> `{destination}`"); + existingEntries.Remove (source); + existingEntries.Add (destination); + } + ); + if (fixedPathSeparators) + apk.Commit (); + + foreach (var file in FilesToAddToArchive) { + if (!AddItemToArchive (apk, file, existingEntries)) + return false; + } + + foreach (var entry in existingEntries) { + if (string.Equals (Path.GetFileName (entry), "AndroidManifest.xml", StringComparison.OrdinalIgnoreCase)) + continue; + + Log.LogDebugMessage ($"Removing {entry} as it is no longer required."); + apk.Archive.ReadEntry (entry, StringComparison.Ordinal)?.Delete (); + } + + if (isAab) { + FixupBundleManifest (apk.Archive); + } + + return !Log.HasLoggedErrors; + } + + void RefreshEntriesFromInputArchive (ArchiveUpdateSession apk, List existingEntries, bool isAab) + { + if (ApkInputPath == null) + throw new InvalidOperationException ("ApkInputPath must not be null when refreshing the output archive."); + + DateTime lastWriteOutput = File.Exists (ApkOutputPath) ? File.GetLastWriteTimeUtc (ApkOutputPath) : DateTime.MinValue; + DateTime lastWriteInput = File.GetLastWriteTimeUtc (ApkInputPath); + + using var packaged = ZipArchiveExtensions.OpenZipRead (ApkInputPath); + foreach (var entry in packaged.Entries) { + if (entry.IsDirectory ()) { + continue; + } + + string entryName = entry.FullName; + if (entryName.Contains ("\\")) { + entryName = entryName.Replace ('\\', '/'); + Log.LogDebugMessage ($"Fixing up malformed entry `{entry.FullName}` -> `{entryName}`"); + } + + if (entryName == "AndroidManifest.xml" && isAab) { + Log.LogDebugMessage ("Renaming AndroidManifest.xml to manifest/AndroidManifest.xml"); + entryName = "manifest/AndroidManifest.xml"; + } + + Log.LogDebugMessage ($"Deregistering item {entryName}"); + existingEntries.Remove (entryName); + + if (lastWriteInput <= lastWriteOutput) { + Log.LogDebugMessage ($"Skipping to next item. {lastWriteInput} <= {lastWriteOutput}."); + continue; + } + + var currentEntry = apk.Archive.ReadEntry (entryName, StringComparison.Ordinal); + if (currentEntry != null && entry.Crc32 == currentEntry.Crc32 && entry.CompressedLength == currentEntry.CompressedLength) { + Log.LogDebugMessage ($"Skipping {entryName} from {ApkInputPath} as its up to date."); + continue; + } + + if (currentEntry != null) { + currentEntry.Delete (); + } + + Log.LogDebugMessage ($"Refreshing {entryName} from {ApkInputPath}"); + CopyEntryToArchive (apk.Archive, entryName, entry, ToCompressionLevel (entry.CompressionMethod)); + apk.RecordWrite (entry.Length); + } + } + + bool AddItemToArchive (ArchiveUpdateSession apk, ITaskItem item, List existingEntries) + { + string diskPath = item.ItemSpec; + string archivePath = item.GetMetadata ("ArchivePath") ?? ""; + if (string.IsNullOrWhiteSpace (archivePath)) { + if (!string.IsNullOrEmpty (ArchiveRootDirectory)) { + archivePath = Path.GetRelativePath (ArchiveRootDirectory, diskPath); + } else if (!item.TryGetRequiredMetadata ("FilesToAddToArchive", "ArchivePath", Log, out archivePath)) { + return false; + } + } + + archivePath = archivePath.Replace ('\\', '/'); + + string jarEntryName = GetMetadataOrDefault (item, "JavaArchiveEntry", string.Empty); + if (!string.IsNullOrEmpty (jarEntryName)) { + AddJarEntryToArchive (apk, diskPath, archivePath, jarEntryName, existingEntries); + return !Log.HasLoggedErrors; + } + + AddFileToArchiveIfNewer (apk, diskPath, archivePath, item, existingEntries); + return !Log.HasLoggedErrors; + } + + void AddJarEntryToArchive (ArchiveUpdateSession apk, string diskPath, string archivePath, string jarEntryName, List existingEntries) + { + string jarFilePath = diskPath.Substring (0, diskPath.Length - (jarEntryName.Length + 1)); + bool wasExistingOutputEntry = existingEntries.Remove (archivePath); + var currentEntry = apk.Archive.ReadEntry (archivePath, StringComparison.Ordinal); + + if (currentEntry != null && !wasExistingOutputEntry) { + Log.LogDebugMessage ("Failed to add jar entry {0} from {1}: the same file already exists in the apk", jarEntryName, Path.GetFileName (jarFilePath)); + return; + } + + using var jar = ZipArchiveExtensions.OpenZipRead (jarFilePath); + var jarEntry = jar.ReadEntry (jarEntryName, StringComparison.Ordinal); + if (jarEntry == null) { + Log.LogDebugMessage ("Failed to add jar entry {0} from {1}: entry not found in jar.", jarEntryName, jarFilePath); + if (wasExistingOutputEntry) + existingEntries.Add (archivePath); + return; + } + + if (currentEntry != null && currentEntry.Crc32 == jarEntry.Crc32) { + Log.LogDebugMessage ("Skipping {0} from {1} as it is up to date.", jarEntryName, jarFilePath); + return; + } + + currentEntry?.Delete (); + + using var buffer = MemoryStreamPool.Shared.Rent (); + jarEntry.Extract (buffer); + buffer.Position = 0; + Log.LogDebugMessage ($"Adding {jarEntryName} from {jarFilePath} as the archive file is out of date."); + apk.Archive.AddStream (buffer, archivePath); + apk.RecordWrite (jarEntry.Length); + } + + bool AddFileToArchiveIfNewer (ArchiveUpdateSession apk, string file, string archivePath, ITaskItem item, List existingEntries) + { + ZipCompressionMethod compressionMethod = GetCompressionMethod (item); + existingEntries.Remove (archivePath); + + var entry = apk.Archive.ReadEntry (archivePath, StringComparison.Ordinal); + if (entry == null) { + apk.Archive.AddFile (file, archivePath, ToCompressionLevel (compressionMethod)); + apk.RecordWrite (new FileInfo (file).Length); + Log.LogDebugMessage ($"Adding {file} as it doesn't already exist."); + return true; + } + + if (GetExistingCompressionMethod (entry) != compressionMethod) { + Log.LogDebugMessage ($"Updating {file} as the compression level changed."); + entry.Delete (); + apk.Archive.AddFile (file, archivePath, ToCompressionLevel (compressionMethod)); + apk.RecordWrite (new FileInfo (file).Length); + return true; + } + + uint existingDosTime = DateTimeToDosTime (entry.LastWriteTime.UtcDateTime); + uint fileDosTime = DateTimeToDosTime (File.GetLastWriteTimeUtc (file)); + if (existingDosTime < fileDosTime) { + Log.LogDebugMessage ($"Updating {file} as the file write time is newer: file in zip - '{existingDosTime}', file on disk - '{fileDosTime}'."); + entry.Delete (); + apk.Archive.AddFile (file, archivePath, ToCompressionLevel (compressionMethod)); + apk.RecordWrite (new FileInfo (file).Length); + return true; + } + + Log.LogDebugMessage ($"Skipping {file} as the archive file is up to date."); + return false; + } + + void FixupBundleManifest (ZipArchive apk) + { + var manifest = apk.ReadEntry ("AndroidManifest.xml", StringComparison.Ordinal); + if (manifest == null) { + Log.LogDebugMessage ("No AndroidManifest.xml. Skipping Fixup"); + return; + } + + Log.LogDebugMessage ("Fixing up AndroidManifest.xml to be manifest/AndroidManifest.xml."); + apk.MoveEntry ("AndroidManifest.xml", "manifest/AndroidManifest.xml", ToCompressionLevel (manifest.CompressionMethod)); + } + + void CopyEntryToArchive (ZipArchive archive, string destinationEntryName, ZipArchiveEntry sourceEntry, CompressionLevel compressionLevel) + { + var destinationEntry = archive.CreateEntry (destinationEntryName, compressionLevel); + destinationEntry.LastWriteTime = sourceEntry.LastWriteTime; + using var source = sourceEntry.Open (); + using var destination = destinationEntry.Open (); + source.CopyTo (destination); + } + + ZipCompressionMethod GetCompressionMethod (ITaskItem item) + { + if (UncompressedFileExtensionsSet.Contains (Path.GetExtension (item.ItemSpec))) { + return uncompressedFileCompression == CompressionLevel.NoCompression ? ZipCompressionMethod.Stored : ZipCompressionMethod.Deflate; + } + + return ZipCompressionMethod.Deflate; + } + + static CompressionLevel ToCompressionLevel (ZipCompressionMethod compressionMethod) + { + return compressionMethod switch { + ZipCompressionMethod.Stored => CompressionLevel.NoCompression, + ZipCompressionMethod.Deflate => CompressionLevel.Optimal, + _ => throw new NotSupportedException ($"Unsupported ZIP compression method: {compressionMethod}"), + }; + } + + static ZipCompressionMethod GetExistingCompressionMethod (ZipArchiveEntry entry) + { + return entry.CompressionMethod switch { + ZipCompressionMethod.Stored => ZipCompressionMethod.Stored, + ZipCompressionMethod.Deflate => ZipCompressionMethod.Deflate, + _ => throw new NotSupportedException ($"Unsupported ZIP compression method: {entry.CompressionMethod}"), + }; + } + + HashSet ParseUncompressedFileExtensions () + { + var parsedExtensions = new HashSet (StringComparer.OrdinalIgnoreCase); + + foreach (var extension in UncompressedFileExtensions?.Split ([';', ','], StringSplitOptions.RemoveEmptyEntries) ?? []) { + var normalized = extension.Trim (); + if (string.IsNullOrEmpty (normalized)) { + continue; + } + + if (normalized [0] != '.') { + normalized = $".{normalized}"; + } + + parsedExtensions.Add (normalized); + } + + return parsedExtensions; + } + + static string GetMetadataOrDefault (ITaskItem item, string metadataName, string defaultValue) + { + string metadataValue = item.GetMetadata (metadataName) ?? ""; + if (string.IsNullOrEmpty (metadataValue)) + return defaultValue; + + return metadataValue; + } + + static long ParseFlushLimit (string? value, long defaultValue) + { + if (long.TryParse (value, out long parsedValue) && parsedValue > 0) + return parsedValue; + + return defaultValue; + } + + const int ValidZipDate_YearMin = 1980; + + static uint DateTimeToDosTime (DateTime dateTime) + { + int ret = ((dateTime.Year - ValidZipDate_YearMin) & 0x7F); + ret = (ret << 4) + dateTime.Month; + ret = (ret << 5) + dateTime.Day; + ret = (ret << 5) + dateTime.Hour; + ret = (ret << 6) + dateTime.Minute; + ret = (ret << 5) + (dateTime.Second / 2); + return (uint) ret; + } + + sealed class ArchiveUpdateSession : IDisposable + { + readonly string archivePath; + readonly long flushFilesLimit; + readonly long flushSizeLimit; + long filesWritten; + long bytesWritten; + ZipArchive archive; + + public ZipArchive Archive => archive; + + public ArchiveUpdateSession (string archivePath, long flushFilesLimit, long flushSizeLimit) + { + this.archivePath = archivePath; + this.flushFilesLimit = flushFilesLimit; + this.flushSizeLimit = flushSizeLimit; + archive = ZipArchiveExtensions.OpenZipUpdate (archivePath); + } + + public void RecordWrite (long size) + { + filesWritten++; + bytesWritten += size; + if (filesWritten >= flushFilesLimit || bytesWritten >= flushSizeLimit) + Commit (); + } + + public void Commit () + { + archive.Dispose (); + archive = ZipArchiveExtensions.OpenZipUpdate (archivePath, FileMode.Open); + filesWritten = 0; + bytesWritten = 0; + } + + public void Dispose () + { + archive.Dispose (); + } + } +} diff --git a/src/Microsoft.Android.Build.Tasks/Microsoft.Android.Build.Tasks.csproj b/src/Microsoft.Android.Build.Tasks/Microsoft.Android.Build.Tasks.csproj index 245e9c7c077..536ae23571b 100644 --- a/src/Microsoft.Android.Build.Tasks/Microsoft.Android.Build.Tasks.csproj +++ b/src/Microsoft.Android.Build.Tasks/Microsoft.Android.Build.Tasks.csproj @@ -2,7 +2,7 @@ +