Implement Bundler version 2 (#33413)

* Implement Bundler version 2

Implement support for single-file version 2 layout as described in https://github.com/dotnet/designs/blob/master/accepted/2020/single-file/bundler.md

The supporting changes are:
* Implement new HostModel interfaces to communicate additional information from the SDK to the HostModel
   * The SDK changes will be done in an upcoming PR.
   * Some depricated APIs are maintained so that SDK build doesn't break in the meantime]
* Handle various bundling options described in https://github.com/dotnet/designs/blob/master/accepted/2020/single-file/design.md#optional-settings
   * This requires ability to recognize native binaries for various architectures.
   * Added ability to minimaly parse ELF binaries. PE/MachO support already exists.
   * Refactored out PE processing from BinaryUtils, so that PE-ELF-MachO parsers have similar abstractions.
* Create bundles with the appropriate layout (`v1` for `netcoreapp3.0`, `v2` for `net5`)
* Consume the new layout from the host bundle process/extraction code.
* Test cases
* Some of the AppHost rewriter files imported from the SDK repo had the license banner in a differnt format; make them consistent with rest of the files in the installer partition.

src/installer/corehost/cli/apphost/bundle/file_entry.h

@@ -36,7 +36,7 @@ namespace bundle
         file_entry_t()
             : m_offset(0)
             , m_size(0)
-            , m_type(bundle::file_type_t::__last)
+            , m_type(file_type_t::__last)
             , m_relative_path()
         {
         }

src/installer/corehost/cli/apphost/bundle/file_type.h

@@ -19,11 +19,12 @@ namespace bundle
 
     enum file_type_t : uint8_t
     {
+        unknown,
         assembly,
-        ready2run,
+        native_binary,
         deps_json,
         runtime_config_json,
-        extract,
+        symbols,
         __last
     };
 }

src/installer/corehost/cli/apphost/bundle/header.cpp

@@ -9,18 +9,29 @@
 
 using namespace bundle;
 
-bool header_fixed_t::is_valid() const
+// The AppHost expects the bundle_header to be an exact_match for which it was built.
+// The framework accepts backwards compatible header versions.
+bool header_fixed_t::is_valid(bool exact_match) const
 {
-    return num_embedded_files > 0 &&
-           ((major_version < header_t::major_version) ||
+    if (num_embedded_files <= 0)
+    {
+        return false;
+    }
+
+    if (exact_match)
+    {
+        return (major_version == header_t::major_version) && (minor_version == header_t::minor_version);
+    }
+
+    return ((major_version < header_t::major_version) ||
             (major_version == header_t::major_version && minor_version <= header_t::minor_version));
 }
 
-header_t header_t::read(reader_t& reader)
+header_t header_t::read(reader_t& reader, bool need_exact_version)
 {
     const header_fixed_t* fixed_header = reinterpret_cast<const header_fixed_t*>(reader.read_direct(sizeof(header_fixed_t)));
 
-    if (!fixed_header->is_valid())
+    if (!fixed_header->is_valid(need_exact_version))
     {
         trace::error(_X("Failure processing application bundle."));
         trace::error(_X("Bundle header version compatibility check failed."));
@@ -33,5 +44,10 @@ header_t header_t::read(reader_t& reader)
     // bundle_id is a component of the extraction path
     reader.read_path_string(header.m_bundle_id);
 
+    if (fixed_header->major_version > 1)
+    {
+        header.m_v2_header = reinterpret_cast<const header_fixed_v2_t*>(reader.read_direct(sizeof(header_fixed_v2_t)));
+    }
+
     return header;
 }

src/installer/corehost/cli/apphost/bundle/header.h

@@ -11,13 +11,18 @@
 
 namespace bundle
 {
-    // The Bundle Header contains:
+    // The Bundle Header (v1)
     // Fixed size thunk (header_fixed_t)
     //   - Major Version     
     //   - Minor Version     
     //   - Number of embedded files
     // Variable size portion:
     //   - Bundle ID (7-bit extension encoded length prefixed string)
+    // The Bundle Header (v2) [additional content]
+    // Fixed size thunk (header_fixed_v2_t)
+    //   - DepsJson Location (Offset, Size)
+    //   - RuntimeConfig Location (Offset, Size)
+    //   - Flags
 
 #pragma pack(push, 1)
     struct header_fixed_t
@@ -27,7 +32,39 @@ namespace bundle
         uint32_t minor_version;
         int32_t num_embedded_files;
 
-        bool is_valid() const;
+        bool is_valid(bool exact_match = false) const;
+    };
+#pragma pack(pop)
+
+    // netcoreapp3_compat_mode flag is set on a .net5 app, which chooses to build single-file apps in .netcore3.x compat mode,
+    // This indicates that:
+    //   All published files are bundled into the app; some of them will be extracted to disk.
+    //   AppContext.BaseDirectory is set to the extraction directory (and not the AppHost directory).
+    enum header_flags_t : uint64_t
+    {
+        none = 0,
+        netcoreapp3_compat_mode = 1
+    };
+
+#pragma pack(push, 1)
+    struct location_t
+    {
+    public:
+        int64_t offset;
+        int64_t size;
+    };
+
+    // header_fixed_v2_t is available in single-file apps targetting .net5+ frameworks.
+    // It stores information that facilitates the host to process contents of the bundle without extraction.
+    //
+    // The location of deps.json and runtimeconfig.json is already available in the Bundle manifest.
+    // However, the data is cached here in order to simplify the bundle-processing performed by hostfxr.
+    struct header_fixed_v2_t
+    {
+    public:
+        location_t deps_json_location;
+        location_t runtimeconfig_json_location;
+        header_flags_t flags;
     };
 #pragma pack(pop)
 
@@ -37,19 +74,22 @@ namespace bundle
         header_t(int32_t num_embedded_files = 0)
             : m_num_embedded_files(num_embedded_files)
             , m_bundle_id()
+            , m_v2_header(NULL)
+
         {
         }
 
-        static header_t read(reader_t& reader);
+        static header_t read(reader_t& reader, bool need_exact_version);
         const pal::string_t& bundle_id() { return m_bundle_id; }
-        int32_t num_embedded_files() { return m_num_embedded_files;  }
+        int32_t num_embedded_files() { return m_num_embedded_files; }
 
-        static const uint32_t major_version = 1;
+        static const uint32_t major_version = 2;
         static const uint32_t minor_version = 0;
 
     private:
         int32_t m_num_embedded_files;
         pal::string_t m_bundle_id;
+        const header_fixed_v2_t* m_v2_header;
 
     };
 }

src/installer/corehost/cli/apphost/bundle/runner.cpp

@@ -44,7 +44,7 @@ StatusCode runner_t::extract()
 
         // Read the bundle header
         reader.set_offset(marker_t::header_offset());
-        header_t header = header_t::read(reader);
+        header_t header = header_t::read(reader, /* need_exact_version: */ true);
 
         // Read the bundle manifest
         // Reader is at the correct offset

src/installer/managed/Microsoft.NET.HostModel/AppHost/BinaryUtils.cs

@@ -1,11 +1,10 @@
-// Copyright (c) .NET Foundation and contributors. All rights reserved.
-// Licensed under the MIT license. See LICENSE file in the project root for full license information.
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+// See the LICENSE file in the project root for more information.
 
 using System;
 using System.IO;
 using System.IO.MemoryMappedFiles;
-using System.Text;
-using System.Runtime.InteropServices;
 
 namespace Microsoft.NET.HostModel.AppHost
 {
@@ -164,160 +163,6 @@ private static unsafe int KMPSearch(byte[] pattern, byte* bytes, long bytesLengt
             return -1;
         }
 
-        /// <summary>
-        /// The first two bytes of a PE file are a constant signature.
-        /// </summary>
-        private const UInt16 PEFileSignature = 0x5A4D;
-
-        /// <summary>
-        /// The offset of the PE header pointer in the DOS header.
-        /// </summary>
-        private const int PEHeaderPointerOffset = 0x3C;
-
-        /// <summary>
-        /// The offset of the Subsystem field in the PE header.
-        /// </summary>
-        private const int SubsystemOffset = 0x5C;
-
-        /// <summary>
-        /// The value of the sybsystem field which indicates Windows GUI (Graphical UI)
-        /// </summary>
-        private const UInt16 WindowsGUISubsystem = 0x2;
-
-        /// <summary>
-        /// The value of the subsystem field which indicates Windows CUI (Console)
-        /// </summary>
-        private const UInt16 WindowsCUISubsystem = 0x3;
-
-        /// <summary>
-        /// Check whether the apphost file is a windows PE image by looking at the first few bytes.
-        /// </summary>
-        /// <param name="accessor">The memory accessor which has the apphost file opened.</param>
-        /// <returns>true if the accessor represents a PE image, false otherwise.</returns>
-        internal static unsafe bool IsPEImage(MemoryMappedViewAccessor accessor)
-        {
-            byte* pointer = null;
-
-            try
-            {
-                accessor.SafeMemoryMappedViewHandle.AcquirePointer(ref pointer);
-                byte* bytes = pointer + accessor.PointerOffset;
-
-                // https://en.wikipedia.org/wiki/Portable_Executable
-                // Validate that we're looking at Windows PE file
-                if (((UInt16*)bytes)[0] != PEFileSignature || accessor.Capacity < PEHeaderPointerOffset + sizeof(UInt32))
-                {
-                    return false;
-                }
-                return true;
-            }
-            finally
-            {
-                if (pointer != null)
-                {
-                    accessor.SafeMemoryMappedViewHandle.ReleasePointer();
-                }
-            }
-        }
-
-        /// <summary>
-        /// This method will attempt to set the subsystem to GUI. The apphost file should be a windows PE file.
-        /// </summary>
-        /// <param name="accessor">The memory accessor which has the apphost file opened.</param>
-        internal static unsafe void SetWindowsGraphicalUserInterfaceBit(MemoryMappedViewAccessor accessor)
-        {
-            byte* pointer = null;
-
-            try
-            {
-                accessor.SafeMemoryMappedViewHandle.AcquirePointer(ref pointer);
-                byte* bytes = pointer + accessor.PointerOffset;
-
-                // https://en.wikipedia.org/wiki/Portable_Executable
-                UInt32 peHeaderOffset = ((UInt32*)(bytes + PEHeaderPointerOffset))[0];
-
-                if (accessor.Capacity < peHeaderOffset + SubsystemOffset + sizeof(UInt16))
-                {
-                    throw new AppHostNotPEFileException();
-                }
-
-                UInt16* subsystem = ((UInt16*)(bytes + peHeaderOffset + SubsystemOffset));
-
-                // https://docs.microsoft.com/en-us/windows/desktop/Debug/pe-format#windows-subsystem
-                // The subsystem of the prebuilt apphost should be set to CUI
-                if (subsystem[0] != WindowsCUISubsystem)
-                {
-                    throw new AppHostNotCUIException();
-                }
-
-                // Set the subsystem to GUI
-                subsystem[0] = WindowsGUISubsystem;
-            }
-            finally
-            {
-                if (pointer != null)
-                {
-                    accessor.SafeMemoryMappedViewHandle.ReleasePointer();
-                }
-            }
-        }
-
-        public static unsafe void SetWindowsGraphicalUserInterfaceBit(string filePath)
-        {
-            using (var mappedFile = MemoryMappedFile.CreateFromFile(filePath))
-            {
-                using (var accessor = mappedFile.CreateViewAccessor())
-                {
-                    SetWindowsGraphicalUserInterfaceBit(accessor);
-                }
-            }
-        }
-
-        /// <summary>
-        /// This method will return the subsystem CUI/GUI value. The apphost file should be a windows PE file.
-        /// </summary>
-        /// <param name="accessor">The memory accessor which has the apphost file opened.</param>
-        internal static unsafe UInt16 GetWindowsGraphicalUserInterfaceBit(MemoryMappedViewAccessor accessor)
-        {
-            byte* pointer = null;
-
-            try
-            {
-                accessor.SafeMemoryMappedViewHandle.AcquirePointer(ref pointer);
-                byte* bytes = pointer + accessor.PointerOffset;
-
-                // https://en.wikipedia.org/wiki/Portable_Executable
-                UInt32 peHeaderOffset = ((UInt32*)(bytes + PEHeaderPointerOffset))[0];
-
-                if (accessor.Capacity < peHeaderOffset + SubsystemOffset + sizeof(UInt16))
-                {
-                    throw new AppHostNotPEFileException();
-                }
-
-                UInt16* subsystem = ((UInt16*)(bytes + peHeaderOffset + SubsystemOffset));
-
-                return subsystem[0];
-            }
-            finally
-            {
-                if (pointer != null)
-                {
-                    accessor.SafeMemoryMappedViewHandle.ReleasePointer();
-                }
-            }
-        }
-
-        public static unsafe UInt16 GetWindowsGraphicalUserInterfaceBit(string filePath)
-        {
-            using (var mappedFile = MemoryMappedFile.CreateFromFile(filePath))
-            {
-                using (var accessor = mappedFile.CreateViewAccessor())
-                {
-                    return GetWindowsGraphicalUserInterfaceBit(accessor);
-                }
-            }
-        }
-
         public static void CopyFile(string sourcePath, string destinationPath)
         {
             var destinationDirectory = new FileInfo(destinationPath).Directory.FullName;
@@ -330,5 +175,13 @@ public static void CopyFile(string sourcePath, string destinationPath)
             File.Copy(sourcePath, destinationPath, overwrite: true);
         }
 
+        // The SDK calls into BinaryUtils.GetWindowsGraphicalUserInterfaceBit()
+        // Keep this method until the HostModel changes reach SDK repo, and the code there
+        // is updated to use PEUtils.GetWindowsGraphicalUserInterfaceBit()
+        public static unsafe UInt16 GetWindowsGraphicalUserInterfaceBit(string filePath)
+        {
+            return PEUtils.GetWindowsGraphicalUserInterfaceBit(filePath);
+        }
+
     }
 }