Refactor and improve Windows builds

CHANGES.md

@@ -1,16 +1,28 @@
 unreleased
 ----------
 
-- Added Ubuntu 23.04 (@mtelvers)
+- Rework Windows images and update their dependencies:
+  + Fix the origin of Install.cmd (avsm -> ocurrent);
+  + Rename Windows.Cygwin.install_from_release to install_cygwin;
+  + Rework Cygwin package list needed for opam and OCaml for Windows;
+  + Remove msvs-tools from the mingw images;
+  + Build opam with MSVS in the MSVS images. Explicitly set MSVS
+    environment vars with msvs-detect.
+  + Update to VC Redist 17 and MSVC 2022;
+  + Track msvs-tools master;
+  + Split MSVC build into multiple build steps;
+  + Internal refactors.
+  (@MisterDA #162)
+- Added Ubuntu 23.04 and Fedora 38. (@mtelvers #164)
 - Add newlines in some cases for better formatting.
   (@MisterDA #161, review by @benmandrew)
+- Various LCU Updates. (@mtelvers #160)
 
 v8.2.1 2023-04-07 Paris
 -----------------------
 
 - Correct sexp generation for dockerfile-opam.
   (@benmandrew #158, review by @MisterDA)
-
 - Switch to root and back to opam user when installing OCaml external
   dependencies in the ocaml stage; fixes depext installation.
   (@MisterDA #146, #157)

src-opam/Install.cmd

@@ -12,3 +12,5 @@ if "%ERRORLEVEL%"=="3010" (
         exit /b !ERR!
     )
 )
+
+exit /b 0

src-opam/opam.ml

@@ -49,18 +49,27 @@ let install_opam_from_source ?(add_default_link = true) ?(prefix = "/usr/local")
 
 (* Build opam in a separate worktree from an already cloned opam *)
 let install_opam_from_source_windows ?cyg ?prefix
-    ?(enable_0install_solver = false) ~branch ~hash () =
+    ?(enable_0install_solver = false) ?(msvs = false) ~branch ~hash () =
+  (* Although opam's readme states it can autodetec the environment
+     with MSVS, it doesn't always work. It's set explicitly here. *)
+  let msvs_env =
+    {|eval $(msvs-detect --arch=x64) && export PATH="$MSVS_PATH:$PATH" && export LIB="$MSVS_LIB" && export INCLUDE="$MSVS_INC" && |}
+  and ocaml_port = "OCAML_PORT=msvc64 " in
   Windows.Cygwin.run_sh ?cyg
     "cd /tmp/opam-sources && cp -P -R -p . ../opam-build-%s && cd \
      ../opam-build-%s && git checkout %s && git config --global --add \
      safe.directory /tmp/opam-build-%s"
     branch branch hash branch
   @@ Windows.Cygwin.run_sh ?cyg
-       "cd /tmp/opam-build-%s && make compiler && make lib-pkg" branch
+       "cd /tmp/opam-build-%s && %smake compiler %s&& make lib-pkg" branch
+       (if msvs then msvs_env else "")
+       (if msvs then ocaml_port else "")
   @@ Windows.Cygwin.run_sh ?cyg
-       "cd /tmp/opam-build-%s && ./configure --enable-cold-check \
-        --with-private-runtime%s%s && make && make install"
+       "cd /tmp/opam-build-%s && %s./configure --enable-cold-check %s%s%s && \
+        make && make install"
        branch
+       (if msvs then msvs_env else "")
+       (if msvs then "" else "--with-private-runtime")
        (Option.fold prefix ~none:"" ~some:(fun prefix ->
             Printf.sprintf {| --prefix="%s"|} prefix))
        (if enable_0install_solver then " --with-0install-solver" else "")
@@ -177,7 +186,7 @@ let from ?win10_revision ?arch ?maintainer ?img ?tag d =
 let header ?win10_revision ?arch ?maintainer ?img ?tag d =
   let parser_directives =
     match D.os_family_of_distro d with
-    | `Windows | `Cygwin -> parser_directive (`Escape '`')
+    | `Windows | `Cygwin -> empty
     | _ -> buildkit_syntax
   in
   parser_directives
@@ -247,7 +256,7 @@ let install_opams ?prefix opam_master_hash opam_branches =
               ~enable_0install_solver ~branch ~hash ())
        empty opam_branches
 
-let install_opams_windows ?cyg ?prefix opam_master_hash opam_branches =
+let install_opams_windows ?cyg ?prefix ?msvs opam_master_hash opam_branches =
   Windows.Cygwin.Git.init ?cyg ~repos:[ "/tmp/opam-sources" ] ()
   @@ Windows.Cygwin.run_sh ?cyg
        "git clone https://github.com/ocaml/opam /tmp/opam && cd /tmp/opam && \
@@ -256,7 +265,7 @@ let install_opams_windows ?cyg ?prefix opam_master_hash opam_branches =
   @@ List.fold_left
        (fun acc { branch; hash; enable_0install_solver; _ } ->
          acc
-         @@ install_opam_from_source_windows ?cyg ?prefix
+         @@ install_opam_from_source_windows ?cyg ?prefix ?msvs
               ~enable_0install_solver ~branch ~hash ())
        empty opam_branches
 
@@ -410,65 +419,102 @@ let pacman_opam2 ?(labels = []) ?arch ~opam_hashes distro () =
   @@ Linux.Pacman.add_user ~uid:1000 ~sudo:true "opam"
   @@ install_bubblewrap_wrappers @@ Linux.Git.init ()
 
-(* TODO: Compile opam-2.0 and 2.1 instead of downloading binaries,
-   add an option to enable 0install-solver,
-   and pass ~hash_opam_2_0 ~hash_opam_2_1 like the cygwin one *)
-(* Native Windows, WinGet, Cygwin based Dockerfiles *)
-let windows_opam2 ?win10_revision ?winget ?(labels = []) ?arch ~opam_hashes
-    distro () =
+let install_winget ?win10_revision ?winget version =
+  match winget with
+  | None when Windows.Winget.is_supported version ->
+      ( Windows.header ~alias:"winget-builder" ?win10_revision ~version ()
+        @@ Windows.Winget.install_from_release (),
+        Windows.install_vc_redist ()
+        @@ Windows.Winget.setup ~from:"winget-builder" ()
+        @@ Windows.Winget.dev_packages ~version () )
+  | _ -> (empty, empty)
+
+(* Native Windows with mingw-w64 and WinGet. *)
+let windows_mingw_opam2 ?win10_revision ?winget ?(labels = []) ?arch
+    ~opam_hashes distro () =
   let opam_master_hash, opam_branches =
     create_opam_branches_windows opam_hashes
   in
   let version = match distro with `Windows (_, v) -> v | _ -> assert false in
   let winget_image, winget_setup =
-    match winget with
-    | None when Windows.Winget.is_supported version ->
-        ( Windows.header ~alias:"winget-builder" ?win10_revision ~version ()
-          @@ Windows.Winget.install_from_release (),
-          Windows.Winget.setup ~from:"winget-builder" ()
-          @@ Windows.Winget.dev_packages ~version () )
-    | _ -> (empty, empty)
+    install_winget ?win10_revision ?winget version
   in
   let opams_image =
+    let packages =
+      (* dra27: we only need g++ in the "builder" image so that
+         opam-putenv gets built properly and so that opam is built
+         with the internal solver as well *)
+      "mingw64-x86_64-gcc-g++" :: "mingw64-i686-gcc-g++"
+      :: Windows.Cygwin.mingw_packages
+    in
     Windows.header ~alias:"opam-builder" ?win10_revision ~version ()
     @@ Windows.sanitize_reg_path ()
-    @@ Windows.Cygwin.(
-         install_from_release
-           ~extra:
-             ("git" :: "patch" :: "mingw64-x86_64-gcc-g++"
-            :: "mingw64-i686-gcc-g++" :: mingw_packages)
-           ())
+    @@ Windows.Cygwin.install_cygwin ~extra:packages ()
     @@ install_opams_windows opam_master_hash opam_branches
   in
   (* 2022-10-12: Docker Engine 20.10.18 on Windows fails copying
      C:\cygwin64, so we cannot build Cygwin in a separate image. *)
   let ocaml_for_windows =
-    let extra, vs_build_tools =
-      match distro with
-      | `Windows (`Mingw, _) -> (Windows.Cygwin.mingw_packages, empty)
-      | `Windows (`Msvc, _) ->
-          ( Windows.Cygwin.msvc_packages,
-            Windows.install_visual_studio_build_tools
-              [
-                "Microsoft.VisualStudio.Component.VC.Tools.x86.x64";
-                "Microsoft.VisualStudio.Component.Windows10SDK.18362";
-              ] )
-      | _ -> invalid_arg "Invalid distribution"
-    in
-    let extra, pkgs = Windows.Cygwin.ocaml_for_windows_packages ~extra () in
-    Windows.Cygwin.install_from_release ~msvs_tools:true ~extra ()
-    @@ vs_build_tools @@ pkgs
+    let packages, setup = Windows.Cygwin.install_ocaml_for_windows () in
+    let packages = Windows.Cygwin.mingw_packages @ packages in
+    Windows.Cygwin.install_cygwin ~extra:packages () @@ setup
   in
-  winget_image @@ opams_image
+  parser_directive (`Escape '`')
+  @@ comment "Autogenerated by OCaml-Dockerfile scripts"
+  @@ winget_image @@ opams_image
   @@ header ?win10_revision ?arch distro
   @@ label (("distro_style", "windows") :: labels)
   @@ user "ContainerAdministrator"
-  @@ Windows.install_vc_redist ()
   @@ Windows.sanitize_reg_path ()
   @@ winget_setup @@ ocaml_for_windows
   @@ copy_opams_windows opam_branches
   @@ Windows.Cygwin.setup () @@ Windows.Cygwin.Git.init ()
 
+(* Native Windows with MSVC and WinGet. *)
+let windows_msvc_opam2 ?win10_revision ?winget ?(labels = []) ~opam_hashes
+    distro () =
+  let opam_master_hash, opam_branches =
+    create_opam_branches_windows opam_hashes
+  in
+  let version = match distro with `Windows (_, v) -> v | _ -> assert false in
+  let winget_image, winget_setup =
+    install_winget ?win10_revision ?winget version
+  in
+  let cygwin_msvc_image =
+    let packages, vs_build_tools =
+      ( Windows.Cygwin.msvc_packages,
+        Windows.install_visual_studio_build_tools
+          [
+            "Microsoft.VisualStudio.Component.VC.Tools.x86.x64";
+            (* Without 18362, rc.exe is missing from the Path. *)
+            "Microsoft.VisualStudio.Component.Windows10SDK.18362";
+          ] )
+    in
+    Windows.header ~alias:"cygwin-msvc" ?win10_revision ~version ()
+    @@ Windows.sanitize_reg_path ()
+    @@ vs_build_tools
+    @@ Windows.Cygwin.install_cygwin ~msvs_tools:true ~extra:packages ()
+  in
+  let opams_image =
+    Dockerfile.from ~alias:"opam-builder" "cygwin-msvc"
+    @@ install_opams_windows ~msvs:true opam_master_hash opam_branches
+  in
+  (* 2022-10-12: Docker Engine 20.10.18 on Windows fails copying
+     C:\cygwin64, so we cannot build Cygwin in a separate image. *)
+  let ocaml_for_windows =
+    let packages, setup = Windows.Cygwin.install_ocaml_for_windows () in
+    Windows.Cygwin.install packages @@ setup
+  in
+  parser_directive (`Escape '`')
+  @@ comment "Autogenerated by OCaml-Dockerfile scripts"
+  @@ winget_image @@ cygwin_msvc_image @@ opams_image
+  @@ Dockerfile.from "cygwin-msvc"
+  @@ label (("distro_style", "windows") :: labels)
+  @@ user "ContainerAdministrator"
+  @@ ocaml_for_windows @@ winget_setup
+  @@ copy_opams_windows opam_branches
+  @@ Windows.Cygwin.setup () @@ Windows.Cygwin.Git.init ()
+
 let gen_opam2_distro ?win10_revision ?winget ?(clone_opam_repo = true) ?arch
     ?labels ~opam_hashes d =
   let fn =
@@ -488,8 +534,14 @@ let gen_opam2_distro ?win10_revision ?winget ?(clone_opam_repo = true) ?arch
           ~c_devtools_libs ~opam_hashes d ()
     | `Zypper -> zypper_opam2 ?labels ?arch ~opam_hashes d ()
     | `Pacman -> pacman_opam2 ?labels ?arch ~opam_hashes d ()
-    | `Windows ->
-        windows_opam2 ?win10_revision ?winget ?labels ?arch ~opam_hashes d ()
+    | `Windows -> (
+        match d with
+        | `Windows (`Mingw, _) ->
+            windows_mingw_opam2 ?win10_revision ?winget ?labels ?arch
+              ~opam_hashes d ()
+        | `Windows (`Msvc, _) ->
+            windows_msvc_opam2 ?win10_revision ?winget ?labels ~opam_hashes d ()
+        | _ -> assert false)
     | `Cygwin ->
         failwith
           "OCaml/opam Docker images with the Cygwin port are not supported."