[3.6] Backport #6540 (add MSVC CI)

.gitattributes

@@ -2,6 +2,7 @@
 *.rst             text eol=lf
 *.c               text eol=lf
 *.t               text eol=lf -linguist-detectable
+*.ps1             text working-tree-encoding=UTF-16 eol=crlf
 dune              text eol=lf
 dune.inc          text eol=lf
 .gitignore        text eol=lf

.github/workflows/workflow.yml

@@ -102,6 +102,36 @@ jobs:
         run: opam install ./dune-configurator.opam
         if: ${{ matrix.configurator == true }}
 
+  dkml:
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+          - gh_os: windows-2019
+            abi_pattern: win32-windows_x86
+            dkml_host_abi: windows_x86
+          - gh_os: windows-2019
+            abi_pattern: win32-windows_x86_64
+            dkml_host_abi: windows_x86_64
+    runs-on: ${{ matrix.gh_os }}
+    name: MSVC / ${{ matrix.dkml_host_abi }}
+
+    steps:
+      - uses: actions/checkout@v3
+
+      - name: Setup DKML on a Windows host
+        if: startsWith(matrix.dkml_host_abi, 'windows_')
+        uses: ./ci/setup-dkml/gh-windows/pre
+
+      - name: Build and test the package on Windows host
+        if: startsWith(matrix.dkml_host_abi, 'windows_')
+        shell: msys2 {0}
+        run: ci/build-test.sh
+
+      - name: Teardown DKML on a Windows host
+        if: startsWith(matrix.dkml_host_abi, 'windows_')
+        uses: ./ci/setup-dkml/gh-windows/post
+
   nix:
     name: Nix
     strategy:

.gitignore

@@ -24,3 +24,7 @@ result
 
 .DS_Store
 nix/profiles/
+
+# dkml desktop CI
+/msys64
+/.ci

CHANGES.md

@@ -5,8 +5,6 @@
 
 - Fix build with MSVC compiler (#6517, @nojb)
 
-- Do not shadow library interface modules (#6549, fixes #6545, @rgrinberg)
-
 3.6.0 (2022-11-14)
 ------------------
 

Makefile

@@ -31,6 +31,7 @@ ctypes \
 # Dependencies recommended for developing dune locally,
 # but not wanted in CI
 DEV_DEPS := \
+"dkml-workflows>=1.1.0" \
 patdiff
 
 TEST_OCAMLVERSION := 4.14.0
@@ -74,6 +75,10 @@ dev-depext:
 dev-deps:
 	opam install -y $(TEST_DEPS)
 
+.PHONY: dev-deps-sans-melange
+dev-deps-sans-melange:
+	opam install -y $(TEST_DEPS)
+
 .PHONY: dev-switch
 dev-switch:
 	opam update
@@ -101,6 +106,9 @@ test-melange: $(BIN)
 test-all: $(BIN)
 	$(BIN) build @runtest @runtest-js @runtest-coq @runtest-melange
 
+test-all-sans-melange: $(BIN)
+	$(BIN) build @runtest @runtest-js @runtest-coq
+
 .PHONY: check
 check: $(BIN)
 	@$(BIN) build @check
@@ -137,6 +145,39 @@ livedoc:
 update-jbuilds: $(BIN)
 	$(BIN) build @doc/runtest --auto-promote
 
+# you will need to use a dev-switch or do opam install dkml-workflows.
+# we run --auto-promote twice since first may fail (ex. missing files with Unable to resolve symlink)
+update-dkml: $(BIN)
+	opam exec -- generate-setup-dkml-scaffold
+	$(BIN) build @gen-dkml --auto-promote || $(BIN) build @gen-dkml --auto-promote
+	rm -rf ci/setup-dkml/gh-darwin ci/setup-dkml/gh-linux ci/setup-dkml/gl
+	rm -rf ci/setup-dkml/pc/setup-dkml-darwin_*.sh ci/setup-dkml/pc/setup-dkml-linux_*.sh
+	$(BIN) build @ci/fmt --auto-promote || $(BIN) build @ci/fmt --auto-promote
+
+# assumes MSYS2 or Cygwin, and Visual Studio. Do not use 'with-dkml make ...'
+desktop-ci-windows_x86:
+	if command -v pwsh; then \
+	  pwsh ./ci/setup-dkml/pc/setup-dkml-windows_x86.ps1; \
+	else \
+	  powershell ./ci/setup-dkml/pc/setup-dkml-windows_x86.ps1; \
+	fi
+	/usr/bin/env PATH=/usr/bin \
+	  dkml_host_abi=windows_x86 abi_pattern=win32-windows_x86_64; \
+	  opam_root=.ci/o exe_ext=.exe \
+	  /bin/sh ci/build-test.sh
+
+# assumes MSYS2 or Cygwin, and Visual Studio. Do not use 'with-dkml make ...'
+desktop-ci-windows_x86_64:
+	if command -v pwsh; then \
+	  pwsh ./ci/setup-dkml/pc/setup-dkml-windows_x86_64.ps1; \
+	else \
+	  powershell ./ci/setup-dkml/pc/setup-dkml-windows_x86_64.ps1; \
+	fi
+	/usr/bin/env PATH=/usr/bin \
+	  dkml_host_abi=windows_x86 abi_pattern=win32-windows_x86_64 \
+	  opam_root=.ci/o exe_ext=.exe \
+	  /bin/sh ci/build-test.sh
+
 # If the first argument is "run"...
 ifeq (dune,$(firstword $(MAKECMDGOALS)))
   # use the rest as arguments for "run"

ci/build-test.sh

@@ -0,0 +1,130 @@
+#!/bin/sh
+set -euf
+
+# shellcheck disable=SC2154
+echo "
+=============
+build-test.sh
+=============
+.
+------
+Matrix
+------
+dkml_host_abi=$dkml_host_abi
+abi_pattern=$abi_pattern
+opam_root=$opam_root
+exe_ext=${exe_ext:-}
+.
+"
+
+# Set project directory
+if [ -n "${CI_PROJECT_DIR:-}" ]; then
+    PROJECT_DIR="$CI_PROJECT_DIR"
+elif [ -n "${PC_PROJECT_DIR:-}" ]; then
+    PROJECT_DIR="$PC_PROJECT_DIR"
+elif [ -n "${GITHUB_WORKSPACE:-}" ]; then
+    PROJECT_DIR="$GITHUB_WORKSPACE"
+else
+    PROJECT_DIR="$PWD"
+fi
+if [ -x /usr/bin/cygpath ]; then
+    PROJECT_DIR=$(/usr/bin/cygpath -au "$PROJECT_DIR")
+fi
+
+# PATH. Add opamrun
+export PATH="$PROJECT_DIR/.ci/sd4/opamrun:$PATH"
+
+# Initial Diagnostics (optional but useful)
+opamrun switch
+opamrun list
+opamrun var
+opamrun config report
+opamrun option
+opamrun exec -- ocamlc -config
+
+# Update
+opamrun update
+
+# ------------------
+# Dune's build logic.
+#   For now it mimics the "build" job in .github/workflows/workflow.yml
+# ------------------
+
+echo ======== Unpin Dune from DKML defaults
+opamrun pin list
+opamrun pin remove dune --no-action
+opamrun pin remove dune-configurator --no-action
+
+echo ======== Set git user
+git config --global user.name 'github-actions[bot]'
+git config --global user.email 'github-actions[bot]@users.noreply.github.com'
+
+echo ======== Install ocamlfind-secondary and ocaml-secondary-compiler, if needed
+opamrun install ./dune.opam --deps-only --with-test
+
+case "$dkml_host_abi" in
+darwin_*)
+    echo ======== Install system deps on macOS
+    brew install coreutils
+    ;;
+esac
+
+# dune doesn't have any additional dependencies so we can build it right
+# away this makes it possible to see build errors as soon as possible
+echo ======== Build boot dune.exe
+opamrun exec -- make _boot/dune.exe
+
+echo ======== Ensure Dune can build itself
+opamrun exec -- make bootstrap
+
+case "$dkml_host_abi" in
+windows_*)
+    # Mitigate bug in opam:
+    #
+    #    #=== ERROR while compiling dune.3.6.0 =========================================#
+    #    # path        Z:\source\dune
+    #    # command     C:\Users\beckf\AppData\Local\Programs\DiskuvOCaml\tools\MSYS2\usr\bin\cp.exe -PRp Z:\source\dune\.ci\o\dkml\.opam-switch\sources\dune.3.6.0 Z:\source\dune\.ci\o\dkml\.opam-switch\build\dune.3.6.0
+    #    # exit-code   1
+    #    # env-file    Z:\source\dune\.ci\o\log\log-685364-345b91.env
+    #    # output-file Z:\source\dune\.ci\o\log\log-685364-345b91.out
+    #    ### output ###
+    #    # /usr/bin/cp: cannot create regular file 'Z:\source\dune\.ci\o\dkml\.opam-switch\build\dune.3.6.0/dune': File exists
+    #
+    # Basically the 'cp' fallback logic in opam's src/core/opamSystem.ml:copy_dir is broken.
+    # Using 'rsync' avoids that, so install it on MSYS2.
+    if command -v pacman; then
+        echo ======== Install rsync on Win32
+        pacman --sync --needed --noconfirm rsync
+    fi
+
+    echo ======== Install deps on Win32
+    opamrun install ./dune-configurator.opam --deps-only --with-test --yes
+
+    echo ======== Run test suite on Win32
+    opamrun exec -- make test-windows
+    ;;
+*)
+    echo ======== Install deps on Unix
+    opamrun install . --deps-only --with-test --yes
+
+    # Remove DKML pins (keep alphabetical; the pins will eventually be upstreamed in DKML)
+    opamrun pin remove base --no-action
+    opamrun pin remove core_kernel --no-action
+    opamrun pin remove ocamlformat --no-action
+    opamrun pin remove ppx_expect --no-action
+    opamrun pin remove time_now --no-action
+
+    # This should have been:
+    #   opam exec -- make dev-deps
+    # but melange requires ocaml 4.14 (DKML hasn't been updated as of 2022-11)
+    opamrun exec -- make dev-deps-sans-melange
+
+    echo ======== Run test suite on Unix
+    # This should have been:
+    #   opam exec -- make test
+    # but melange requires ocaml 4.14 (DKML hasn't been updated as of 2022-11)
+    opamrun exec -- make test-all-sans-melange
+esac
+
+echo ======== Build configurator
+opamrun install ./dune-configurator.opam --yes

ci/setup-dkml/README.md

@@ -0,0 +1,32 @@
+# setup-dkml
+
+Everything in this folder except this README.md is auto-generated by `setup-dkml`.
+
+The documentation for `setup-dkml` is at https://github.com/diskuv/dkml-workflows#readme.
+
+Bug reports go to https://github.com/diskuv/dkml-workflows/issues
+
+## FAQ
+
+### What is the workflow to update the generated files?
+
+Use the following to update the generated code:
+
+```bash
+opam update
+make update-dkml
+```
+
+Of course that only works if there are updates available.
+
+### What is the version of MSVC targeted? What is the required environment to execute the `desktop-ci-*` targets in the main `Makefile`?
+
+There is no "target" per se. The CI machine (or your developer machine) is queried for all MSVC versions, and any versions that are known to be compatible with OCaml are used.
+The precise definition is `Get-CompatibleVisualStudios` at https://github.com/diskuv/dkml-runtime-distribution/blob/main/src/windows/Machine/Machine.psm1; that package is available for common use in the central Opam repository.
+
+The compatible versions as of 2022-11-23 are Visual Studio 2019 with
+a) Windows 10 SDK (10.0.18362.0) and b) MSVC v142 - VS 2019 C++ x64/x86 build tools (v14.25) or (v14.26).
+
+*Advanced: Machine.psm1 has notes why other MSVC 2019 versions are not compatible with OCaml*
+
+More importantly, just running it will show what is on the system and what Visual Studio components are missing (if any).

ci/setup-dkml/gh-windows/post/action.yml

@@ -0,0 +1,102 @@
+# setup-dkml
+#   Short form: sd4
+
+# Any GitHub Job that includes this action must be in a strategy matrix.
+# The matrix variables must include:
+# - gh_os: windows-2019
+#   abi_pattern: win32-windows_x86
+#   dkml_host_abi: windows_x86
+
+name: post-dkml
+author: Diskuv, Inc.
+description: Teardown after building OCaml native executables for Windows
+
+runs:
+  using: "composite"
+
+  steps:
+    - name: Full matrix variables
+      shell: bash # bash on Windows is Git Bash (an non-upgradable MSYS2 system)
+      # Every matrix variable lookup in this Action should use the output of this step. Even
+      # the matrix variables that the user must specify (ex. dkml_host_abi) should be
+      # referenced using [steps.full_matrix_vars.outputs.dkml_host_abi] rather than
+      # [matrix.dkml_host_abi] so that there is a single place to edit for variable changes.
+      id: full_matrix_vars
+      run: |
+        # Select correct Windows matrix variables
+        case "${{ matrix.dkml_host_abi }}" in
+
+        windows_x86)
+          dkml_host_os='windows';
+          opam_root_cacheable='D:/.opam';
+          abi_pattern='win32-windows_x86';
+          gh_os='windows-2019';
+          gh_unix_shell='msys2 {0}';
+          msys2_system='MINGW32';
+          msys2_packages='mingw-w64-i686-pkg-config';
+          exe_ext='.exe';
+          bootstrap_opam_version='2.2.0-dkml20220801T155940Z';
+          opam_abi='windows_x86';
+          dkml_host_abi='windows_x86';
+          opam_root='D:/.opam';
+          vsstudio_hostarch='x64';
+          vsstudio_arch='x86';
+          ocaml_options='ocaml-option-32bit' ;;
+
+        windows_x86_64)
+          dkml_host_os='windows';
+          opam_root_cacheable='D:/.opam';
+          abi_pattern='win32-windows_x86_64';
+          gh_os='windows-2019';
+          gh_unix_shell='msys2 {0}';
+          msys2_system='CLANG64';
+          msys2_packages='mingw-w64-clang-x86_64-pkg-config';
+          exe_ext='.exe';
+          bootstrap_opam_version='2.2.0-dkml20220801T155940Z';
+          opam_abi='windows_x86_64';
+          dkml_host_abi='windows_x86_64';
+          opam_root='D:/.opam';
+          vsstudio_hostarch='x64';
+          vsstudio_arch='x64' ;;
+
+        *)  echo "FATAL: Unsupported dkml_host_abi=$dkml_host_abi in Windows action.yml"; exit 107 ;;
+        esac
+
+        add() {
+          echo "$1=$2" | tee -a $GITHUB_OUTPUT | tee -a $GITHUB_ENV
+        }
+
+        add dkml_host_abi "$dkml_host_abi"
+        add abi_pattern "$abi_pattern"
+        add opam_root "$opam_root"
+        add opam_root_cacheable "$opam_root_cacheable"
+        add exe_ext "${exe_ext:-}"
+        add bootstrap_opam_version "${bootstrap_opam_version:-}"
+        add ocaml_options "${ocaml_options:-}"
+
+    - name: Teardown DKML build apparatus
+      shell: msys2 {0}
+      env:
+        _STUB_FOR_AUTOGEN: "ON" # 
+
+        # autogen from global_env_vars.
+        DEFAULT_DKML_COMPILER: '4.12.1-v1.0.2'
+        PIN_BASE: 'v0.14.3'
+        PIN_BIGSTRINGAF: '0.8.0'
+        PIN_CORE_KERNEL: 'v0.14.2'
+        PIN_CTYPES_FOREIGN: '0.19.2-windowssupport-r4'
+        PIN_CTYPES: '0.19.2-windowssupport-r4'
+        PIN_CURLY: '0.2.1-windows-env_r2'
+        PIN_DIGESTIF: '1.0.1'
+        PIN_DUNE: '2.9.3+shim.1.0.2~r0'
+        PIN_DUNE_CONFIGURATOR: '2.9.3'
+        PIN_DKML_APPS: '1.0.1'
+        PIN_OCAMLBUILD: '0.14.0'
+        PIN_OCAMLFIND: '1.9.1'
+        PIN_OCP_INDENT: '1.8.2-windowssupport'
+        PIN_PPX_EXPECT: 'v0.14.1'
+        PIN_PTIME: '0.8.6-msvcsupport'
+        PIN_TIME_NOW: 'v0.14.0'
+        PIN_WITH_DKML: '1.0.1'
+      run: |
+        sh .ci/sd4/run-teardown-dkml.sh GITHUB_WORKSPACE "$GITHUB_WORKSPACE"