Merge branch 'env-cr-win' into switch-revert

.github/scripts/cygwin.cmd

@@ -49,8 +49,6 @@ if not exist %CYGWIN_CACHE_DIR%\%CYGWIN_DISTRO%\cache.tar (
 ::        C:\Windows\system32 (overriding curl, bash, etc. in System32) but after Mercurial
 ::        and Git (so that they are not overridden).
 set Path=C:\Program Files\Mercurial;C:\Program Files\Git\cmd;%CYGWIN_ROOT%\bin;C:\Windows\system32;C:\Windows;C:\Windows\System32\Wbem;C:\Windows\System32\WindowsPowerShell\v1.0\;C:\Windows\System32\OpenSSH\
-if "%3" equ "i686-w64-mingw32" set Path=%CYGWIN_ROOT%\usr\%3\sys-root\mingw\bin;%Path%
-if "%3" equ "x86_64-w64-mingw32" set Path=%CYGWIN_ROOT%\usr\%3\sys-root\mingw\bin;%Path%
 if "%3" equ "x86_64-pc-cygwin" set Path=%CYGWIN_ROOT%\bin;%Path%
 
 ::echo %CYGWIN_ROOT%\bin>> %GITHUB_PATH%

.github/workflows/ci.ml

@@ -339,6 +339,7 @@ let main_build_job ~analyse_job ~cygwin_job ?section runner start_version ~oc ~w
     ++ build_cache OCaml platform "${{ matrix.ocamlv }}" host
     ++ run "Build" ["bash -exu .github/scripts/main/main.sh " ^ host]
     ++ not_on Windows (run "Test (basic)" ["bash -exu .github/scripts/main/test.sh"])
+    ++ only_on Windows (run ~cond:(Predicate(false, EndsWith("matrix.host", "-pc-cygwin"))) "Test \"static\" binaries on Windows" ["ldd ./opam.exe | test \"$(grep -v -F /cygdrive/c/Windows/)\" = ''"])
     ++ only_on Windows (run "Test (basic - Cygwin)" ~cond:(Predicate(true, EndsWith("matrix.host", "-pc-cygwin"))) ["bash -exu .github/scripts/main/test.sh"])
     ++ only_on Windows (run "Test (basic - native Windows)" ~env:[("OPAMROOT", {|D:\a\opam\opam\.opam|})] ~shell:"cmd" ~cond:(Predicate(false, EndsWith("matrix.host", "-pc-cygwin")))
          ({|set Path=D:\Cache\ocaml-local\bin;%Path%|} ::
@@ -470,8 +471,8 @@ let main oc : unit =
     ("OPAM12CACHE", "~/.cache/opam1.2/cache");
     (* These should be identical to the values in appveyor.yml *)
     ("OPAM_REPO", "https://github.com/ocaml/opam-repository.git");
-    ("OPAM_TEST_REPO_SHA", "b251e0d9d0a8bcb0742d4d495c70f479effe16c2");
-    ("OPAM_REPO_SHA", "b251e0d9d0a8bcb0742d4d495c70f479effe16c2");
+    ("OPAM_TEST_REPO_SHA", "c687553afb80373165068d67f33512b7ad39e6a3");
+    ("OPAM_REPO_SHA", "c687553afb80373165068d67f33512b7ad39e6a3");
     ("SOLVER", "");
     (* Cygwin configuration *)
     ("CYGWIN_MIRROR", "http://mirrors.kernel.org/sourceware/cygwin/");

.github/workflows/main.yml

@@ -42,8 +42,8 @@ env:
   OPAMBSROOT: ~/.cache/.opam.cached
   OPAM12CACHE: ~/.cache/opam1.2/cache
   OPAM_REPO: https://github.com/ocaml/opam-repository.git
-  OPAM_TEST_REPO_SHA: b251e0d9d0a8bcb0742d4d495c70f479effe16c2
-  OPAM_REPO_SHA: b251e0d9d0a8bcb0742d4d495c70f479effe16c2
+  OPAM_TEST_REPO_SHA: c687553afb80373165068d67f33512b7ad39e6a3
+  OPAM_REPO_SHA: c687553afb80373165068d67f33512b7ad39e6a3
   SOLVER:
   CYGWIN_MIRROR: http://mirrors.kernel.org/sourceware/cygwin/
   CYGWIN_ROOT: D:\cygwin
@@ -204,6 +204,9 @@ jobs:
       run: bash -exu .github/scripts/main/ocaml-cache.sh ${{ runner.os }} ${{ matrix.ocamlv }} ${{ matrix.host }}
     - name: Build
       run: bash -exu .github/scripts/main/main.sh ${{ matrix.host }}
+    - name: Test "static" binaries on Windows
+      if: endsWith(matrix.host, '-pc-cygwin') == false
+      run: ldd ./opam.exe | test "$(grep -v -F /cygdrive/c/Windows/)" = ''
     - name: Test (basic - Cygwin)
       if: endsWith(matrix.host, '-pc-cygwin')
       run: bash -exu .github/scripts/main/test.sh

configure.ac

@@ -76,6 +76,12 @@ AC_ARG_WITH([private_runtime],
   AS_HELP_STRING([--with-private-runtime],[For a mingw-w64 build, manifest the runtime DLLs locally in Opam.Runtime.arch]),,[with_private_runtime=no]
 )
 
+AC_ARG_ENABLE([static],
+  AS_HELP_STRING([--enable-static],
+    [Compile the opam binary statically. Only Musl-based Linux distributions and Windows with MinGW is currently supported. This is the default on Windows with MinGW]),
+  [],
+  [enable_static=auto])
+
 AC_ARG_ENABLE([cold_check],
   AS_HELP_STRING([--enable-cold-check],[Fail on some check necessary for make cold]),[COLD_CHECK=yes],[]
 )
@@ -113,6 +119,9 @@ AC_MSG_CHECKING([for compiler system])
 SYSTEM=`$OCAML shell/print_config.ml system 2>/dev/null | grep -F -v "Cannot find" || $OCAMLC -config | tr -d '\r' | sed -n -e "s/system: //p"`
 AC_MSG_RESULT([$SYSTEM])
 AC_SUBST(SYSTEM)
+AC_MSG_CHECKING([for compiler target])
+TARGET=`$OCAML shell/print_config.ml target 2>/dev/null | grep -F -v "Cannot find" || $OCAMLC -config | tr -d '\r' | sed -n -e "s/target: //p"`
+AC_MSG_RESULT([$TARGET])
 
 LIB_PREPEND=
 INC_PREPEND=
@@ -327,6 +336,33 @@ AS_IF([test "x${with_private_runtime}" != "xno"],[
 ],[CONF_MANIFEST_O=])
 AC_SUBST(CONF_MANIFEST_O)
 
+# Handle --enable-static
+AC_MSG_CHECKING([for linking method])
+support_static=no
+default_static=no
+AS_CASE([$TARGET],
+  [*-linux-musl*],[
+    support_static=yes
+    platform_dependant_stuff="-cclib -lstdc++ -cclib -static-libgcc -cclib -static"
+  ],
+  [*-*-mingw32*],[
+    support_static=yes
+    default_static=yes
+    # NOTE: On Windows, the Windows specific dlls should stay dynamic for security reasons
+    # NOTE: -l:libstdc++.a is necessary (vs. -lstdc++) as flexlink will use libstdc++.dll.a
+    # which still depends on the DLL at runtime instead of libstdc++.a (that looks like a bug in flexlink)
+    platform_dependant_stuff="-cclib -lopam_stubs_win32_stubs -cclib -l:libstdc++.a -cclib -l:libpthread.a -cclib -Wl,-static -cclib -ladvapi32 -cclib -lgdi32 -cclib -luser32 -cclib -lshell32"
+  ])
+AS_CASE([${support_static},${enable_static}],
+  [no,yes],[AC_MSG_ERROR([--enable-static is not available on this platform (${TARGET}).])],
+  [*,auto],[enable_static=${default_static}])
+AS_IF([test "${enable_static}" = yes],[
+  echo "(-noautolink -cclib -lunix -cclib -lmccs_stubs -cclib -lmccs_glpk_stubs -cclib -lsha_stubs ${platform_dependant_stuff})" > src/client/linking.sexp
+  AC_MSG_RESULT([static])
+],[
+  AC_MSG_RESULT([shared])
+])
+
 echo
 
 # Dependencies
@@ -347,7 +383,7 @@ AC_CHECK_OCAML_PKG([sha])
 AC_CHECK_OCAML_PKG([swhid_core])
 
 # Optional dependencies
-AC_CHECK_OCAML_PKG_AT_LEAST([mccs],[1.1+9])
+AC_CHECK_OCAML_PKG_AT_LEAST([mccs],[1.1+17])
 
 AS_IF([test "x${with_mccs}" = "xno" && test "x$OCAML_PKG_mccs" != "xno"],
       [AC_MSG_ERROR([Option --without-mccs is not available without uninstalling the 'mccs' package])])

doc/pages/Install.md

@@ -148,6 +148,12 @@ pkg_add opam
 
 Opam is available in the ports and packages tree on FreeBSD 11 or higher.
 
+```
+pkg install ocaml-opam
+```
+
+or to install from source:
+
 ```
 cd /usr/ports/devel/ocaml-opam
 make install

master_changes.md

@@ -28,6 +28,7 @@ users)
 ## Remove
 
 ## Switch
+  * Precise message for Ctrl-c-ed opam switch creation "Switch left partially installed" [#5713 @rjbou - fix #5710]
 
 ## Config
 
@@ -42,9 +43,14 @@ users)
 ## Update / Upgrade
 
 ## Tree
+  * Fix `--dev` option, force dev dependencies when option is given [#5687 @rjbou - fix #5675]
+  * Fix `--no-switch` option, instead of emptying switch from it installed packages, load a virtual switch at the beginning when `--no-switch` is given [#5687 @rjbou - fix #5675]
 
 ## Exec
 
+## Env
+  * Fix `opam env` containing carriage return on Cygwin [#5715 @rjbou - fix #5684]
+
 ## Source
 
 ## Lint
@@ -68,11 +74,15 @@ users)
 ## VCS
 
 ## Build
+  * Vendor mccs.1.1+17 [#5769 @kit-ty-kate]
+  * Require mccs >= 1.1+17 [#5769 @kit-ty-kate]
+  * Add ./configure --enable-static to compile the opam binary statically [#5680 @kit-ty-kate - fixes #5647]
 
 ## Infrastructure
 
 ## Release scripts
   * Workaround incorrect `NGROUPS_MAX` in `<limits.h>` in musl for release builds [#5383 @dra27]
+  * Fix check for adding `-lsha_stubs` only on `master` on OpenBSD [#5733 @punchagan]
 
 ## Admin
 
@@ -83,6 +93,7 @@ users)
 ## Opam file format
 
 ## Solver
+  * Fix debug logs showing up regardless of verbosity on macOS 12.7.1 / 13.6.3 / 14.2 and FreeBSD [#5769 @kit-ty-kate]
 
 ## Client
 
@@ -97,6 +108,7 @@ users)
 ## Reftests
 ### Tests
   * switch-set: add test that checks unsetting `OPAMSWITCH` when it was set by `opam env --set-switch` on an already set `OPAMSWITCH` variable in environment [#5742 rjbou]
+  * Add some additional test to tree, for `--dev` && `--no-switch` [#5687 @rjbou]
 
 ### Engine
   * Set `SHELL` to `/bin/sh` in Windows to ensure `opam env` commands are consistent [#5723 @dra27]
@@ -105,6 +117,7 @@ users)
 ## Github Actions
 
 ## Doc
+  * Improve the installation instructions on FreeBSD [#5775 lukstafi]
 
 ## Security fixes
 
@@ -121,3 +134,5 @@ users)
 
 ## opam-core
   * `OpamSystem.apply_cygpath`: runs `cygpath` over the argument [#5723 @dra27 - function itself added in #3348]
+  * Add `OpamConsole.disable_carriage_return` to create a new stdout in binary mode
+  * `OpamConsole.msg`: Add an optional `?stdout` argument to specify which channel to use

opam-solver.opam

@@ -30,7 +30,7 @@ build: [
 depends: [
   "ocaml" {>= "4.08.0"}
   "opam-format" {= version}
-  "mccs" {>= "1.1+9"}
+  "mccs" {>= "1.1+17"}
   "dose3" {>= "6.1"}
   "cudf" {>= "0.7"}
   "re" {>= "1.9.0"}

release/Makefile

@@ -52,7 +52,7 @@ build/%.image: build/Dockerfile.%
 	docker build -t opam-build-$* -f $^ build
 	touch $@
 
-SHA_LINK = $(if $(shell tar xOf "$(OUTDIR)/opam-full-$(VERSION).tar.gz" opam-full-$(VERSION)/src_ext/Makefile.sources | grep -F URL_sha),-lsha_stubs)
+SHA_LINK = $(if $(shell mkdir -p build/tmp_extract && tar xzf "$(OUTDIR)/opam-full-$(VERSION).tar.gz" -C build/tmp_extract && grep -F URL_sha build/tmp_extract/opam-full-$(VERSION)/src_ext/Makefile.sources),-lsha_stubs)
 
 # Actually, this is for alpine 3.13, and varies
 CLINKING_linux = \
@@ -110,13 +110,13 @@ host: $(OUTDIR)/opam-full-$(VERSION).tar.gz build/$(HOST).env
 	    MAKE=$(MAKE) \
 	    $(EXPORTS_$(HOST_OS)); \
 	  cd build/opam-full-$(VERSION) && \
-	  ./configure && \
-	  { $(MAKE) lib-ext || ./configure --with-vendored-deps; } && \
+	  { ./configure --with-vendored-deps || { ./configure && $(MAKE) lib-ext; }; } && \
 	  echo "$(call LINKING,$(HOST_OS))" >src/client/linking.sexp && \
 	  $(MAKE) opam; \
 	)
 	cp build/opam-full-$(VERSION)/opam $(OUTDIR)/opam-$(VERSION)-$(HOST)
 	strip $(OUTDIR)/opam-$(VERSION)-$(HOST)
+	$(OUTDIR)/opam-$(VERSION)-$(HOST) --version
 	rm -rf build/opam-full-$(VERSION)
 
 # Containerised builds

release/readme.md

@@ -47,7 +47,7 @@
 
 ## Device requirements
 * Mac M1
-* installed: git, gpg
+* installed: git, gpg, qemu>=8.1.0, docker>=24.0.0
 * opam repo with the tag fetched
 * Have the secret key available
 * Launch docker using the Docker GUI macOS app

release/release.sh

@@ -27,7 +27,6 @@ cd "$DIR"
 LC_ALL=C
 CWD=$(pwd)
 JOBS=$(sysctl -n hw.ncpu)
-JOBS=$(echo "${JOBS} / 1.5" | bc)
 SSH="ssh -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no"
 
 OUTDIR="out/$TAG"
@@ -42,13 +41,11 @@ qemu_build() {
 
   if ! ${SSH} -p "${port}" root@localhost true; then
       qemu-img convert -O raw "./qemu-base-images/${image}.qcow2" "./qemu-base-images/${image}.raw"
-      "qemu-system-${arch}" -drive "file=./qemu-base-images/${image}.raw,format=raw" -nic "user,hostfwd=tcp::${port}-:22" -m 2G &
+      "qemu-system-${arch}" -drive "file=./qemu-base-images/${image}.raw,format=raw" -nic "user,hostfwd=tcp::${port}-:22" -m 2G -smp "${JOBS}" &
       sleep 60
   fi
   ${SSH} -p "${port}" root@localhost "${install}"
-  # NOTE: JOBS=1 because qemu does not support proper multithreading from arm64 to x86_64 yet because of memory model differences.
-  # See https://wiki.qemu.org/Features/tcg-multithread
-  make TAG="$TAG" JOBS=1 qemu QEMU_PORT="${port}" REMOTE_MAKE="${make}" REMOTE_DIR=opam-release-$TAG
+  make TAG="$TAG" JOBS="${JOBS}" qemu QEMU_PORT="${port}" REMOTE_MAKE="${make}" REMOTE_DIR="opam-release-$TAG"
   ${SSH} -p "${port}" root@localhost "shutdown -p now"
 }
 
@@ -62,6 +59,6 @@ make JOBS="${JOBS}" TAG="$TAG" s390x-linux
 [ -f "${OUTDIR}/opam-$TAG-x86_64-macos" ] || make TAG="$TAG" JOBS="${JOBS}" macos-local MACOS_ARCH=x86_64 REMOTE_DIR=opam-release-$TAG GIT_URL="$CWD/.."
 [ -f "${OUTDIR}/opam-$TAG-arm64-macos" ] || make TAG="$TAG" JOBS="${JOBS}" macos-local MACOS_ARCH=arm64 REMOTE_DIR=opam-release-$TAG GIT_URL="$CWD/.."
 [ -d ./qemu-base-images ] || git clone https://gitlab.com/kit-ty-kate/qemu-base-images.git
-[ -f "${OUTDIR}/opam-$TAG-x86_64-openbsd" ] || qemu_build 9999 OpenBSD-7.0-amd64 "pkg_add gmake curl bzip2" gmake x86_64 &
-[ -f "${OUTDIR}/opam-$TAG-x86_64-freebsd" ] || qemu_build 9998 FreeBSD-13.0-RELEASE-amd64 "env IGNORE_OSVERSION=yes pkg install -y gmake curl bzip2" gmake x86_64 &
+[ -f "${OUTDIR}/opam-$TAG-x86_64-openbsd" ] || qemu_build 9999 OpenBSD-7.4-amd64 "pkg_add gmake curl bzip2" gmake x86_64 &
+[ -f "${OUTDIR}/opam-$TAG-x86_64-freebsd" ] || qemu_build 9998 FreeBSD-13.2-RELEASE-amd64 "pkg install -y gmake curl bzip2" gmake x86_64 &
 wait