From e74cf5528e22489c1dbf02701a1755478d39cf11 Mon Sep 17 00:00:00 2001 From: Julian Ospald Date: Mon, 6 Dec 2021 11:06:32 +0100 Subject: [PATCH] Build on FreeBSD12 only (#2420) * Build on FreeBSD12 only * Use deb10 for linux builds * Improve gitlab CI * Fix taring up Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com> Co-authored-by: Javier Neira --- .gitlab-ci.yml | 227 +++++++++++++++++++++++++++++++++++++++++++------ .gitlab/ci.sh | 55 +++++++++--- .gitlab/tar.sh | 26 ++++++ 3 files changed, 268 insertions(+), 40 deletions(-) create mode 100755 .gitlab/tar.sh diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index b52ea3e9ff..f7036a699e 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -1,10 +1,14 @@ -# Used for ci setup in the gitlab mirror of the project: +stages: + - build + - tar + +# Used for ci setup in the gitlab mirror of the project: # https://gitlab.haskell.org/haskell/haskell-language-server/-/pipelines variables: # Commit of ghc/ci-images repository from which to pull Docker images - DOCKER_REV: "853f348f9caf38b08740b280296fbd34e09abb3a" + DOCKER_REV: "4ed1a4f27828ba96a34662dc954335e29b470cd2" - CABAL_INSTALL_VERSION: 3.4.0.0 + CABAL_INSTALL_VERSION: 3.6.2.0 .default_matrix: &default_matrix matrix: @@ -32,20 +36,45 @@ workflow: when: always - when: never -.build: - script: - - bash .gitlab/ci.sh +.artifacts: artifacts: expire_in: 2 week paths: - out +.artifacts:short: + artifacts: + expire_in: 1 day + paths: + - out + +.build: + extends: .artifacts:short + stage: build + script: + - bash .gitlab/ci.sh + build-aarch64-linux-deb10: extends: .build tags: - aarch64-linux image: "registry.gitlab.haskell.org/ghc/ci-images/aarch64-linux-deb10:$DOCKER_REV" parallel: *arm_matrix + variables: + ADD_CABAL_ARGS: "" + +tar-aarch64-linux-deb10: + extends: .artifacts + stage: tar + needs: ["build-aarch64-linux-deb10"] + tags: + - aarch64-linux + image: "registry.gitlab.haskell.org/ghc/ci-images/aarch64-linux-deb10:$DOCKER_REV" + script: + - ./.gitlab/tar.sh + variables: + TARBALL_ARCHIVE_SUFFIX: aarch64-linux-deb10 + TARBALL_EXT: tar.xz build-armv7-linux-deb10: extends: .build @@ -53,62 +82,206 @@ build-armv7-linux-deb10: - armv7-linux image: "registry.gitlab.haskell.org/ghc/ci-images/armv7-linux-deb10:$DOCKER_REV" parallel: *arm_matrix + variables: + ADD_CABAL_ARGS: "" + +tar-armv7-linux-deb10: + extends: .artifacts + stage: tar + needs: ["build-armv7-linux-deb10"] + tags: + - armv7-linux + image: "registry.gitlab.haskell.org/ghc/ci-images/armv7-linux-deb10:$DOCKER_REV" + script: + - ./.gitlab/tar.sh + variables: + TARBALL_ARCHIVE_SUFFIX: armv7-linux-deb10 + TARBALL_EXT: tar.xz build-x86_64-linux: extends: .build tags: - x86_64-linux - image: "registry.gitlab.haskell.org/ghc/ci-images/x86_64-linux-deb9:$DOCKER_REV" + image: "registry.gitlab.haskell.org/ghc/ci-images/x86_64-linux-deb10:$DOCKER_REV" parallel: *default_matrix + variables: + ADD_CABAL_ARGS: "--enable-split-sections" + +tar-x86_64-linux: + extends: .artifacts + stage: tar + needs: ["build-x86_64-linux"] + tags: + - x86_64-linux + image: "registry.gitlab.haskell.org/ghc/ci-images/x86_64-linux-deb10:$DOCKER_REV" + script: + - ./.gitlab/tar.sh + variables: + TARBALL_ARCHIVE_SUFFIX: x86_64-linux + TARBALL_EXT: tar.xz build-x86_64-linux-alpine: extends: .build tags: - x86_64-linux image: "registry.gitlab.haskell.org/ghc/ci-images/x86_64-linux-alpine3_12:$DOCKER_REV" + before_script: + - sudo apk add --no-cache zlib zlib-dev zlib-static bzip2 bzip2-dev bzip2-static gmp gmp-dev xz xz-dev ncurses-static + parallel: *default_matrix + variables: + ADD_CABAL_ARGS: "--enable-split-sections --enable-executable-static" + +tar-x86_64-linux-alpine: + extends: .artifacts + stage: tar + needs: ["build-x86_64-linux-alpine"] + tags: + - x86_64-linux + image: "registry.gitlab.haskell.org/ghc/ci-images/x86_64-linux-alpine3_12:$DOCKER_REV" + script: + - ./.gitlab/tar.sh + variables: + TARBALL_ARCHIVE_SUFFIX: x86_64-linux-alpine + TARBALL_EXT: tar.xz + +build-x86_64-freebsd12: + extends: .build + tags: + - x86_64-freebsd12 parallel: *default_matrix + variables: + ADD_CABAL_ARGS: "--enable-split-sections" + +tar-x86_64-freebsd12: + extends: .artifacts + stage: tar + needs: ["build-x86_64-freebsd12"] + tags: + - x86_64-freebsd12 + script: + - ./.gitlab/tar.sh + variables: + TARBALL_ARCHIVE_SUFFIX: x86_64-freebsd12 + TARBALL_EXT: tar.xz -build-x86_64-freebsd: +build-x86_64-freebsd13: extends: .build tags: - x86_64-freebsd13 parallel: *default_matrix + before_script: + - sudo pkg update + - sudo pkg install --yes compat12x-amd64 + - sudo ln -s libncurses.so.6 /usr/local/lib/libncurses.so.6.2 + variables: + ADD_CABAL_ARGS: "--enable-split-sections" + +tar-x86_64-freebsd13: + extends: .artifacts + stage: tar + needs: ["build-x86_64-freebsd13"] + tags: + - x86_64-freebsd13 + script: + - ./.gitlab/tar.sh + variables: + TARBALL_ARCHIVE_SUFFIX: x86_64-freebsd13 + TARBALL_EXT: tar.xz build-x86_64-darwin: extends: .build tags: - x86_64-darwin parallel: *default_matrix + variables: + ADD_CABAL_ARGS: "" + +tar-x86_64-darwin: + extends: .artifacts + stage: tar + needs: ["build-x86_64-darwin"] + tags: + - x86_64-darwin + script: + - ./.gitlab/tar.sh + variables: + TARBALL_ARCHIVE_SUFFIX: x86_64-darwin + TARBALL_EXT: tar.xz build-aarch64-darwin: + extends: .artifacts:short + stage: build tags: - aarch64-darwin-m1 + before_script: + # Install brew locally in the project dir. Packages will also be installed here. + - '[ -e "$CI_PROJECT_DIR/.brew" ] || git clone --depth=1 https://github.com/Homebrew/brew $CI_PROJECT_DIR/.brew' + - export PATH="$CI_PROJECT_DIR/.brew/bin:$CI_PROJECT_DIR/.brew/sbin:$PATH" + + # otherwise we seem to get intel binaries + - export HOMEBREW_CHANGE_ARCH_TO_ARM=1 + + # make sure to not pollute the machine with temp files etc + - mkdir -p $CI_PROJECT_DIR/.brew_cache + - export HOMEBREW_CACHE=$CI_PROJECT_DIR/.brew_cache + - mkdir -p $CI_PROJECT_DIR/.brew_logs + - export HOMEBREW_LOGS=$CI_PROJECT_DIR/.brew_logs + - mkdir -p /private/tmp/.brew_tmp + - export HOMEBREW_TEMP=/private/tmp/.brew_tmp + + # update and install packages + - brew update + - brew install llvm + - brew install autoconf automake coreutils script: | - set -Eeuo pipefail - function runInNixShell() { - time nix-shell $CI_PROJECT_DIR/.gitlab/shell-aarch64-darwin.nix \ - -I nixpkgs=https://github.com/angerman/nixpkgs/archive/75f7281738b.tar.gz \ - --argstr system "aarch64-darwin" \ - --pure \ - --keep CI_PROJECT_DIR \ - --keep MACOSX_DEPLOYMENT_TARGET \ - --keep GHC_VERSION \ - --keep CABAL_PROJECT \ - --keep CABAL_INSTALL_VERSION \ - --run "$1" 2>&1 - } - runInNixShell "cabal update && mkdir vendored && cd vendored && cabal unpack network-3.1.2.1 && cd network-3.1.2.1 && autoreconf -fi" 2>&1 - runInNixShell "./.gitlab/ci.sh" 2>&1 + export PATH="$CI_PROJECT_DIR/.brew/opt/llvm/bin:$CI_PROJECT_DIR/.brew/bin:$CI_PROJECT_DIR/.brew/sbin:$PATH" + export CC=$CI_PROJECT_DIR/.brew/opt/llvm/bin/clang + export CXX=$CI_PROJECT_DIR/.brew/opt/llvm/bin/clang++ + export LD=ld + export AR=$CI_PROJECT_DIR/.brew/opt/llvm/bin/llvm-ar + export RANLIB=$CI_PROJECT_DIR/.brew/opt/llvm/bin/llvm-ranlib + arch -arm64 /bin/bash ./.gitlab/ci.sh + after_script: + - rm -Rf /private/tmp/.brew_tmp variables: MACOSX_DEPLOYMENT_TARGET: "10.7" + ADD_CABAL_ARGS: "" parallel: *m1_matrix - artifacts: - expire_in: 2 week - paths: - - out + +tar-aarch64-darwin: + extends: .artifacts + stage: tar + needs: ["build-aarch64-darwin"] + tags: + - aarch64-darwin-m1 + script: + - arch -arm64 /bin/bash ./.gitlab/tar.sh + variables: + TARBALL_ARCHIVE_SUFFIX: aarch64-darwin + TARBALL_EXT: tar.xz build-x86_64-windows: extends: .build tags: - new-x86_64-windows parallel: *default_matrix + script: + - bash '-lc' 'pacman --noconfirm -S zip' + - $env:CHERE_INVOKING = "yes" + - bash '-lc' "ADD_CABAL_ARGS=$env:ADD_CABAL_ARGS GHC_VERSION=$env:GHC_VERSION CABAL_INSTALL_VERSION=$CABAL_INSTALL_VERSION .gitlab/ci.sh" + variables: + ADD_CABAL_ARGS: "" + +tar-x86_64-windows: + extends: .artifacts + stage: tar + needs: ["build-x86_64-windows"] + tags: + - new-x86_64-windows + script: + - $env:CHERE_INVOKING = "yes" + - bash '-lc' "TARBALL_ARCHIVE_SUFFIX=$env:TARBALL_ARCHIVE_SUFFIX TARBALL_EXT=$env:TARBALL_EXT .gitlab/tar.sh" + variables: + TARBALL_ARCHIVE_SUFFIX: x86_64-windows + TARBALL_EXT: zip + diff --git a/.gitlab/ci.sh b/.gitlab/ci.sh index 30133f8422..26ab30c85e 100755 --- a/.gitlab/ci.sh +++ b/.gitlab/ci.sh @@ -4,7 +4,6 @@ set -Eeuxo pipefail source "$CI_PROJECT_DIR/.gitlab/common.sh" - export GHCUP_INSTALL_BASE_PREFIX="$CI_PROJECT_DIR/toolchain" export CABAL_DIR="$CI_PROJECT_DIR/cabal" @@ -28,24 +27,54 @@ export BOOTSTRAP_HASKELL_CABAL_VERSION="$CABAL_INSTALL_VERSION" export BOOTSTRAP_HASKELL_VERBOSE=1 export BOOTSTRAP_HASKELL_ADJUST_CABAL_CONFIG=yes -curl --proto '=https' --tlsv1.2 -sSf https://get-ghcup.haskell.org | sh +# for some reason the subshell doesn't pick up the arm64 environment on darwin +# and starts installing x86_64 GHC +case "$(uname -s)" in + "Darwin"|"darwin") + case "$(/usr/bin/arch)" in + aarch64|arm64|armv8l) + curl --proto '=https' --tlsv1.2 -sSf https://get-ghcup.haskell.org | arch -arm64 /bin/bash + ;; + *) + curl --proto '=https' --tlsv1.2 -sSf https://get-ghcup.haskell.org | sh + ;; + esac + ;; + *) + curl --proto '=https' --tlsv1.2 -sSf https://get-ghcup.haskell.org | sh + ;; +esac # some alpines need workaround if ghc --info | grep -q integer-simple ; then echo -e 'package blaze-textual\n flags: +integer-simple' >> cabal.project.local fi -run cabal v2-install exe:haskell-language-server exe:haskell-language-server-wrapper \ - -O2 \ - -w "ghc-$GHC_VERSION" \ - --project-file "$CABAL_PROJECT" \ - --installdir="$CI_PROJECT_DIR/out" \ - --install-method=copy \ - --overwrite-policy=always \ - --enable-executable-static \ - --disable-profiling \ - --disable-tests \ - --enable-split-sections \ +# Shorten binary names +sed -i.bak -e 's/haskell-language-server/hls/g' \ + -e 's/haskell_language_server/hls/g' \ + haskell-language-server.cabal cabal.project +sed -i.bak -e 's/Paths_haskell_language_server/Paths_hls/g' \ + src/**/*.hs exe/*.hs + +args=( + -O2 + -w "ghc-$GHC_VERSION" + --project-file "$CABAL_PROJECT" + --disable-profiling + --disable-tests --enable-executable-stripping + ${ADD_CABAL_ARGS} +) + +run cabal v2-build ${args[@]} exe:hls exe:hls-wrapper + +mkdir "$CI_PROJECT_DIR/out" + +cp "$(cabal list-bin ${args[@]} exe:hls)" "$CI_PROJECT_DIR/out/haskell-language-server-${GHC_VERSION}" +cp "$(cabal list-bin ${args[@]} exe:hls-wrapper)" "$CI_PROJECT_DIR/out/haskell-language-server-wrapper" cp dist-newstyle/cache/plan.json "$CI_PROJECT_DIR/out/plan.json" + +cd "$CI_PROJECT_DIR/out/" + diff --git a/.gitlab/tar.sh b/.gitlab/tar.sh new file mode 100755 index 0000000000..7fd719589d --- /dev/null +++ b/.gitlab/tar.sh @@ -0,0 +1,26 @@ +#!/usr/bin/env bash + +set -Eeuxo pipefail + +source "$CI_PROJECT_DIR/.gitlab/common.sh" + +ls -la out/ +cd out/ + +# create tarball/zip +TARBALL_PREFIX="haskell-language-server-$("$CI_PROJECT_DIR/out/haskell-language-server-wrapper" --numeric-version)" +case "${TARBALL_EXT}" in + zip) + zip "${TARBALL_PREFIX}-${TARBALL_ARCHIVE_SUFFIX}.${TARBALL_EXT}" haskell-language-server-* + find . -type f ! -name '*.zip' -delete + ;; + tar.xz) + tar caf "${TARBALL_PREFIX}-${TARBALL_ARCHIVE_SUFFIX}.${TARBALL_EXT}" haskell-language-server-* + find . -type f ! -name '*.tar.xz' -delete + ;; + *) + fail "Unknown TARBALL_EXT: ${TARBALL_EXT}" + ;; +esac + +