Merge pull request #13 from clash-lang/peter/clean-setup-ci

Setup CI

.ci/build.sh

@@ -0,0 +1,26 @@
+#!/bin/bash
+set -xueo pipefail
+
+# test that we can create a build plan with the index-state in cabal.project
+mv cabal.project.local cabal.project.local.disabled
+[[ ! -f cabal.project.freeze ]] || mv cabal.project.freeze cabal.project.freeze.disabled
+if ! cabal v2-build --dry-run all > /dev/null; then
+  echo Maybe the index-state should be updated?
+  exit 1
+fi
+mv cabal.project.local.disabled cabal.project.local
+[[ ! -f cabal.project.freeze.disabled ]] || mv cabal.project.freeze.disabled cabal.project.freeze
+
+# Build with default constraints
+cabal v2-build all
+
+# Put all the test binaries in a predictable location
+TESTS="
+clash-cores:doctests
+clash-cores:unit-tests
+cores-hdl-tests:cores-hdl-tests
+"
+mkdir bin
+for TEST in $TESTS; do
+  ln -s "$(realpath --relative-to=bin "$(cabal list-bin $TEST)")" bin/$TEST
+done

.ci/build_docs.sh

@@ -0,0 +1,44 @@
+#!/bin/bash
+set -xueo pipefail
+
+mkdir -p hadocs
+
+# Cache dependencies
+cabal v2-build clash-cores -O0 --enable-documentation --only-dependencies
+
+cabal v2-haddock clash-cores -O0 --enable-documentation \
+  |  tee haddock_log
+
+set +e
+
+# Temporarily disabled, as there are hundreds of TH-generated instances without
+# documentation.
+#if grep -q "Missing documentation" haddock_log; then
+#  echo -e "\e[1m\e[31mMissing documentation! Scroll up for full log.\e[0m"
+#  grep --color=always -n -C 5 "Missing documentation" haddock_log
+#  exit 1
+#fi
+
+out_of_scope_warn="If you qualify the identifier, haddock can try to link it anyway"
+if grep -q "${out_of_scope_warn}" haddock_log; then
+  echo -e "\e[1m\e[31mIdentifier out of scope! Scroll up for full log.\e[0m"
+  grep --color=always -n -C 5 "${out_of_scope_warn}" haddock_log
+  exit 1
+fi
+
+link_dest_warn="could not find link destinations for:"
+if grep -q "${link_dest_warn}" haddock_log; then
+  echo -e "\e[1m\e[31mCould not find link destination! Scroll up for full log.\e[0m"
+  grep --color=always -n -C 5 "${link_dest_warn}" haddock_log
+  exit 1
+fi
+
+ambiguous_warn="You may be able to disambiguate the identifier by"
+if grep -q "${ambiguous_warn}" haddock_log; then
+  echo -e "\e[1m\e[31mAmbiguous identifier found! Scroll up for full log.\e[0m"
+  grep --color=always -n -C 5 "${ambiguous_warn}" haddock_log
+  exit 1
+fi
+
+# Copy documention to hadocs/
+ln -s "$(dirname "$(tail -n1 haddock_log)")" hadocs/

.ci/cabal.project-import-master

@@ -0,0 +1,2 @@
+import: cabal.project-common
+import: cabal.project-master

.ci/cabal.project.local

@@ -0,0 +1,30 @@
+-- use latest packages from hackage
+index-state: HEAD
+
+-- don't generate haddock for all our deps
+package *
+  documentation: False
+
+  -- Dynamic executables save oozles of space when archiving it on CI
+  executable-dynamic: True
+
+package clash-prelude
+  flags: -workaround-ghc-mmap-crash
+
+package clash-lib
+  flags: -workaround-ghc-mmap-crash
+
+package clash-ghc
+  flags: -workaround-ghc-mmap-crash
+
+package clash-cores
+  ghc-options: -Werror
+  tests: True
+
+  -- clash-cores unittests fail with dynamic executables because of a bug in
+  -- Cabal-the-library in combination with custom setup where the library path
+  -- is not added to the RPATH
+  executable-dynamic: False
+
+package clash-testsuite
+  flags: -workaround-ghc-mmap-crash

.ci/effective_cpus.sh

@@ -0,0 +1,26 @@
+#!/usr/bin/env bash
+# `nproc` doesn't account for limits set by cgroups/docker. This script tries
+# to determine the effective number of cpus we can use by inspecting the shares
+# it has been given.
+set -euo pipefail
+IFS=$'\n\t'
+
+if [ -f /sys/fs/cgroup/cpu/cpu.cfs_quota_us ]; then
+  # Older kernels (<= Ubuntu 20.04)
+  cfs_quota_us=$(cat /sys/fs/cgroup/cpu/cpu.cfs_quota_us)
+  cfs_period_us=$(cat /sys/fs/cgroup/cpu/cpu.cfs_period_us)
+elif [ -f /sys/fs/cgroup/cpu.max ]; then
+  # Newer kernels (>= Ubuntu 22.04)
+  cfs_quota_us=$(cat /sys/fs/cgroup/cpu.max | awk '{ print $1 }')
+  cfs_period_us=$(cat /sys/fs/cgroup/cpu.max | awk '{ print $2 }')
+else
+  echo "Could not determine number of effective CPUs" >&2
+  exit 1
+fi
+
+if [[ ${cfs_quota_us} == @(-1|max) ]]; then
+  # No limits set
+  nproc
+else
+  expr "${cfs_quota_us}" / "${cfs_period_us}"
+fi

.ci/get_build_dist.sh

@@ -0,0 +1,35 @@
+#!/bin/bash
+set -euo pipefail
+IFS=$'\n\t'
+
+GIT_ROOT=$(git rev-parse --show-toplevel)
+PLAN_JSON=${GIT_ROOT}/dist-newstyle/cache/plan.json
+
+CABAL_HOME=${HOME}/.cabal
+GHC_VERSION=$(ghc --version | grep -E -o '[0-9.]+$')
+if ghc --info | grep -q "Project Unit Id"; then
+  GHC_ABI=$(ghc --info | grep "Project Unit Id" | tail -c 7 | cut -c 1-4)
+  PKG_STORE_DIR=${CABAL_HOME}/store/ghc-${GHC_VERSION}-${GHC_ABI}
+else
+  PKG_STORE_DIR=${CABAL_HOME}/store/ghc-${GHC_VERSION}
+fi
+
+# Extract packages used from install plan
+pkgs=$(jq -r '."install-plan"[].id' "${PLAN_JSON}")
+
+# Check which exists (filters global packages)
+for pkg in ${pkgs}; do
+  if [[ -d "${PKG_STORE_DIR}/${pkg}" ]]; then
+    echo "${PKG_STORE_DIR}/${pkg}"
+    conf_file="${PKG_STORE_DIR}/package.db/${pkg}.conf"
+    if [[ -f "${conf_file}" ]]; then
+      echo "${conf_file}"
+    fi
+  fi
+done
+
+echo "${GIT_ROOT}"/dist-newstyle
+ls "${GIT_ROOT}"/.ghc.environment.*
+
+# Include symlinks to built binaries
+echo "${GIT_ROOT}"/bin

.ci/gitlab/branch.yaml

@@ -0,0 +1,117 @@
+include:
+  - '/.ci/gitlab/common.yaml'
+
+stages:
+  - pre
+  - test
+  - post
+
+tests-always:
+  stage: test
+  needs: []
+  trigger:
+    include: .ci/gitlab/test.yaml
+    strategy: depend
+  variables:
+    CLASH_BRANCH: "$CLASH_BRANCH"
+  parallel:
+    matrix:
+      - GHC_VERSION: 9.8.2
+
+      - GHC_VERSION: 9.0.2
+        WORKAROUND_GHC_MMAP_CRASH: "yes"
+
+      - GHC_VERSION: 8.10.7
+        WORKAROUND_GHC_MMAP_CRASH: "yes"
+
+tests-release:
+  stage: test
+  rules:
+    - if: $CLASH_BRANCH == "release"
+  needs: []
+  trigger:
+    include: .ci/gitlab/test.yaml
+    strategy: depend
+  variables:
+    CLASH_BRANCH: "$CLASH_BRANCH"
+  parallel:
+    matrix:
+      - GHC_VERSION: [9.6.6, 9.4.8, 9.2.8]
+
+stack-build:
+  extends: .common-local
+  image: fpco/stack-build:lts-22.33
+  needs: []
+  stage: test
+  rules:
+    - if: $CLASH_BRANCH == "release"
+  variables:
+    CACHE_FALLBACK_KEY: $CI_JOB_NAME-master-$CI_JOB_IMAGE-1-3-non_protected
+  before_script:
+    - apt-get update
+    - apt-get install -y zstd
+    - export THREADS=$(./.ci/effective_cpus.sh)
+    - export
+    - tar -xf cache.tar.zst -C / || true
+    # Print stack.yaml for debugging
+    - cat stack.yaml
+  script:
+    - stack build -j$THREADS --pedantic
+  after_script:
+    - export THREADS=$(./.ci/effective_cpus.sh)
+    - tar -cf - /root/.stack | zstd -T$THREADS -3 > cache.tar.zst
+
+nix-build:
+  image: nixos/nix:2.10.1
+  needs: []
+  stage: test
+  rules:
+    - if: $CLASH_BRANCH == "release"
+  before_script:
+    - nix-env -i gawk
+    - export
+  script:
+    - nix-build -j$(./.ci/effective_cpus.sh) --log-format raw --max-silent-time 3600
+  tags:
+    - local
+
+haddock:
+  extends: .common-local
+  needs: []
+  stage: test
+  artifacts:
+    paths:
+      - hadocs/*/*
+    expire_in: 1 month
+  script:
+    - .ci/build_docs.sh
+
+# We manually report status to GitHub to work around
+# <https://gitlab.com/gitlab-org/gitlab/-/issues/216629>.
+#
+# If we ever remove this manual GitHub reporting, we may have to rethink the
+# use of the interruptible flag: see <https://github.com/clash-lang/clash-compiler/pull/2616#discussion_r1409429185>
+.report_status:
+  # interruptible: false
+  image: curlimages/curl
+  dependencies: []
+
+set_pending:
+  extends: .report_status
+  stage: pre
+  script:
+    - .ci/report_status.sh $CLASH_BRANCH pending
+
+set_success:
+  extends: .report_status
+  stage: post
+  when: on_success
+  script:
+    - .ci/report_status.sh $CLASH_BRANCH success
+
+set_failure:
+  extends: .report_status
+  stage: post
+  when: on_failure
+  script:
+    - .ci/report_status.sh $CLASH_BRANCH failure

.ci/gitlab/common.yaml

@@ -0,0 +1,51 @@
+default:
+  # Make all tasks interruptible by default
+  interruptible: true
+  retry:
+    max: 2
+    when:
+      - runner_system_failure
+      - stuck_or_timeout_failure
+
+.common:
+  image: ghcr.io/clash-lang/clash-ci:$GHC_VERSION-$CLASH_DOCKER_TAG
+  timeout: 10 minutes
+  stage: build
+  variables:
+    CLASH_DOCKER_TAG: 20240905
+    CACHE_BUST_TOKEN: 1
+    # Note that we copy+paste the image name into CACHE_FALLBACK_KEY. If we don't,
+    # $GHC_VERSION gets inserted at verbatim, instead of resolving to some ghc version.
+    CACHE_FALLBACK_KEY: $CI_JOB_NAME-clash-$CLASH_BRANCH-main-ghcr.io/clash-lang/clash-ci:$GHC_VERSION-$CLASH_DOCKER_TAG-$CACHE_BUST_TOKEN-non_protected
+    TERM: xterm-color
+  cache:
+    key: $CI_JOB_NAME-clash-$CLASH_BRANCH-$CI_COMMIT_REF_SLUG-$CI_JOB_IMAGE-$CACHE_BUST_TOKEN
+    when: always
+    paths:
+      - cache.tar.zst
+  before_script:
+    # Check GitLab still sets -o pipefail. If they ever disable it, we would no
+    # longer directly notice errors in pipes in these YAML files.
+    - false | true && { echo "-o pipefail no longer active"; false ; }
+
+    - export THREADS=$(./.ci/effective_cpus.sh)
+    - export CABAL_JOBS=$(./.ci/effective_cpus.sh)
+    - export CABAL_DIR=$HOME/.cabal
+    - export
+    - tar -xf cache.tar.zst -C / || true
+    - .ci/setup.sh
+  after_script:
+    - export THREADS=$(./.ci/effective_cpus.sh)
+    - tar -cf - /root/.cabal | zstd -T$THREADS -3 > cache.tar.zst
+
+# We run tests on local machines if:
+#
+#   * A job may take more than 3 minutes to complete on public runners.
+#   * A job needs specific capabilities public runners cannot provide, e.g.
+#     more than 4GB memory of memory, proprietary synthesis tooling.
+#
+.common-local:
+  extends: .common
+  timeout: 2 hours
+  tags:
+    - local