From 0909fff6cd89c845abc26ebb36100925d8c171f8 Mon Sep 17 00:00:00 2001 From: Emma Casolin Date: Wed, 6 Jul 2022 13:51:39 +1000 Subject: [PATCH] ci: chocolatey caching Optimised Windows CI/CD setup by internalising and caching chocolatey packages Fixes #397 --- .gitlab-ci.yml | 86 ++++----- package-lock.json | 1 - package.json | 1 - ...enerate.sh => build-platforms-generate.sh} | 182 +++++++++--------- ...est-generate.sh => check-test-generate.sh} | 5 +- scripts/choco-install.ps1 | 39 ++++ 6 files changed, 179 insertions(+), 135 deletions(-) rename scripts/{build:platforms-generate.sh => build-platforms-generate.sh} (50%) rename scripts/{check:test-generate.sh => check-test-generate.sh} (93%) create mode 100644 scripts/choco-install.ps1 diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 0cb026187..81d0cf74b 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -34,6 +34,8 @@ cache: - ./tmp/ts-node-cache/ # Homebrew cache is only used by the macos runner - ./tmp/Homebrew + # Chocolatey cache is only used by the windows runner + - ./tmp/chocolatey/ # `jest` cache is configured in jest.config.js - ./tmp/jest/ @@ -63,19 +65,19 @@ check:lint: - if: $CI_COMMIT_BRANCH && $CI_COMMIT_BRANCH != 'master' && $CI_COMMIT_TITLE !~ /^[0-9]+\.[0-9]+\.[0-9]+(?:-.*[0-9]+)?$/ when: manual -# check:nix-dry: -# stage: check -# needs: [] -# script: -# - nix-build -v -v --dry-run ./release.nix -# rules: -# # Runs on feature and staging commits and ignores version commits -# - if: $CI_COMMIT_BRANCH =~ /^(?:feature.*|staging)$/ && $CI_COMMIT_TITLE !~ /^[0-9]+\.[0-9]+\.[0-9]+(?:-.*[0-9]+)?$/ -# # Runs on tag pipeline where the tag is a prerelease or release version -# - if: $CI_COMMIT_TAG =~ /^v[0-9]+\.[0-9]+\.[0-9]+(?:-.*[0-9]+)?$/ -# # Manually run on commits other than master and ignore version commits -# - if: $CI_COMMIT_BRANCH && $CI_COMMIT_BRANCH != 'master' && $CI_COMMIT_TITLE !~ /^[0-9]+\.[0-9]+\.[0-9]+(?:-.*[0-9]+)?$/ -# when: manual +check:nix-dry: + stage: check + needs: [] + script: + - nix-build -v -v --dry-run ./release.nix + rules: + # Runs on feature and staging commits and ignores version commits + - if: $CI_COMMIT_BRANCH =~ /^(?:feature.*|staging)$/ && $CI_COMMIT_TITLE !~ /^[0-9]+\.[0-9]+\.[0-9]+(?:-.*[0-9]+)?$/ + # Runs on tag pipeline where the tag is a prerelease or release version + - if: $CI_COMMIT_TAG =~ /^v[0-9]+\.[0-9]+\.[0-9]+(?:-.*[0-9]+)?$/ + # Manually run on commits other than master and ignore version commits + - if: $CI_COMMIT_BRANCH && $CI_COMMIT_BRANCH != 'master' && $CI_COMMIT_TITLE !~ /^[0-9]+\.[0-9]+\.[0-9]+(?:-.*[0-9]+)?$/ + when: manual check:test-generate: stage: check @@ -83,12 +85,12 @@ check:test-generate: script: - > nix-shell --run ' - ./scripts/check:test-generate.sh > ./tmp/check:test.yml + ./scripts/check-test-generate.sh > ./tmp/check-test.yml ' artifacts: when: always paths: - - ./tmp/check:test.yml + - ./tmp/check-test.yml rules: # Runs on staging commits and ignores version commits - if: $CI_COMMIT_BRANCH =~ /^feature.*$/ && $CI_COMMIT_TITLE !~ /^[0-9]+\.[0-9]+\.[0-9]+(?:-.*[0-9]+)?$/ @@ -96,23 +98,23 @@ check:test-generate: - if: $CI_COMMIT_BRANCH && $CI_COMMIT_BRANCH !~ /^(?:master|staging)$/ && $CI_COMMIT_TITLE !~ /^[0-9]+\.[0-9]+\.[0-9]+(?:-.*[0-9]+)?$/ when: manual -# check:test: -# stage: check -# needs: -# - check:test-generate -# inherit: -# variables: false -# trigger: -# include: -# - artifact: tmp/check:test.yml -# job: check:test-generate -# strategy: depend -# rules: -# # Runs on staging commits and ignores version commits -# - if: $CI_COMMIT_BRANCH =~ /^feature.*$/ && $CI_COMMIT_TITLE !~ /^[0-9]+\.[0-9]+\.[0-9]+(?:-.*[0-9]+)?$/ -# # Manually run on commits other than master and staging and ignore version commits -# - if: $CI_COMMIT_BRANCH && $CI_COMMIT_BRANCH !~ /^(?:master|staging)$/ && $CI_COMMIT_TITLE !~ /^[0-9]+\.[0-9]+\.[0-9]+(?:-.*[0-9]+)?$/ -# when: manual +check:test: + stage: check + needs: + - check:test-generate + inherit: + variables: false + trigger: + include: + - artifact: tmp/check-test.yml + job: check:test-generate + strategy: depend + rules: + # Runs on staging commits and ignores version commits + - if: $CI_COMMIT_BRANCH =~ /^feature.*$/ && $CI_COMMIT_TITLE !~ /^[0-9]+\.[0-9]+\.[0-9]+(?:-.*[0-9]+)?$/ + # Manually run on commits other than master and staging and ignore version commits + - if: $CI_COMMIT_BRANCH && $CI_COMMIT_BRANCH !~ /^(?:master|staging)$/ && $CI_COMMIT_TITLE !~ /^[0-9]+\.[0-9]+\.[0-9]+(?:-.*[0-9]+)?$/ + when: manual build:merge: stage: build @@ -148,18 +150,17 @@ build:platforms-generate: script: - > nix-shell --run ' - ./scripts/build:platforms-generate.sh > ./tmp/build:platforms.yml + ./scripts/build-platforms-generate.sh > ./tmp/build-platforms.yml ' artifacts: when: always paths: - - ./tmp/build:platforms.yml + - ./tmp/build-platforms.yml rules: # Runs on staging commits and ignores version commits - - if: $CI_COMMIT_BRANCH =~ /^feature.*$/ && $CI_COMMIT_TITLE !~ /^[0-9]+\.[0-9]+\.[0-9]+(?:-.*[0-9]+)?$/ - # Manually run on commits other than master and staging and ignore version commits - - if: $CI_COMMIT_BRANCH && $CI_COMMIT_BRANCH !~ /^(?:master|staging)$/ && $CI_COMMIT_TITLE !~ /^[0-9]+\.[0-9]+\.[0-9]+(?:-.*[0-9]+)?$/ - when: manual + - if: $CI_COMMIT_BRANCH == 'staging' && $CI_COMMIT_TITLE !~ /^[0-9]+\.[0-9]+\.[0-9]+(?:-.*[0-9]+)?$/ + # Runs on tag pipeline where the tag is a prerelease or release version + - if: $CI_COMMIT_TAG =~ /^v[0-9]+\.[0-9]+\.[0-9]+(?:-.*[0-9]+)?$/ build:platforms: stage: build @@ -169,15 +170,14 @@ build:platforms: variables: false trigger: include: - - artifact: tmp/build:platforms.yml + - artifact: tmp/build-platforms.yml job: build:platforms-generate strategy: depend rules: # Runs on staging commits and ignores version commits - - if: $CI_COMMIT_BRANCH =~ /^feature.*$/ && $CI_COMMIT_TITLE !~ /^[0-9]+\.[0-9]+\.[0-9]+(?:-.*[0-9]+)?$/ - # Manually run on commits other than master and staging and ignore version commits - - if: $CI_COMMIT_BRANCH && $CI_COMMIT_BRANCH !~ /^(?:master|staging)$/ && $CI_COMMIT_TITLE !~ /^[0-9]+\.[0-9]+\.[0-9]+(?:-.*[0-9]+)?$/ - when: manual + - if: $CI_COMMIT_BRANCH == 'staging' && $CI_COMMIT_TITLE !~ /^[0-9]+\.[0-9]+\.[0-9]+(?:-.*[0-9]+)?$/ + # Runs on tag pipeline where the tag is a prerelease or release version + - if: $CI_COMMIT_TAG =~ /^v[0-9]+\.[0-9]+\.[0-9]+(?:-.*[0-9]+)?$/ build:dist: stage: build diff --git a/package-lock.json b/package-lock.json index 0d042805b..fb1cb2217 100644 --- a/package-lock.json +++ b/package-lock.json @@ -31,7 +31,6 @@ "google-protobuf": "^3.14.0", "ip-num": "^1.3.3-0", "isomorphic-git": "^1.8.1", - "jest-junit": "^13.2.0", "jose": "^4.3.6", "lexicographic-integer": "^1.1.0", "multiformats": "^9.4.8", diff --git a/package.json b/package.json index 32fc20269..ac8498b95 100644 --- a/package.json +++ b/package.json @@ -98,7 +98,6 @@ "google-protobuf": "^3.14.0", "ip-num": "^1.3.3-0", "isomorphic-git": "^1.8.1", - "jest-junit": "^13.2.0", "jose": "^4.3.6", "lexicographic-integer": "^1.1.0", "multiformats": "^9.4.8", diff --git a/scripts/build:platforms-generate.sh b/scripts/build-platforms-generate.sh similarity index 50% rename from scripts/build:platforms-generate.sh rename to scripts/build-platforms-generate.sh index ab8ecc3af..e35bb23c7 100755 --- a/scripts/build:platforms-generate.sh +++ b/scripts/build-platforms-generate.sh @@ -36,6 +36,8 @@ cache: - ./tmp/ts-node-cache/ # Homebrew cache is only used by the macos runner - ./tmp/Homebrew + # Chocolatey cache is only used by the windows runner + - ./tmp/chocolatey/ # `jest` cache is configured in jest.config.js - ./tmp/jest/ @@ -47,61 +49,61 @@ EOF printf "\n" -# # Each test directory has its own job -# for test_dir in tests/**/*/; do -# test_files=("$test_dir"*.test.ts) -# if [ ${#test_files[@]} -eq 0 ]; then -# continue -# fi -# # Remove trailing slash -# test_dir="${test_dir%\/}" -# # Remove `tests/` prefix -# test_dir="${test_dir#*/}" -# cat << EOF -# build:linux $test_dir: -# stage: build -# needs: [] -# script: -# - > -# nix-shell --run ' -# npm test -- --ci --coverage ${test_files[@]}; -# ' -# artifacts: -# when: always -# reports: -# junit: -# - ./tmp/junit/junit.xml -# coverage_report: -# coverage_format: cobertura -# path: ./tmp/coverage/cobertura-coverage.xml -# coverage: '/All files[^|]*\|[^|]*\s+([\d\.]+)/' -# EOF -# printf "\n" -# done - -# # All top-level test files are accumulated into 1 job -# test_files=(tests/*.test.ts) -# cat << EOF -# build:linux index: -# stage: build -# needs: [] -# script: -# - > -# nix-shell --run ' -# npm test -- --ci --coverage ${test_files[@]}; -# ' -# artifacts: -# when: always -# reports: -# junit: -# - ./tmp/junit/junit.xml -# coverage_report: -# coverage_format: cobertura -# path: ./tmp/coverage/cobertura-coverage.xml -# coverage: '/All files[^|]*\|[^|]*\s+([\d\.]+)/' -# EOF - -# printf "\n" +# Each test directory has its own job +for test_dir in tests/**/*/; do + test_files=("$test_dir"*.test.ts) + if [ ${#test_files[@]} -eq 0 ]; then + continue + fi + # Remove trailing slash + test_dir="${test_dir%\/}" + # Remove `tests/` prefix + test_dir="${test_dir#*/}" + cat << EOF +build:linux $test_dir: + stage: build + needs: [] + script: + - > + nix-shell --run ' + npm test -- --ci --coverage --runInBand ${test_files[@]}; + ' + artifacts: + when: always + reports: + junit: + - ./tmp/junit/junit.xml + coverage_report: + coverage_format: cobertura + path: ./tmp/coverage/cobertura-coverage.xml + coverage: '/All files[^|]*\|[^|]*\s+([\d\.]+)/' +EOF + printf "\n" +done + +# All top-level test files are accumulated into 1 job +test_files=(tests/*.test.ts) +cat << EOF +build:linux index: + stage: build + needs: [] + script: + - > + nix-shell --run ' + npm test -- --ci --coverage --runInBand ${test_files[@]}; + ' + artifacts: + when: always + reports: + junit: + - ./tmp/junit/junit.xml + coverage_report: + coverage_format: cobertura + path: ./tmp/coverage/cobertura-coverage.xml + coverage: '/All files[^|]*\|[^|]*\s+([\d\.]+)/' +EOF + +printf "\n" # Using shards to optimise tests # In the future we can incorporate test durations rather than using @@ -110,38 +112,38 @@ printf "\n" # Number of parallel shards to split the test suite into CI_PARALLEL=2 -# cat << "EOF" -# build:windows: -# stage: build -# needs: [] -# EOF -# cat << EOF -# parallel: $CI_PARALLEL -# EOF -# cat << "EOF" -# tags: -# - windows -# before_script: -# - mkdir -Force "$CI_PROJECT_DIR/tmp" -# - choco install nodejs --version=16.14.2 -y -# - refreshenv -# script: -# - npm config set msvs_version 2019 -# - npm install --ignore-scripts -# - $env:Path = "$(npm bin);" + $env:Path -# - npm test -- --ci --coverage --shard=$CI_NODE_INDEX/$CI_NODE_TOTAL -# artifacts: -# when: always -# reports: -# junit: -# - ./tmp/junit/junit.xml -# coverage_report: -# coverage_format: cobertura -# path: ./tmp/coverage/cobertura-coverage.xml -# coverage: '/All files[^|]*\|[^|]*\s+([\d\.]+)/' -# EOF - -# printf "\n" +cat << "EOF" +build:windows: + stage: build + needs: [] +EOF +cat << EOF + parallel: $CI_PARALLEL +EOF +cat << "EOF" + tags: + - windows + before_script: + - mkdir -Force "$CI_PROJECT_DIR/tmp" + - .\scripts\choco-install.ps1 + - refreshenv + script: + - npm config set msvs_version 2019 + - npm install --ignore-scripts + - $env:Path = "$(npm bin);" + $env:Path + - npm test -- --ci --coverage --shard=$CI_NODE_INDEX/$CI_NODE_TOTAL --maxWorkers=50% + artifacts: + when: always + reports: + junit: + - ./tmp/junit/junit.xml + coverage_report: + coverage_format: cobertura + path: ./tmp/coverage/cobertura-coverage.xml + coverage: '/All files[^|]*\|[^|]*\s+([\d\.]+)/' +EOF + +printf "\n" cat << "EOF" build:macos: @@ -169,12 +171,16 @@ cat << "EOF" script: - npm install --ignore-scripts - export PATH="$(npm bin):$PATH" - - npm test -- --ci --shard=$CI_NODE_INDEX/$CI_NODE_TOTAL --maxWorkers=50% + - npm test -- --ci --coverage --shard=$CI_NODE_INDEX/$CI_NODE_TOTAL --maxWorkers=50% artifacts: when: always reports: junit: - ./tmp/junit/junit.xml + coverage_report: + coverage_format: cobertura + path: ./tmp/coverage/cobertura-coverage.xml + coverage: '/All files[^|]*\|[^|]*\s+([\d\.]+)/' EOF printf "\n" diff --git a/scripts/check:test-generate.sh b/scripts/check-test-generate.sh similarity index 93% rename from scripts/check:test-generate.sh rename to scripts/check-test-generate.sh index ef78f2ccf..a58b799ee 100755 --- a/scripts/check:test-generate.sh +++ b/scripts/check-test-generate.sh @@ -28,6 +28,7 @@ variables: # Cached directories shared between jobs & pipelines per-branch per-runner cache: key: $CI_COMMIT_REF_SLUG + when: 'always' paths: - ./tmp/npm/ - ./tmp/ts-node-cache/ @@ -59,7 +60,7 @@ check:test $test_dir: script: - > nix-shell --run ' - npm test -- --ci --coverage ${test_files[@]}; + npm test -- --ci --coverage --runInBand ${test_files[@]}; ' artifacts: when: always @@ -83,7 +84,7 @@ check:test index: script: - > nix-shell --run ' - npm test -- --ci --coverage ${test_files[@]}; + npm test -- --ci --coverage --runInBand ${test_files[@]}; ' artifacts: when: always diff --git a/scripts/choco-install.ps1 b/scripts/choco-install.ps1 new file mode 100644 index 000000000..5fcb58e9d --- /dev/null +++ b/scripts/choco-install.ps1 @@ -0,0 +1,39 @@ +$nodejs = "nodejs.install" +$python = "python3" + +function Save-ChocoPackage { + param ( + $PackageName + ) + Rename-Item -Path "$env:ChocolateyInstall\lib\$PackageName\$PackageName.nupkg" -NewName "$PackageName.nupkg.zip" -ErrorAction:SilentlyContinue + Expand-Archive -LiteralPath "$env:ChocolateyInstall\lib\$PackageName\$PackageName.nupkg.zip" -DestinationPath "$env:ChocolateyInstall\lib\$PackageName" -Force + Remove-Item "$env:ChocolateyInstall\lib\$PackageName\_rels" -Recurse + Remove-Item "$env:ChocolateyInstall\lib\$PackageName\package" -Recurse + Remove-Item "$env:ChocolateyInstall\lib\$PackageName\[Content_Types].xml" + New-Item -Path "${PSScriptRoot}\..\tmp\chocolatey\$PackageName" -ItemType "directory" -ErrorAction:SilentlyContinue + choco pack "$env:ChocolateyInstall\lib\$PackageName\$PackageName.nuspec" --outdir "${PSScriptRoot}\..\tmp\chocolatey\$PackageName" +} + +# Check for existence of required environment variables +if ( $null -eq $env:ChocolateyInstall ) { + [Console]::Error.WriteLine('Missing $env:ChocolateyInstall environment variable') + exit 1 +} + +# Add the cached packages with source priority 1 (Chocolatey community is 0) +New-Item -Path "${PSScriptRoot}\..\tmp\chocolatey" -ItemType "directory" -ErrorAction:SilentlyContinue +choco source add --name="cache" --source="${PSScriptRoot}\..\tmp\chocolatey" --priority=1 + +# Install nodejs v16.14.2 (will use cache if exists) +choco install "$nodejs" --version="16.14.2" --checksum="hello" -y +# Internalise nodejs to cache if doesn't exist +if ( -not (Test-Path -Path "${PSScriptRoot}\..\tmp\chocolatey\$nodejs\$nodejs.16.14.2.nupkg" -PathType Leaf) ) { + Save-ChocoPackage -PackageName $nodejs +} + +# Install python v3.9.12 (will use cache if exists) +choco install $python --version="3.9.12" --checksum="hello" -y +# Internalise python to cache if doesn't exist +if ( -not (Test-Path -Path "${PSScriptRoot}\..\tmp\chocolatey\$python\$python.3.9.12.nupkg" -PathType Leaf) ) { + Save-ChocoPackage -PackageName $python +}