diff --git a/.github/workflows/perlbrew.sha256 b/.github/workflows/perlbrew.sha256 new file mode 100644 index 00000000..d9992912 --- /dev/null +++ b/.github/workflows/perlbrew.sha256 @@ -0,0 +1 @@ +c3996e4fae37a0ae01839cdd73752fb7b17e81bac2a8b39712463a7d518c4945 perlbrew.sh diff --git a/.github/workflows/run-tests.yml b/.github/workflows/run-tests.yml new file mode 100644 index 00000000..2962c4b8 --- /dev/null +++ b/.github/workflows/run-tests.yml @@ -0,0 +1,156 @@ +name: "Unit tests" + +on: [push, pull_request] + +jobs: + build: + runs-on: ubuntu-latest + + defaults: + run: + shell: bash -l -e -o pipefail {0} + + env: + PERL_CACHE: ~/perl5 # Perlbrew and CPAN modules installed here, cached + NPG_LIB: ~/perl5npg # NPG modules installed here, not cached + WSI_CONDA_CHANNEL: "https://dnap.cog.sanger.ac.uk/npg/conda/devel/generic" + CONDA_TEST_ENV: test-environment + NPM_CONFIG_PREFIX: "$HOME/.npm-global" + WTSI_NPG_GITHUB_URL: https://github.com/wtsi-npg + WTSI_NPG_BUILD_BRANCH: ${GITHUB_HEAD_REF} + + strategy: + fail-fast: false + matrix: + perl: ["5.26.3", "5.34.1"] + node: ['14.8.0'] + + steps: + - uses: actions/checkout@v3 + + - name: "Install OS dependencies" + run: | + sudo apt-get update + # https://github.com/actions/runner-images/issues/2139 + sudo apt-get remove -y nginx libgd3 + sudo apt-get install -y libgd-dev uuid-dev libgd-text-perl + + - name: "Initialize Miniconda" + run: | + echo 'source $CONDA/etc/profile.d/conda.sh' >> "$HOME/.bash_profile" + + - name: "Cache Conda" + id: minicondaCache + uses: actions/cache@v3 + with: + path: | + ~/conda/pkgs + ~/conda/envs + key: ${{ runner.os }}-build-miniconda + + - name: "Install Conda packages" + env: + CONDA_TEST_ENV: test-environment + run: | + conda config --prepend pkgs_dirs ~/conda/pkgs + conda config --prepend envs_dirs ~/conda/envs + + conda config --set auto_update_conda False + conda config --prepend channels "$WSI_CONDA_CHANNEL" + conda config --append channels conda-forge + conda info + + conda create -y -n "$CONDA_TEST_ENV" + conda install -y -n "$CONDA_TEST_ENV" nodejs=="${{ matrix.node }}" + conda install -y -n "$CONDA_TEST_ENV" npg_qc_utils + conda install -y -n "$CONDA_TEST_ENV" baton + + - name: "Cache Perl" + id: cache-perl + uses: actions/cache@v3 + with: + path: ${{ env.PERL_CACHE }} + key: ${{ runner.os }}-${{ matrix.perl }}-perl + + - name: "Install Perlbrew" + if: steps.cache-perl.outputs.cache-hit != 'true' + run: | + curl -sSL https://install.perlbrew.pl -o perlbrew.sh + sha256sum -c .github/workflows/perlbrew.sha256 + export PERLBREW_ROOT=${{ env.PERL_CACHE }} + sh perlbrew.sh + + source ${{ env.PERL_CACHE }}/etc/bashrc + perlbrew available + perlbrew install --notest perl-${{ matrix.perl }} + perlbrew use perl-${{ matrix.perl }} + perlbrew install-cpanm + + - name: "Initialize Perlbrew" + run: | + echo "source ${{ env.PERL_CACHE }}/etc/bashrc" >> "$HOME/.bash_profile" + + - name: "NPM" + run: | + mkdir -p "$NPM_CONFIG_PREFIX" + ${GITHUB_WORKSPACE}/scripts/before_install.sh + + - name: "Install Perl dependencies" + run: | + cpanm --local-lib=${{ env.PERL_CACHE }} local::lib + eval $(perl -I ${{ env.PERL_CACHE }}/lib/perl5/ -Mlocal::lib="$NPG_LIB") + eval $(perl -I ${{ env.PERL_CACHE }}/lib/perl5/ -Mlocal::lib) + + cpanm --quiet --notest Alien::Tidyp + cpanm --quiet --notest LWP::Protocol::https + cpanm --quiet --notest https://github.com/chapmanb/vcftools-cpan/archive/v0.953.tar.gz + + ./scripts/install_wsi_dependencies.sh "$NPG_LIB" \ + perl-dnap-utilities \ + ml_warehouse \ + npg_tracking \ + npg_seq_common \ + perl-irods-wrap + + cpanm --installdeps --notest . + + pushd npg_qc_viewer + export PERL5LIB="$PERL5LIB:../lib" + cpanm --notest --installdeps . + + - name: "Log install failure" + if: ${{ failure() }} + run: | + find ~/.cpanm/work -cmin -1 -name '*.log' -exec tail -n20 {} \; + + - name: "Archive CPAN logs on failure" + if: ${{ failure() }} + uses: actions/upload-artifact@v2 + with: + name: cpan_log + path: ~/.cpanm/work/*/build.log + retention-days: 5 + + - name: "Run tests" + run: | + conda activate "$CONDA_TEST_ENV" + conda info --envs + + eval $(perl -I ${{ env.PERL_CACHE }}/lib/perl5/ -Mlocal::lib) + eval $(perl -I ${{ env.PERL_CACHE }}/lib/perl5/ -Mlocal::lib="$NPG_LIB") + + export PATH=$HOME/.nvm/versions/node/v${{ matrix.node }}/bin:$PATH + export TEST_AUTHOR=1 + export WTSI_NPG_iRODS_Test_Resource=testResc + export WTSI_NPG_iRODS_Test_IRODS_ENVIRONMENT_FILE=$HOME/.irods/irods_environment.json + + perl Build.PL + ./Build test --verbose + ./Build install + + pushd npg_qc_viewer + export PERL5LIB="$PERL5LIB:../lib" + perl Build.PL --installjsdeps + ./Build + ./Build test --verbose + $(npm bin)/grunt -v diff --git a/.github/workflows/testing_and_building_repos.yml b/.github/workflows/testing_and_building_repos.yml deleted file mode 100644 index c9d8f5a4..00000000 --- a/.github/workflows/testing_and_building_repos.yml +++ /dev/null @@ -1,106 +0,0 @@ -name: testing_and_building_repo -on: [push, pull_request] -jobs: - build: - strategy: - fail-fast: false - matrix: - perl: ['5.26'] - node: ['14.8.0'] - - runs-on: ubuntu-latest - - defaults: - run: - shell: bash -l -e -o pipefail {0} - - env: - WSI_CONDA_CHANNEL: "https://dnap.cog.sanger.ac.uk/npg/conda/devel/generic" - NPM_CONFIG_PREFIX: "$HOME/.npm-global" - - name: Perl ${{ matrix.perl }} and Node ${{ matrix.node }} - steps: - - uses: actions/checkout@v3 - - # Caching cpanm external modules - - name: Cache cpanm external modules - id: cpanmCache - uses: actions/cache@v2 - with: - path: ~/perl5ext - key: ${{ runner.os }}-build-cpanm-external - - - name: Initialize Miniconda - run: | - echo 'source $CONDA/etc/profile.d/conda.sh' >> "$HOME/.bash_profile" - - # Caching conda - - name: Cache conda - id: minicondaCache - uses: actions/cache@v3 - with: - path: | - ~/conda/pkgs - ~/conda/envs - key: ${{ runner.os }}-build-miniconda - - - name: install libgd-dev and uuid-dev - run: | - sudo apt-get update - # https://github.com/actions/runner-images/issues/2139 - sudo apt-get remove -y nginx libgd3 - sudo apt-get install -qq uuid-dev # For Perl UUID module - sudo apt-get install -qq libgd-dev libgd-text-perl # For npg_tracking - - - name: install conda client, baton,samtools, set up conda environment - run: | - conda config --prepend pkgs_dirs ~/conda/pkgs - conda config --prepend envs_dirs ~/conda/envs - - conda config --set auto_update_conda False - conda config --prepend channels "$WSI_CONDA_CHANNEL" - conda config --append channels conda-forge - conda info - - conda create -y -n "$CONDA_TEST_ENV" - conda install -y -n "$CONDA_TEST_ENV" nodejs=="${{ matrix.node }}" - conda install -y -n "$CONDA_TEST_ENV" npg_qc_utils - conda install -y -n "$CONDA_TEST_ENV" baton - env: - CONDA_TEST_ENV: test-environment - - - name: install cpanm - run: | - wget -qO - https://cpanmin.us | /usr/bin/perl - --sudo App::cpanminus - - - name: Run install scripts - run: | - mkdir -p "$NPM_CONFIG_PREFIX" - ${GITHUB_WORKSPACE}/scripts/before_install.sh - - cpanm --local-lib=~/perl5 local::lib && eval $(perl -I ~/perl5/lib/perl5/ -Mlocal::lib) - ${GITHUB_WORKSPACE}/scripts/install.sh $WTSI_NPG_BUILD_BRANCH $WTSI_NPG_GITHUB_URL - env: - NPM_VERSION: "${{ matrix.npm }}" - WTSI_NPG_BUILD_BRANCH: ${GITHUB_HEAD_REF} - WTSI_NPG_GITHUB_URL: https://github.com/wtsi-npg - - - name: run script - run: | - conda activate "$CONDA_TEST_ENV" - conda info --envs - cpanm --local-lib=~/perl5 local::lib && eval $(perl -I ~/perl5/lib/perl5/ -Mlocal::lib) - ${GITHUB_WORKSPACE}/scripts/script.sh "${{ matrix.node }}" $WTSI_NPG_BUILD_BRANCH - env: - CONDA_TEST_ENV: test-environment - NODE_VERSION: ${{ matrix.node }} - WTSI_NPG_BUILD_BRANCH: ${GITHUB_HEAD_REF} - - # Archive logs if failure - - name: Archive CPAN logs - if: ${{ failure() }} - uses: actions/upload-artifact@v2 - with: - name: cpan_log - path: /home/runner/.cpanm/work/*/build.log - retention-days: 5 diff --git a/scripts/install.sh b/scripts/install.sh deleted file mode 100755 index 5662593a..00000000 --- a/scripts/install.sh +++ /dev/null @@ -1,56 +0,0 @@ -#!/bin/bash - -set -e -u -x - -WTSI_NPG_BUILD_BRANCH=$1 - -WTSI_NPG_GITHUB_URL=$2 - -# CPAN as in npg_npg_deploy -eval $(perl -I ~/perl5ext/lib/perl5/ -Mlocal::lib=~/perl5ext) -cpanm --notest --reinstall App::cpanminus -cpanm --quiet --notest Alien::Tidyp -cpanm --quiet --notest LWP::Protocol::https -cpanm --quiet --notest https://github.com/chapmanb/vcftools-cpan/archive/v0.953.tar.gz - -# adding in perl5lib location for npg_qc locations -export PERL5LIB=${WTSI_NPG_BUILD_BRANCH}/lib/npg_qc/:$PERL5LIB - -# WTSI NPG Perl repo dependencies -repos="" -for repo in perl-dnap-utilities ml_warehouse npg_tracking npg_seq_common perl-irods-wrap; do - cd /tmp - # Always clone master when using depth 1 to get current tag - git clone --branch master --depth 1 ${WTSI_NPG_GITHUB_URL}/${repo}.git ${repo}.git - cd /tmp/${repo}.git - # Shift off master to appropriate branch (if possible) - git ls-remote --heads --exit-code origin ${WTSI_NPG_BUILD_BRANCH} && git pull origin ${WTSI_NPG_BUILD_BRANCH} && echo "Switched to branch ${WTSI_NPG_BUILD_BRANCH}" - repos=$repos" /tmp/${repo}.git" -done - -# Finally, bring any common dependencies up to the latest version and -# install -for repo in $repos -do - export PERL5LIB=$repo/blib/lib:$PERL5LIB:$repo/lib -done - -for repo in $repos -do - cd $repo - cpanm --quiet --notest --installdeps . - perl Build.PL - ./Build -done - -eval $(perl -I ~/perl5ext/lib/perl5/ -Mlocal::lib=~/perl5npg) - -for repo in $repos -do - cd $repo - cpanm --quiet --notest --installdeps . - ./Build install - perl Build.PL - ./Build -done -cd diff --git a/scripts/install_wsi_dependencies.sh b/scripts/install_wsi_dependencies.sh new file mode 100755 index 00000000..9d1e506f --- /dev/null +++ b/scripts/install_wsi_dependencies.sh @@ -0,0 +1,54 @@ +#!/bin/bash + +set -e -u -x + +WSI_NPG_GITHUB_URL=${WSI_NPG_GITHUB_URL:=https://github.com/wtsi-npg} +WSI_NPG_BUILD_BRANCH=${WSI_NPG_BUILD_BRANCH:=devel} + +# The first argument is the install base for NPG modules, enabling them to be +# installed independently of CPAN dependencies. E.g. for cases where we want +# different caching behaviour. +NPG_ROOT="$1" +shift + +repos="" +for repo in "$@" ; do + cd /tmp + + # Clone deeper than depth 1 to get the tag even if something has been already + # committed over the tag + git clone --branch master --depth 3 "$WSI_NPG_GITHUB_URL/${repo}.git" "${repo}.git" + cd "/tmp/${repo}.git" + + # Shift off master to appropriate branch (if possible) + git ls-remote --heads --exit-code origin "$WSI_NPG_BUILD_BRANCH" && \ + git pull origin "$WSI_NPG_BUILD_BRANCH" && \ + echo "Switched to branch $WSI_NPG_BUILD_BRANCH" + repos="$repos /tmp/${repo}.git" +done + +# Install CPAN dependencies. The src libs are on PERL5LIB because of +# circular dependencies. The blibs are on PERL5LIB because the package +# version, which cpanm requires, is inserted at build time. They must +# be before the libs for cpanm to pick them up in preference. +for repo in $repos +do + export PERL5LIB="$PERL5LIB:$repo/blib/lib:$repo/lib" +done + +for repo in $repos +do + cd "$repo" + cpanm --quiet --notest --installdeps . + perl Build.PL + ./Build +done + +# Finally, bring any common dependencies up to the latest version and +# install +for repo in $repos +do + cd "$repo" + cpanm --quiet --notest --installdeps . + ./Build install --install-base "$NPG_ROOT" +done diff --git a/scripts/script.sh b/scripts/script.sh deleted file mode 100755 index 410bd3e6..00000000 --- a/scripts/script.sh +++ /dev/null @@ -1,32 +0,0 @@ -#!/bin/bash - -set -e -u -x - -NODE_VERSION=$1 - -WTSI_NPG_BUILD_BRANCH=$2 - -export PATH=$HOME/.nvm/versions/node/v${NODE_VERSION}/bin:$PATH -export TEST_AUTHOR=1 -export WTSI_NPG_iRODS_Test_Resource=testResc -export WTSI_NPG_iRODS_Test_IRODS_ENVIRONMENT_FILE=$HOME/.irods/irods_environment.json -eval $(perl -I ~/perl5ext/lib/perl5/ -Mlocal::lib=~/perl5npg) -eval $(perl -I ~/perl5ext/lib/perl5/ -Mlocal::lib=~/perl5ext) - -cpanm --notest --installdeps . || find $HOME/.cpanm/work -cmin -1 -name '*.log' -exec tail -n20 {} \; -perl Build.PL -./Build -./Build test --verbose - -eval $(perl -I ~/perl5ext/lib/perl5/ -Mlocal::lib=~/perl5npg) -./Build install - -eval $(perl -I ~/perl5ext/lib/perl5/ -Mlocal::lib=~/perl5ext) -pushd npg_qc_viewer -cpanm --notest --installdeps . || find $HOME/.cpanm/work -cmin -1 -name '*.log' -exec tail -n20 {} \; - -perl Build.PL --installjsdeps -./Build -./Build test --verbose -$(npm bin)/grunt -v -popd