Build git-annex on Ubuntu #1556
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Build git-annex on Ubuntu | |
| on: | |
| # Trigger the workflow on pull requests | |
| pull_request: | |
| paths: | |
| - '.github/workflows/build-ubuntu.yaml' | |
| - 'patches/*.patch' | |
| schedule: | |
| - cron: '30 02 * * *' | |
| workflow_dispatch: | |
| inputs: | |
| commitish: | |
| description: The upstream commitish to build | |
| pr: | |
| description: The number of the PR to build | |
| defaults: | |
| run: | |
| shell: bash | |
| env: | |
| LANG: C | |
| DEB_SIGN_KEYID: 13A1093296154584245E0300C98FC49D36DAB17F | |
| DEB_BUILD_OPTIONS: nocheck | |
| bbuild_log: git-annex-build.log | |
| jobs: | |
| build-package: | |
| runs-on: ubuntu-latest | |
| outputs: | |
| build-version: ${{ steps.build-version.outputs.version }} | |
| steps: | |
| - name: Checkout this repository | |
| uses: actions/checkout@v4 | |
| with: | |
| fetch-depth: 0 | |
| - name: Create pending PR status | |
| if: github.event.inputs.pr != '' | |
| run: | | |
| .github/workflows/tools/set-pr-status \ | |
| "${{ github.event.inputs.pr }}" \ | |
| Ubuntu \ | |
| build-package \ | |
| pending | |
| env: | |
| GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} | |
| - name: Set up system | |
| run: | | |
| bash <(wget -q -O- http://neuro.debian.net/_files/neurodebian-travis.sh) | |
| sudo apt-get update -qq | |
| sudo apt-get install eatmydata | |
| sudo eatmydata apt-get install gnupg moreutils strace | |
| - name: Install Singularity | |
| run: | | |
| # use if decide to get most recent release of singularity | |
| # release="$(gh api --jq .tag_name repos/sylabs/singularity/releases/latest)" | |
| release="v3.10.4" | |
| codename="$(lsb_release -cs)" | |
| arch="$(dpkg --print-architecture)" | |
| cd /tmp | |
| wget -O singularity-ce.deb https://github.com/sylabs/singularity/releases/download/$release/singularity-ce_${release#v}-${codename}_$arch.deb | |
| sudo dpkg -i singularity-ce.deb | |
| sudo apt-get install -f | |
| - name: Determine git-annex ref to build | |
| run: | | |
| . .github/workflows/tools/set-build-commit | |
| echo "BUILD_COMMIT=$BUILD_COMMIT" >> "$GITHUB_ENV" | |
| env: | |
| GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} | |
| INPUT_PR: ${{ github.event.inputs.pr }} | |
| INPUT_COMMITISH: ${{ github.event.inputs.commitish }} | |
| - name: Check out source files | |
| run: | | |
| # The goal here is for $BUILD_COMMIT to be the HEAD (necessary for | |
| # git-annex's version detection to use the correct git commit) with | |
| # the .github/, clients/, and patches/ trees from master — or | |
| # whatever ref is being used as the workflow source — also available. | |
| git checkout "$BUILD_COMMIT" | |
| git checkout "$GITHUB_SHA" -- .github clients patches | |
| - name: Get build version | |
| id: build-version | |
| run: | | |
| version="$(git describe "$BUILD_COMMIT" | sed -e 's/-/+git/')" | |
| arch="$(dpkg --print-architecture)" | |
| echo "Building $version" | |
| echo "version=${version}_$arch" >> "$GITHUB_OUTPUT" | |
| - name: Apply local patches | |
| run: | | |
| .github/workflows/tools/apply-patches patches ${{ github.event_name }} | |
| env: | |
| GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} | |
| - name: Adjust changelog generation script to use original branch | |
| run: sed -i -e "s,HEAD,$BUILD_COMMIT,g" debian/create-standalone-changelog | |
| - name: Import GPG keys | |
| if: github.event_name != 'pull_request' || github.event.pull_request.head.repo.fork == false | |
| run: | | |
| # private key for signing - comes from secrets | |
| echo '${{ secrets.datalad_builder_gpgkey }}' | gpg --import | |
| # public key for verification | |
| gpg --import .github/workflows/tools/datalad-builder-key.asc | |
| - name: Get the git-annex build environment container | |
| run: chronic singularity pull --name buildenv.sif docker://datalad/buildenv-git-annex | |
| working-directory: /tmp | |
| - name: Build source packages | |
| run: | | |
| chronic singularity exec /tmp/buildenv.sif make debianstandalone-dsc | |
| mv ../git-annex_*.* . | |
| - name: Build binary package from the source package | |
| run: | | |
| sign="${{ github.event_name != 'pull_request' || github.event.pull_request.head.repo.fork == false }}" | |
| if [ "x$sign" = xtrue ] | |
| then opts= | |
| else opts='-uc -us' | |
| fi | |
| if ! singularity exec /tmp/buildenv.sif \ | |
| bash -c "set -x; export LANG=C; dpkg-source -x *.dsc git-annex-source && cd git-annex-source && dpkg-buildpackage -b $opts" >| "${bbuild_log}" 2>&1; then | |
| echo "E: build failed" | |
| echo "I: the tail of the log" | |
| tail -n 50 "${bbuild_log}" | |
| echo "I: FAIL line matches with some context" | |
| grep -B10 '^FAIL$' "${bbuild_log}" | |
| exit 1 | |
| fi | |
| - name: Check that all tests were run and passed | |
| run: grep -E '^All [[:digit:]]{3} tests passed' "${bbuild_log}" | |
| if: "!contains(env.DEB_BUILD_OPTIONS, 'nocheck')" | |
| - name: Upload packages | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: git-annex-debianstandalone-packages_${{ steps.build-version.outputs.version }} | |
| path: | | |
| git-annex[-_]*.* | |
| dist/build-version | |
| - name: Clone datalad/git-annex-ci-client-jobs | |
| if: contains(fromJSON('["schedule", "workflow_dispatch"]'), github.event_name) | |
| uses: actions/checkout@v4 | |
| with: | |
| repository: datalad/git-annex-ci-client-jobs | |
| fetch-depth: 1 | |
| path: client-jobs | |
| ssh-key: ${{ secrets.CLIENT_JOBS_SSH_KEY }} | |
| - name: Push installer to datalad/git-annex-ci-client-jobs | |
| if: contains(fromJSON('["schedule", "workflow_dispatch"]'), github.event_name) | |
| run: | | |
| set -ex -o pipefail | |
| git checkout --orphan build | |
| git rm -rf . | |
| cp -i ../git-annex*.deb . | |
| git add git-annex*.deb | |
| buildno="${{ github.run_number }}" | |
| git commit \ | |
| --gpg-sign=13A1093296154584245E0300C98FC49D36DAB17F \ | |
| -m "Installer artifact from build $buildno" | |
| git ls-remote --heads origin "build-*" | cut -f2 | cut -d/ -f3- > builds.txt | |
| git ls-remote --heads origin "result-*" | cut -f2 | cut -d/ -f3- > results.txt | |
| yq -r "keys | .[]" ../clients/clients.yaml | while read clientid | |
| do build_branch="build-$clientid-$buildno" | |
| result_branch="result-$clientid-$buildno" | |
| if grep -Fqx "$result_branch" results.txt | |
| then echo "[INFO] Result branch $result_branch already exists; not creating build branch" | |
| elif grep -Fqx "$build_branch" builds.txt | |
| then echo "[INFO] Build branch $build_branch already exists; leaving alone" | |
| else git branch "$build_branch" build | |
| git push origin "$build_branch" | |
| fi | |
| done | |
| working-directory: client-jobs | |
| - name: Create new release | |
| if: github.event.inputs.commitish != '' | |
| run: | | |
| printf '[DEBUG] INPUT_COMMITISH=%b\n' "$INPUT_COMMITISH" | |
| if git rev-parse refs/tags/"$INPUT_COMMITISH" &> /dev/null | |
| then | |
| echo "[INFO] Building a tag; uploading assets to release ..." | |
| echo '[DEBUG] BEGIN gh release list' | |
| gh release list | tee releases.txt | cat -v | |
| echo '[DEBUG] END gh release list' | |
| if grep -q "^$INPUT_COMMITISH\b" releases.txt | |
| then | |
| echo "[INFO] Release already exists; uploading assets" | |
| gh release upload "$INPUT_COMMITISH" git-annex[-_]*.* | |
| else | |
| echo "[INFO] Creating release" | |
| gh release create \ | |
| --notes "This is an unofficial release build provided by the DataLad team." \ | |
| "$INPUT_COMMITISH" git-annex[-_]*.* | |
| fi | |
| else | |
| echo "[INFO] Not building a tag; no release to make" | |
| fi | |
| env: | |
| GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} | |
| INPUT_COMMITISH: ${{ github.event.inputs.commitish }} | |
| - name: Set final PR status | |
| if: always() && github.event.inputs.pr != '' | |
| run: | | |
| .github/workflows/tools/set-pr-status \ | |
| "${{ github.event.inputs.pr }}" \ | |
| Ubuntu \ | |
| build-package \ | |
| "${{ job.status }}" | |
| env: | |
| GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} | |
| - name: Send e-mail on failed run | |
| if: failure() && contains(fromJSON('["schedule", "workflow_dispatch"]'), github.event_name) | |
| uses: dawidd6/action-send-mail@v3 | |
| with: | |
| server_address: ${{ secrets.NOTIFY_SMTP_HOST }} | |
| server_port: ${{ secrets.NOTIFY_SMTP_PORT }} | |
| username: ${{ secrets.NOTIFY_SMTP_USERNAME }} | |
| password: ${{ secrets.NOTIFY_SMTP_PASSWORD }} | |
| from: GitHub Actions Notifications | |
| to: ${{ secrets.NOTIFY_RECIPIENT }} | |
| subject: '[${{ github.repository }}] Build on Ubuntu failed!' | |
| body: | | |
| A build (via ${{ github.event_name }}) of git-annex for Ubuntu failed! | |
| See <https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}> for more information. | |
| test-annex: | |
| runs-on: ${{ matrix.os }} | |
| needs: build-package | |
| strategy: | |
| matrix: | |
| flavor: ["normal", "crippled-tmp", "crippled-home", "nfs-home", "custom-config1"] | |
| os: [ubuntu-latest] | |
| fail-fast: false | |
| steps: | |
| - name: Checkout this repository | |
| uses: actions/checkout@v4 | |
| - name: Create pending PR status | |
| if: github.event.inputs.pr != '' | |
| run: | | |
| .github/workflows/tools/set-pr-status \ | |
| "${{ github.event.inputs.pr }}" \ | |
| Ubuntu \ | |
| "test-annex (${{ matrix.flavor }}, ${{ matrix.os }})" \ | |
| pending | |
| env: | |
| GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} | |
| - name: Download git-annex package | |
| uses: actions/download-artifact@v4 | |
| with: | |
| name: git-annex-debianstandalone-packages_${{ needs.build-package.outputs.build-version }} | |
| - name: Install git-annex package | |
| run: | | |
| sudo dpkg -i git-annex*.deb | |
| - name: Print git-annex version | |
| run: git annex version | |
| - name: Run tests | |
| run: | | |
| if echo "${{ matrix.flavor }}" | grep -q "crippled" ; then | |
| # source so that CRIPPLEDFS_PATH will be set: | |
| . .github/workflows/tools/setup_crippledfs crippledfs 500 | |
| fi | |
| if echo "${{ matrix.flavor }}" | grep -q "nfs" ; then | |
| mkdir /tmp/nfsmount_ /tmp/nfsmount | |
| echo "/tmp/nfsmount_ localhost(rw)" | sudo bash -c 'cat - > /etc/exports' | |
| sudo apt-get install -y nfs-kernel-server | |
| sudo exportfs -a | |
| sudo mount -t nfs localhost:/tmp/nfsmount_ /tmp/nfsmount | |
| fi | |
| case "${{ matrix.flavor }}" in | |
| # For git-annex it causes only few temporary directories to be on the crippled FS, | |
| # while the main ones produced by git annex test reside in CWD, for which we use | |
| # $HOME | |
| crippled-tmp) | |
| export TMPDIR="$CRIPPLEDFS_PATH" | |
| ;; | |
| nfs-tmp) | |
| export TMPDIR=/tmp/nfsmount | |
| ;; | |
| # Also used as CWD where running the tests, so in effect tests annex operating | |
| # on those file systems | |
| crippled-home) | |
| export HOME="$CRIPPLEDFS_PATH" | |
| ;; | |
| nfs-home) | |
| export HOME=/tmp/nfsmount | |
| ;; | |
| normal|custom-config1) | |
| ;; | |
| *) | |
| echo "Unknown flavor ${{ matrix.flavor }}" | |
| exit 1 | |
| esac | |
| # Do it after we possibly setup HOME | |
| git config --global user.email "[email protected]" | |
| git config --global user.name "GitHub Almighty" | |
| test_opts=( ) | |
| case "${{ matrix.flavor }}" in | |
| # For git-annex it causes only few temporary directories to be on the crippled FS, | |
| # while the main ones produced by git annex test reside in CWD, for which we use | |
| # $HOME | |
| custom-config1) | |
| test_opts=( --test-git-config annex.stalldetection=1KB/120s ) | |
| ;; | |
| *) | |
| ;; | |
| esac | |
| cd $HOME | |
| export | grep -e crippledfs || : | |
| timeout 3600 git annex test "${test_opts[@]:-}" | |
| - name: Set final PR status | |
| if: always() && github.event.inputs.pr != '' | |
| run: | | |
| .github/workflows/tools/set-pr-status \ | |
| "${{ github.event.inputs.pr }}" \ | |
| Ubuntu \ | |
| "test-annex (${{ matrix.flavor }}, ${{ matrix.os }})" \ | |
| "${{ job.status }}" | |
| env: | |
| GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} | |
| - name: Send e-mail on failed run | |
| if: failure() && contains(fromJSON('["schedule", "workflow_dispatch"]'), github.event_name) | |
| uses: dawidd6/action-send-mail@v3 | |
| with: | |
| server_address: ${{ secrets.NOTIFY_SMTP_HOST }} | |
| server_port: ${{ secrets.NOTIFY_SMTP_PORT }} | |
| username: ${{ secrets.NOTIFY_SMTP_USERNAME }} | |
| password: ${{ secrets.NOTIFY_SMTP_PASSWORD }} | |
| from: GitHub Actions Notifications | |
| to: ${{ secrets.NOTIFY_RECIPIENT }} | |
| subject: '[${{ github.repository }}] Tests of Ubuntu build failed!' | |
| body: | | |
| The tests for a build (via ${{ github.event_name }}) of git-annex for Ubuntu (flavor: ${{ matrix.flavor }}, OS: ${{ matrix.os }}) failed! | |
| See <https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}> for more information. | |
| test-annex-more: | |
| runs-on: ubuntu-latest | |
| needs: build-package | |
| steps: | |
| - name: Checkout this repository | |
| uses: actions/checkout@v4 | |
| - name: Create pending PR status | |
| if: github.event.inputs.pr != '' | |
| run: | | |
| .github/workflows/tools/set-pr-status \ | |
| "${{ github.event.inputs.pr }}" \ | |
| Ubuntu \ | |
| test-annex-more \ | |
| pending | |
| env: | |
| GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} | |
| - name: Download git-annex package | |
| uses: actions/download-artifact@v4 | |
| with: | |
| name: git-annex-debianstandalone-packages_${{ needs.build-package.outputs.build-version }} | |
| - name: Install git-annex package | |
| run: | | |
| sudo dpkg -i git-annex*.deb | |
| - name: Seek of dynlibs | |
| run: | | |
| mkdir /tmp/testrepo; cd /tmp/testrepo; git init | |
| function nfailed() { | |
| strace -f git-annex "$1" 2>&1 | awk "/$2.*ENOENT/{print}" | tee /dev/fd/2 | wc -l | |
| } | |
| # We should get some reasonable number (not 40) of directories look up for dynamic libraries | |
| liblookups= | |
| PS4='> '; set -x | |
| test $(nfailed version "libpcre.*so") -lt 7 | |
| test $(nfailed init "libpcre.*so") -lt 260 | |
| - name: Set final PR status | |
| if: always() && github.event.inputs.pr != '' | |
| run: | | |
| .github/workflows/tools/set-pr-status \ | |
| "${{ github.event.inputs.pr }}" \ | |
| Ubuntu \ | |
| test-annex-more \ | |
| "${{ job.status }}" | |
| env: | |
| GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} | |
| test-datalad: | |
| runs-on: ubuntu-latest | |
| needs: build-package | |
| strategy: | |
| matrix: | |
| version: [master, maint, release] | |
| fail-fast: false | |
| steps: | |
| - name: Checkout this repository | |
| uses: actions/checkout@v4 | |
| - name: Create pending PR status | |
| if: github.event.inputs.pr != '' | |
| run: | | |
| .github/workflows/tools/set-pr-status \ | |
| "${{ github.event.inputs.pr }}" \ | |
| Ubuntu \ | |
| "test-datalad (${{ matrix.version }})" \ | |
| pending | |
| env: | |
| GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} | |
| - name: Download git-annex package | |
| uses: actions/download-artifact@v4 | |
| with: | |
| name: git-annex-debianstandalone-packages_${{ needs.build-package.outputs.build-version }} | |
| - name: Install git-annex package | |
| run: | | |
| sudo dpkg -i git-annex*.deb | |
| - name: Set up SSH target | |
| shell: bash | |
| run: | | |
| curl -fSsL \ | |
| https://raw.githubusercontent.com/datalad/datalad/master/tools/ci/prep-travis-forssh.sh \ | |
| | bash | |
| echo DATALAD_TESTS_SSH=1 >> "$GITHUB_ENV" | |
| - name: Set up environment | |
| run: | | |
| git config --global user.email "[email protected]" | |
| git config --global user.name "GitHub Almighty" | |
| - name: Set up Python 3.8 | |
| uses: actions/setup-python@v5 | |
| with: | |
| python-version: 3.8 | |
| - name: Install ${{ matrix.version }} Datalad | |
| run: | | |
| if [ "${{ matrix.version }}" = "release" ]; then | |
| commitish="$(gh api --jq .tag_name repos/datalad/datalad/releases/latest)" | |
| else | |
| commitish="${{ matrix.version }}" | |
| fi | |
| python -m pip install --upgrade pip | |
| pip install git+https://github.com/datalad/datalad@${commitish} | |
| env: | |
| # Authorize so that we don't run up against low API rate limits | |
| GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} | |
| - name: Install nose et al (just in case!) | |
| run: pip install nose pytest vcrpy mock | |
| - name: WTF!? | |
| run: datalad wtf | |
| # needed for ssh certs under ubuntu and tox.ini everywhere | |
| - name: Checkout datalad | |
| uses: actions/checkout@v4 | |
| with: | |
| repository: datalad/datalad | |
| path: datalad | |
| fetch-depth: 1 | |
| - name: Set up test SSH certs | |
| run: bash tools/ci/deploy_datalad-rootca | |
| working-directory: datalad | |
| - name: Run datalad tests | |
| run: | | |
| mkdir -p __testhome__ | |
| cd __testhome__ | |
| python -m pytest -c ../datalad/tox.ini -s -v --pyargs datalad | |
| - name: Set final PR status | |
| if: always() && github.event.inputs.pr != '' | |
| run: | | |
| .github/workflows/tools/set-pr-status \ | |
| "${{ github.event.inputs.pr }}" \ | |
| Ubuntu \ | |
| "test-datalad (${{ matrix.version }})" \ | |
| "${{ job.status }}" | |
| env: | |
| GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} | |
| # vim:set et sts=2: |