Use tardis-refdata from Azure mirror (CI/CD)

azure-pipelines/simple_test_framework.yml

@@ -1,80 +1,42 @@
-# Python package
-# Create and test a Python package on multiple Python versions.
-# Add steps that analyze code, save the dist with the build record, publish to a PyPI-compatible index, and more:
-# https://docs.microsoft.com/azure/devops/pipelines/languages/python
+# For more information on how to use this pipeline please refer to:
+# http://tardis-sn.github.io/tardis/development/continuous_integration.html
 
 trigger:
   - master
 
 variables:
-  system.debug: "true"
-  ref.data.home: "$(Agent.BuildDirectory)/tardis-refdata"
-  ref.data.github.url: "https://github.com/tardis-sn/tardis-refdata.git"
-  tardis.build.dir: $(Build.Repository.LocalPath)
-  codecov.token: "a876d307-9ed5-4f5d-a6c4-e58291ac4111"
+  codecov.token: 'a876d307-9ed5-4f5d-a6c4-e58291ac4111'
+  system.debug: false
 
 jobs:
-  - job: "Test"
+  - job: test
+    displayName: 'TARDIS test'
+
     pool:
       vmImage: $[variables.vm_Image]
 
     strategy:
       matrix:
         linux:
-          vm_Image: "Ubuntu-16.04"
+          vm_Image: 'ubuntu-latest'
+
         mac:
-          vm_Image: "macOS-10.14"
+          vm_Image: 'macOS-latest'
+
       maxParallel: 2
 
     steps:
-      - bash: echo "##vso[task.setvariable variable=shellopts]errexit"
-        displayName: "Force exit on error (bash) "
-
-      - bash: echo "##vso[task.prependpath]$CONDA/bin"
-        displayName: "Add conda to PATH"
-
-      - bash: sudo chown -R $USER $CONDA
-        condition: eq( variables['Agent.OS'], 'Darwin' )
-        displayName: "Take ownership of conda installation (macOS)"
-
-      - bash: conda install mamba=0.4.0 -c conda-forge -y
-        displayName: "Install mamba"
-
-      - bash: mamba env create -f tardis_env3.yml
-        displayName: "Install TARDIS environment"
-
-      - bash: |
-          GIT_LFS_SKIP_SMUDGE=1 git clone $(ref.data.github.url) $(ref.data.home)
-          cd $(ref.data.home)
-          git fetch origin
-          git checkout origin/master
-          git lfs pull --include="atom_data/kurucz_cd23_chianti_H_He.h5" origin
-          git lfs pull --include="atom_data/chianti_He.h5" origin
-          git lfs pull --include="unit_test_data.h5" origin
-          git lfs pull --include="packet_unittest.h5" origin
-          echo MD5: `md5sum unit_test_data.h5`
-        displayName: "Fetch reference data"
-        condition: false
-
-      - bash: |
-          source activate tardis
-          pip install gdown
-          gdown https://drive.google.com/uc\?id\=1LwfZ6MXuV2j9a23uu9HTcoDDFI9gIuf1 -O tardis-refdata.tar.gz
-          tar zxf tardis-refdata.tar.gz
-          mv tardis-refdata $(ref.data.home)
-        displayName: "Fetch reference data (GDrive backup)"
-        condition: true
-      - bash: |
-          source activate tardis
-          python setup.py build_ext --inplace
-        displayName: "TARDIS build"
+      - template: templates/default.yml
+        parameters:
+          fetchRefdata: true
+          useMamba: true
 
       - bash: |
+          cd $(tardis.dir)
           source activate tardis
           pip install pytest-azurepipelines
-          pytest tardis --tardis-refdata=$(ref.data.home) --cov=tardis --cov-report=xml --cov-report=html
-        displayName: "TARDIS test"
+          pytest tardis --tardis-refdata=$(refdata.dir) --cov=tardis --cov-report=xml --cov-report=html
+        displayName: 'TARDIS test'
 
-      - bash: |
-          bash <(curl -s https://codecov.io/bash)
-        displayName: "Upload to codecov.io"
+      - bash: bash <(curl -s https://codecov.io/bash)
+        displayName: 'Upload to codecov.io'

azure-pipelines/templates/default.yml

@@ -0,0 +1,64 @@
+# For more information on how to use this template please refer to:
+# http://tardis-sn.github.io/tardis/development/continuous_integration.html
+
+parameters:
+  - name: fetchRefdata
+    type: boolean
+    default: false
+
+  - name: useMamba
+    type: boolean
+    default: false
+
+  - name: skipInstall
+    type: boolean
+    default: false
+
+steps:
+  - bash: echo "##vso[task.setvariable variable=shellopts]errexit"
+    displayName: 'Force BASH exit if a command exits with non-zero status'
+    condition: eq(variables['Agent.OS'], 'Linux')
+
+  - bash: |
+      echo "##vso[task.setvariable variable=sources.dir]$(Build.SourcesDirectory)"
+      echo "##vso[task.setvariable variable=tardis.dir]$(Build.SourcesDirectory)/tardis"
+      echo "##vso[task.setvariable variable=refdata.dir]$(Build.SourcesDirectory)/tardis-refdata"
+    displayName: 'Set custom variables'
+
+  - checkout: self
+    displayName: 'Fetch TARDIS repository (GitHub)'
+
+  - ${{ if eq(parameters.fetchRefdata, true) }}:
+    - checkout: git://TARDIS/tardis-refdata
+      lfs: true
+      displayName: 'Fetch reference data repository (Azure Repos)'
+
+  - bash: echo "##vso[task.prependpath]$CONDA/bin"
+    displayName: 'Add conda to PATH'
+
+  - bash: sudo chown -R $USER $CONDA
+    displayName: 'Take ownership of conda installation (macOS)'
+    condition: eq(variables['Agent.OS'], 'Darwin')
+
+  - ${{ if eq(parameters.useMamba, true) }}:
+    - bash: conda install mamba -c conda-forge -y
+      displayName: 'Install Mamba package manager'
+
+  - ${{ if and( eq(parameters.skipInstall, false), eq(parameters.useMamba, false)) }}:
+    - bash: |
+        cd $(tardis.dir)
+        conda env create -f tardis_env3.yml
+      displayName: 'TARDIS environment install (conda)'
+
+  - ${{ if and( eq(parameters.skipInstall, false), eq(parameters.useMamba, true)) }}:
+    - bash: |
+        cd $(tardis.dir)
+        mamba env create -f tardis_env3.yml
+      displayName: 'TARDIS environment install (mamba)'
+
+  - ${{ if eq(parameters.skipInstall, false) }}:
+    - bash: |
+        cd $(tardis.dir)
+        source activate tardis
+        python setup.py build_ext --inplace
+      displayName: 'TARDIS build & install'

docs/development/azure_links.inc

@@ -1,22 +1,13 @@
 
 .. azure stuff
-.. _azure devops: http://azure.microsoft.com/en-us/services/devops/?nav=mi
-.. _azure ssh-task: http://docs.microsoft.com/en-us/azure/devops/pipelines/tasks/utility/install-ssh-key?view=azure-devops#example-setup-using-github
-.. _azure secure files: http://docs.microsoft.com/en-us/azure/devops/pipelines/library/secure-files?view=azure-devops#how-do-i-authorize-a-secure-file-for-use-in-all-pipelines
-.. _azure variables: http://docs.microsoft.com/en-us/azure/devops/pipelines/process/variables?view=azure-devops&tabs=yaml%2Cbatch#secret-variables
-.. _azure secure files: http://docs.microsoft.com/en-us/azure/devops/pipelines/library/secure-files?view=azure-devops#how-do-i-authorize-a-secure-file-for-use-in-all-pipelines
-.. _azure task: http://github.com/microsoft/azure-pipelines-tasks
-.. _azure pipelines agent releases: https://github.com/Microsoft/azure-pipelines-agent/releases
-.. _azure self hosted agents: https://docs.microsoft.com/en-us/azure/devops/pipelines/agents/agents?view=azure-devops#install
-.. _azure tutorial: https://docs.microsoft.com/en-us/azure/devops/pipelines/create-first-pipeline?view=azure-devops&tabs=tfs-2018-2
-.. _azure condition statements: https://docs.microsoft.com/en-us/azure/devops/pipelines/process/conditions?view=azure-devops&tabs=yaml
-.. _personal access token PAT: https://docs.microsoft.com/en-us/azure/devops/organizations/accounts/use-personal-access-tokens-to-authenticate?view=azure-devops
-.. _self agent services: https://docs.microsoft.com/en-us/azure/devops/pipelines/agents/v2-linux?view=azure-devops#run-as-a-systemd-service
-.. _agent pool security roles: https://docs.microsoft.com/en-us/azure/devops/organizations/security/about-security-roles?view=azure-devops#agent-pool-security-roles-project-level
-.. _service file: https://docs.microsoft.com/en-us/azure/devops/pipelines/agents/v2-linux?view=azure-devops#service-files
-
+.. _azure DevOps: http://azure.microsoft.com/en-us/services/devops/?nav=mi
+.. _azure documentation section on triggers: https://docs.microsoft.com/en-us/azure/devops/pipelines/repos/github?view=azure-devops&tabs=yaml#ci-triggers
+.. _azure documentation section on variables: https://docs.microsoft.com/en-us/azure/devops/pipelines/process/variables?view=azure-devops&tabs=yaml%2Cbatch
+.. _azure documentation section on jobs: https://docs.microsoft.com/en-us/azure/devops/pipelines/process/phases?view=azure-devops&tabs=yaml
+.. _azure documentation section on templates: https://docs.microsoft.com/en-us/azure/devops/pipelines/process/templates?view=azure-devops
+.. _checking out multiple repositories: https://docs.microsoft.com/en-us/azure/devops/pipelines/repos/multi-repo-checkout?view=azure-devops
 .. general stuff
-.. _Continuous Integration: https://en.wikipedia.org/wiki/Continuous_integration
+.. _continuous integration: https://en.wikipedia.org/wiki/Continuous_integration
 
 .. |emdash| unicode:: U+02014