From f3790f2eb6efffd0fff7c45d29ae220e6d71fed7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ezequiel=20P=C3=A1ssaro?= Date: Tue, 27 Oct 2020 10:15:22 -0300 Subject: [PATCH 1/6] explicitly set path for self repo --- azure-pipelines/templates/default.yml | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/azure-pipelines/templates/default.yml b/azure-pipelines/templates/default.yml index 8ce061f352f..4d34631ad03 100644 --- a/azure-pipelines/templates/default.yml +++ b/azure-pipelines/templates/default.yml @@ -20,7 +20,6 @@ steps: condition: eq(variables['Agent.OS'], 'Linux') - bash: | - echo "##vso[task.setvariable variable=sources.dir]$(Build.SourcesDirectory)" echo "##vso[task.setvariable variable=tardis.dir]$(Build.SourcesDirectory)/tardis" echo "##vso[task.setvariable variable=refdata.dir]$(Build.SourcesDirectory)/tardis-refdata" displayName: 'Set source directories' @@ -34,14 +33,15 @@ steps: - bash: | echo "##vso[task.setvariable variable=package.manager]mamba" displayName: 'Set package manager' - + - checkout: self - displayName: 'Fetch TARDIS repository from GitHub' + path: tardis + displayName: 'Fetch main repository' - ${{ if eq(parameters.fetchRefdata, true) }}: - checkout: git://TARDIS/tardis-refdata lfs: true - displayName: 'Fetch reference data repository from Azure Repos' + displayName: 'Fetch reference data repository' - bash: echo "##vso[task.prependpath]$CONDA/bin" displayName: 'Add conda to PATH' @@ -52,17 +52,17 @@ steps: - ${{ if eq(parameters.useMamba, true) }}: - bash: conda install mamba -c conda-forge -y - displayName: 'Install Mamba package manager' + displayName: 'Install package manager' - ${{ if eq(parameters.skipInstall, false) }}: - bash: | cd $(tardis.dir) $(package.manager) env create -f tardis_env3.yml - displayName: 'TARDIS environment install' + displayName: 'Install conda environment' - ${{ if eq(parameters.skipInstall, false) }}: - bash: | cd $(tardis.dir) source activate tardis python setup.py build_ext --inplace - displayName: 'TARDIS build & install' + displayName: 'Build C extensions' From 986b89f7701c8f0f4be1f3e26b60bb889da43a5d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ezequiel=20P=C3=A1ssaro?= Date: Tue, 27 Oct 2020 10:29:49 -0300 Subject: [PATCH 2/6] fix path --- azure-pipelines/templates/default.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/azure-pipelines/templates/default.yml b/azure-pipelines/templates/default.yml index 4d34631ad03..523b74551d5 100644 --- a/azure-pipelines/templates/default.yml +++ b/azure-pipelines/templates/default.yml @@ -35,7 +35,7 @@ steps: displayName: 'Set package manager' - checkout: self - path: tardis + path: $(tardis.dir) displayName: 'Fetch main repository' - ${{ if eq(parameters.fetchRefdata, true) }}: From 5c20bce0bc0477c9b30f2d4bc8c11ebd147ef2ea Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ezequiel=20P=C3=A1ssaro?= Date: Tue, 27 Oct 2020 10:40:38 -0300 Subject: [PATCH 3/6] fix path --- azure-pipelines/templates/default.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/azure-pipelines/templates/default.yml b/azure-pipelines/templates/default.yml index 523b74551d5..32e2f0ec9f7 100644 --- a/azure-pipelines/templates/default.yml +++ b/azure-pipelines/templates/default.yml @@ -35,7 +35,7 @@ steps: displayName: 'Set package manager' - checkout: self - path: $(tardis.dir) + path: s/tardis displayName: 'Fetch main repository' - ${{ if eq(parameters.fetchRefdata, true) }}: From 97c6f5d7098fdf7a180935713a49823c0c06c61d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ezequiel=20P=C3=A1ssaro?= Date: Tue, 27 Oct 2020 11:05:07 -0300 Subject: [PATCH 4/6] test single repo checkout --- azure-pipelines/simple_test_framework.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/azure-pipelines/simple_test_framework.yml b/azure-pipelines/simple_test_framework.yml index 158d68780e1..b4f7bd86901 100644 --- a/azure-pipelines/simple_test_framework.yml +++ b/azure-pipelines/simple_test_framework.yml @@ -28,14 +28,14 @@ jobs: steps: - template: templates/default.yml parameters: - fetchRefdata: true + fetchRefdata: false useMamba: true - bash: | cd $(tardis.dir) source activate tardis pip install pytest-azurepipelines - pytest tardis --tardis-refdata=$(refdata.dir) --cov=tardis --cov-report=xml --cov-report=html + pytest tardis --cov=tardis --cov-report=xml --cov-report=html displayName: 'TARDIS test' - bash: bash <(curl -s https://codecov.io/bash) From b88588ee6af43d271c67993680112f7bbfb142d1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ezequiel=20P=C3=A1ssaro?= Date: Tue, 27 Oct 2020 11:11:10 -0300 Subject: [PATCH 5/6] (undo) test single repo checkout --- azure-pipelines/simple_test_framework.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/azure-pipelines/simple_test_framework.yml b/azure-pipelines/simple_test_framework.yml index b4f7bd86901..158d68780e1 100644 --- a/azure-pipelines/simple_test_framework.yml +++ b/azure-pipelines/simple_test_framework.yml @@ -28,14 +28,14 @@ jobs: steps: - template: templates/default.yml parameters: - fetchRefdata: false + fetchRefdata: true useMamba: true - bash: | cd $(tardis.dir) source activate tardis pip install pytest-azurepipelines - pytest tardis --cov=tardis --cov-report=xml --cov-report=html + pytest tardis --tardis-refdata=$(refdata.dir) --cov=tardis --cov-report=xml --cov-report=html displayName: 'TARDIS test' - bash: bash <(curl -s https://codecov.io/bash) From 7cb93bc984b5ef4fe8811623b3d5407caf491703 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ezequiel=20P=C3=A1ssaro?= Date: Tue, 27 Oct 2020 11:39:34 -0300 Subject: [PATCH 6/6] update docs --- docs/development/azure_links.inc | 2 +- docs/development/continuous_integration.rst | 17 +++++++++-------- 2 files changed, 10 insertions(+), 9 deletions(-) diff --git a/docs/development/azure_links.inc b/docs/development/azure_links.inc index fedd449d400..ed74e3d53ca 100644 --- a/docs/development/azure_links.inc +++ b/docs/development/azure_links.inc @@ -5,7 +5,7 @@ .. _azure documentation section on variables: https://docs.microsoft.com/en-us/azure/devops/pipelines/process/variables?view=azure-devops&tabs=yaml%2Cbatch .. _azure documentation section on jobs: https://docs.microsoft.com/en-us/azure/devops/pipelines/process/phases?view=azure-devops&tabs=yaml .. _azure documentation section on templates: https://docs.microsoft.com/en-us/azure/devops/pipelines/process/templates?view=azure-devops -.. _checking out multiple repositories: https://docs.microsoft.com/en-us/azure/devops/pipelines/repos/multi-repo-checkout?view=azure-devops +.. _checking out multiple repositories: https://github.com/microsoft/azure-pipelines-yaml/blob/master/design/multi-checkout.md#behavior-changes-in-multi-checkout-mode .. general stuff .. _continuous integration: https://en.wikipedia.org/wiki/Continuous_integration diff --git a/docs/development/continuous_integration.rst b/docs/development/continuous_integration.rst index e9a38f5b833..09d6e62c3f4 100644 --- a/docs/development/continuous_integration.rst +++ b/docs/development/continuous_integration.rst @@ -160,14 +160,16 @@ The most important (and confusing) predefined variables are the ones related to paths in Azure: * All folders for a given pipeline are created under ``Agent.BuildDirectory`` - variable, alias ``Pipeline.Workspace``. + variable, alias ``Pipeline.Workspace``. This includes subdirectories like + ``/s`` for sources or ``/a`` for artifacts. -* Cloned repositories are under the ``Build.Repository.LocalPath`` variable - alias ``Build.SourcesDirectory``. In TARDIS we implemented another alias - for this variable, named ``sources.dir``. +* Path to source code varies depending on how many repositories we fetch. + For example, source code is located under the ``Build.Repository.LocalPath`` + variable (alias ``Build.SourcesDirectory``) when fetching a single repository, + but after fetching a second repository code is moved automatically to + ``Build.Repository.LocalPath/repository-name``. -Our advice is to stick with ``Build.SourcesDirectory`` as much as possible -to avoid problems when `checking out multiple repositories`_. +See the Azure documentation to learn more about `checking out multiple repositories`_. Jobs @@ -245,7 +247,6 @@ to start a new pipeline use:: **List of predefined custom variables:** -- ``sources.dir`` is equivalent to ``$(Build.SourcesDirectory)``. - ``tardis.dir`` is equivalent to ``$(Build.SourcesDirectory)/tardis``. - ``refdata.dir`` is equivalent to ``$(Build.SourcesDirectory)/tardis-refdata``. @@ -262,7 +263,7 @@ Documentation pipeline ---------------------- Builds and deploys the TARDIS documentation website. Currently, we are -using a github action to complete this pipeline. The action can be found at ``.github/workflows/documentation-build.yml`` +using GitHub Actions for this purpose. Zenodo JSON pipeline