Merge pull request #27 from CybercentreCanada/template_pipeline

Use templated pipelines

pipelines/.cruft.json

@@ -0,0 +1,20 @@
+{
+  "template": "[email protected]:CybercentreCanada/assemblyline-service-pipeline-generator.git",
+  "commit": "3f3f922ec39c7b953a1c22d29828cb3231d7d47a",
+  "checkout": null,
+  "context": {
+    "cookiecutter": {
+      "__directory_name": "pipelines",
+      "classification": "UNCLASSIFIED",
+      "is_public": true,
+      "test_in_container": false,
+      "test_versions": "default",
+      "__setupscript": "setup.bash",
+      "_extensions": [
+        "local_extensions.extract_versions"
+      ],
+      "_template": "[email protected]:CybercentreCanada/assemblyline-service-pipeline-generator.git"
+    }
+  },
+  "directory": null
+}

pipelines/.gitignore

@@ -0,0 +1 @@
+setup.bash

pipelines/azure-build.yaml

@@ -2,23 +2,6 @@ name: build
 
 variables:
   - group: unittest-samples
-  - name: self_location
-    value: "self_location"
-  - name: full_self_location
-    value: "$(Agent.BuildDirectory)/$(self_location)"
-  - name: samples_location
-    value: "samples_location"
-  - name: full_samples_location
-    value: "$(Agent.BuildDirectory)/$(samples_location)"
-
-resources:
-  repositories:
-  - repository: unittest-samples
-    type: github
-    name: $(unittest_samples_repository)
-    ref: main
-    endpoint: github-repo-sa
-    trigger: none
 
 trigger:
   tags:
@@ -28,54 +11,25 @@ pr: none
 pool:
   vmImage: "ubuntu-20.04"
 
-stages:
-  - stage: deploy
-    jobs:
-      - job: deploy
-        displayName: Deploy containers to dockerhub
-        variables:
-          - group: deployment-information
-        steps:
-          - task: Docker@2
-            displayName: Login to docker hub
-            inputs:
-              command: login
-              containerRegistry: dockerhub
-          - task: Docker@2
-            displayName: Login to chimera
-            inputs:
-              command: login
-              containerRegistry: CHIMERA-U-ACR
-          - checkout: self
-            fetchDepth: 1
-            path: $(self_location)
-          - checkout: unittest-samples
-            fetchDepth: 1
-            path: $(samples_location)
-          - script: |
-              export TAG=${BUILD_SOURCEBRANCH#"refs/tags/v"}
-              if [[ "$TAG" == *stable* ]]; then export BUILD_TYPE=stable; else export BUILD_TYPE=latest; fi
-              docker build --build-arg version=$TAG --build-arg branch=$BUILD_TYPE -t cccs/${BUILD_REPOSITORY_NAME##*/}:$TAG -t cccs/${BUILD_REPOSITORY_NAME##*/}:$BUILD_TYPE -f ./Dockerfile .
-            workingDirectory: $(full_self_location)
-            displayName: Build containers
-          - script: |
-              [ ! -d "$(pwd)/tests" ] && echo "No tests found" && exit
-              export TAG=${BUILD_SOURCEBRANCH#"refs/tags/v"}
-              if [[ "$TAG" == *stable* ]]; then export BUILD_TYPE=stable; else export BUILD_TYPE=latest; fi
-              [ -f "$(pwd)/tests/requirements.txt" ] && docker run -e FULL_SELF_LOCATION=/opt/al_service -e FULL_SAMPLES_LOCATION=/opt/samples -v /usr/share/ca-certificates/mozilla:/usr/share/ca-certificates/mozilla -v $(pwd)/tests/:/opt/al_service/tests/ -v ${FULL_SAMPLES_LOCATION}:/opt/samples cccs/${BUILD_REPOSITORY_NAME##*/}:$BUILD_TYPE bash -c 'pip install -U -r tests/requirements.txt; pytest -p no:cacheprovider -vv' && exit
-              docker run -e FULL_SELF_LOCATION=/opt/al_service -e FULL_SAMPLES_LOCATION=/opt/samples -v /usr/share/ca-certificates/mozilla:/usr/share/ca-certificates/mozilla -v $(pwd)/tests/:/opt/al_service/tests/ -v ${FULL_SAMPLES_LOCATION}:/opt/samples cccs/${BUILD_REPOSITORY_NAME##*/}:$BUILD_TYPE bash -c 'pytest -p no:cacheprovider -vv'
-            workingDirectory: $(full_self_location)
-            displayName: Test containers
-          - script: |
-              export TAG=${BUILD_SOURCEBRANCH#"refs/tags/v"}
-              if [[ "$TAG" == *stable* ]]; then export BUILD_TYPE=stable; else export BUILD_TYPE=latest; fi
-              export SERIES="`expr $TAG : '\([0-9]\+\.[0-9]\+\.\)'`${BUILD_TYPE}"
+resources:
+  repositories:
+    - repository: PipelineTemplates
+      type: github
+      name: CybercentreCanada/assemblyline-pipeline-templates
+      ref: refs/heads/main
+      endpoint: github-repo-sa
+      trigger: none
+    - repository: unittest-samples
+      type: github
+      name: $(unittest_samples_repository)
+      ref: main
+      endpoint: github-repo-sa
+      trigger: none
 
-              for IMAGE in "cccs/" "uchimera.azurecr.io/cccs/"
-              do
-                docker tag cccs/${BUILD_REPOSITORY_NAME##*/}:$BUILD_TYPE ${IMAGE}${BUILD_REPOSITORY_NAME##*/}:$TAG
-                docker tag cccs/${BUILD_REPOSITORY_NAME##*/}:$BUILD_TYPE ${IMAGE}${BUILD_REPOSITORY_NAME##*/}:$BUILD_TYPE
-                docker tag cccs/${BUILD_REPOSITORY_NAME##*/}:$BUILD_TYPE ${IMAGE}${BUILD_REPOSITORY_NAME##*/}:$SERIES
-                docker push ${IMAGE}${BUILD_REPOSITORY_NAME##*/} --all-tags
-              done
-            displayName: Deploy to container repositories
+extends:
+  template: stages/deploy-service.yaml@PipelineTemplates
+  parameters:
+    is_public: "true"
+    samples_repo: unittest-samples
+    labels:
+      classification: "UNCLASSIFIED"

pipelines/azure-tests.yaml

@@ -2,71 +2,31 @@ name: tests
 
 variables:
   - group: unittest-samples
-  - name: self_location
-    value: "self_location"
-  - name: full_self_location
-    value: "$(Agent.BuildDirectory)/$(self_location)"
-  - name: samples_location
-    value: "samples_location"
-  - name: full_samples_location
-    value: "$(Agent.BuildDirectory)/$(samples_location)"
-
-resources:
-  repositories:
-  - repository: unittest-samples
-    type: github
-    name: $(unittest_samples_repository)
-    ref: main
-    endpoint: github-repo-sa
-    trigger: none
 
 trigger: ["*"]
 pr: ["*"]
 
 pool:
   vmImage: "ubuntu-20.04"
 
-jobs:
-  - job: run_test
-    strategy:
-      matrix:
-        Python3_9:
-          python.version: "3.9"
-        #Python3_10:
-        #  python.version: "3.10"
-        #Python3_11:
-        #  python.version: "3.11"
-
-    timeoutInMinutes: 10
+resources:
+  repositories:
+    - repository: PipelineTemplates
+      type: github
+      name: CybercentreCanada/assemblyline-pipeline-templates
+      ref: refs/heads/main
+      endpoint: github-repo-sa
+      trigger: none
+    - repository: unittest-samples
+      type: github
+      name: $(unittest_samples_repository)
+      ref: main
+      endpoint: github-repo-sa
+      trigger: none
 
-    steps:
-      - task: UsePythonVersion@0
-        displayName: Set python version
-        inputs:
-          versionSpec: "$(python.version)"
-      - checkout: self
-        fetchDepth: 1
-        path: $(self_location)
-      - checkout: unittest-samples
-        fetchDepth: 1
-        path: $(samples_location)
-      - script: |
-          [ ! -d "$(pwd)/tests" ] && echo "No tests found" && exit
-          sudo apt-get update
-          sudo apt-get install -y libfuzzy-dev libfuzzy2
-          if [[ -f "$(pwd)/pkglist.txt" ]]; then
-            grep -vE '^#' "$(pwd)/pkglist.txt" | xargs sudo apt install -y
-          fi
-          sudo rm -rf /var/lib/apt/lists/*
-          sudo env "PATH=$PATH" python -m pip install -U --no-cache-dir assemblyline assemblyline_v4_service
-          [ -f $(pwd)/requirements.txt ] && sudo env "PATH=$PATH" python -m pip install -U --no-cache-dir -r $(pwd)/requirements.txt
-          [ -f $(pwd)/tests/requirements.txt ] && sudo env "PATH=$PATH" python -m pip install -U --no-cache-dir -r $(pwd)/tests/requirements.txt
-          sudo rm -rf /tmp/* /var/lib/apt/lists/* ~/.cache/pip
-        workingDirectory: $(full_self_location)
-        displayName: Setup environment
-      - script: |
-          [ ! -d "$(pwd)/tests" ] && echo "No tests found" && exit
-          export REPO_NAME=${BUILD_REPOSITORY_NAME##*/}
-          python -m pytest -p no:cacheprovider --durations=10 -rsx -vv --disable-warnings
-        workingDirectory: $(full_self_location)
-        displayName: Test
+extends:
+  template: stages/test-service.yaml@PipelineTemplates
+  parameters:
+    samples_repo: unittest-samples
+    test_container: "false"
+