From 2710014d6d18bf90a51b48d546b0a261eb89d42e Mon Sep 17 00:00:00 2001 From: Nikola Rados Date: Wed, 13 Nov 2019 15:37:00 -0800 Subject: [PATCH 1/7] Update pipeline stages --- Jenkinsfile | 33 +++++++++++++++++++++++++-------- 1 file changed, 25 insertions(+), 8 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index 12ccb567..f7e17def 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -28,17 +28,34 @@ node { checkout scm } - stage('Build Image') { - String image_name = 'climate-explorer-backend' - String branch_name = BRANCH_NAME.toLowerCase() + def image + String name = BASE_REGISTRY + 'climate-explorer-backend' + + // tag branch + if (BRANCH_NAME == 'master') { + // TODO: detect tags and releases for master + } else { + name = name + ':' + BRANCH_NAME + "_${BUILD_ID}" + } + + stage('Build and Publish Image') { + withDockerServer([uri: PCIC_DOCKER]) { + image = docker.build(name) - // Update image name if we are not on the master branch - if (branch_name != 'master') { - image_name = image_name + '/' + branch_name + docker.withRegistry('', 'PCIC_DOCKERHUB_CREDS') { + image.push() + } } + } - withDockerServer([uri: PCIC_DOCKER]) { - def image = docker.build(image_name) + stage('Security Scan') { + writeFile file: 'anchore_images', text: name + anchore name: 'anchore_images', engineRetries: '700' + } + + stage('Clean Up Local Image') { + withDockerServer([uri: PCIC_DOCKER]){ + sh "docker rmi ${name}" } } } From bf9bf45246a89a6148353691aa43e94db9b599e4 Mon Sep 17 00:00:00 2001 From: Nikola Rados Date: Fri, 15 Nov 2019 11:50:03 -0800 Subject: [PATCH 2/7] Remove build_tag on tag --- Jenkinsfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Jenkinsfile b/Jenkinsfile index f7e17def..5f47951a 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -35,7 +35,7 @@ node { if (BRANCH_NAME == 'master') { // TODO: detect tags and releases for master } else { - name = name + ':' + BRANCH_NAME + "_${BUILD_ID}" + name = name + ':' + BRANCH_NAME } stage('Build and Publish Image') { From b359162610ad9a66a040cce36b8786d73eee2946 Mon Sep 17 00:00:00 2001 From: Nikola Rados Date: Fri, 15 Nov 2019 13:56:26 -0800 Subject: [PATCH 3/7] Test out updated python-geospatial image --- docker/Dockerfile.prod | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/Dockerfile.prod b/docker/Dockerfile.prod index 54d7afd9..23115ea3 100644 --- a/docker/Dockerfile.prod +++ b/docker/Dockerfile.prod @@ -1,4 +1,4 @@ -FROM pcic/geospatial-python +FROM pcic/geospatial-python:add-jenkins-support ADD . /app WORKDIR /app From 862c9dc2f854cf58faad75636208b15f031f0167 Mon Sep 17 00:00:00 2001 From: Nikola Rados Date: Fri, 15 Nov 2019 14:04:28 -0800 Subject: [PATCH 4/7] Revert test changes --- docker/Dockerfile.prod | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/Dockerfile.prod b/docker/Dockerfile.prod index 23115ea3..54d7afd9 100644 --- a/docker/Dockerfile.prod +++ b/docker/Dockerfile.prod @@ -1,4 +1,4 @@ -FROM pcic/geospatial-python:add-jenkins-support +FROM pcic/geospatial-python ADD . /app WORKDIR /app From 70df452e107207a58bc7ecf3fe9821a55eef55e5 Mon Sep 17 00:00:00 2001 From: Nikola Rados Date: Fri, 6 Dec 2019 12:38:22 -0800 Subject: [PATCH 5/7] Add new features to jenkinsfile --- Jenkinsfile | 144 +++++++++++++++++++++++++++++++++++++++++++--------- 1 file changed, 119 insertions(+), 25 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index 5f47951a..d30d7bc4 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -1,8 +1,7 @@ -node { - stage('Code Collection') { - checkout scm - } - +/** + * Does the necessary Python installations then runs the test suite. + */ +def run_tests() { withDockerServer([uri: PCIC_DOCKER]) { // Use image with gdal already installed def gdalenv = docker.image('pcic/geospatial-python') @@ -19,43 +18,138 @@ node { } } } +} + + +/** + * Build a docker image given the name + * + * @param image_name the name of the image + * @return image the built docker image + */ +def build_image(image_name) { + def image + withDockerServer([uri: PCIC_DOCKER]) { + image = docker.build(image_name) + } + + return image +} + +/** + * Get the original branch name. + * + * In the case where a branch has been filed as a PR the `BRANCH_NAME` + * environment varible updates from `some-branch-name` to `PR-[pull request #]`. + * To keep image tagging consistent on Docker Hub we want to use the original + * name. + * + * @return name the name of the branch + */ +def get_branch_name() { + String name + if (BRANCH_NAME.contains('PR')) { + name = CHANGE_BRANCH + } else { + name = BRANCH_NAME + } + + return name +} + + +/** + * If the master branch has been tagged we also add the `latest` tag. Otherwise + * we just use the branch name as the tag. + * + * @return tags a list of the tags for the image + */ +def get_tags() { + String tag = sh (script: 'git tag --contains', returnStdout: true).trim() + + def tags = [] + if(BRANCH_NAME == 'master' && !tag.isEmpty()) { + // It is possible for a commit to have multiple git tags. We want to + // ensure we add all of them in. + tags.addAll(tag.split('\n')) + tags.add('latest') + } else { + String branch_name = get_branch_name() + tags.add(branch_name) + } + + return tags +} + + +/** + * Given an image publish it with a tag to the PCIC docker registry. + * + * @param image to publish + * @return tag to use later in the security scan + */ +def publish_image(image) { + def tags = get_tags() + + withDockerServer([uri: PCIC_DOCKER]){ + docker.withRegistry('', 'PCIC_DOCKERHUB_CREDS') { + tags.each { tag -> + image.push(tag) + } + } + } + + return tags +} + + +node { + stage('Code Collection') { + checkout scm + sh 'git fetch' + } + + // Run the Python test suite + run_tests() stage('Clean Workspace') { cleanWs() } - stage('Recollect Code') { + stage('Re-collect Code') { checkout scm } + // Define image items + def image_name = BASE_REGISTRY + 'climate-explorer-backend' def image - String name = BASE_REGISTRY + 'climate-explorer-backend' + def published_tags + String scan_name - // tag branch - if (BRANCH_NAME == 'master') { - // TODO: detect tags and releases for master - } else { - name = name + ':' + BRANCH_NAME + stage('Build Image') { + image = build_image(image_name) } - stage('Build and Publish Image') { - withDockerServer([uri: PCIC_DOCKER]) { - image = docker.build(name) + stage('Publish Image') { + published_tags = publish_image(image) + } - docker.withRegistry('', 'PCIC_DOCKERHUB_CREDS') { - image.push() - } + // Only conduct security scan on branches filed as pull requests + if(BRANCH_NAME.contains('PR')) { + stage('Security Scan') { + // Use one of our published tags to identify the image to be scanned + scan_name = image_name + ':' + published_tags[0] + + writeFile file: 'anchore_images', text: scan_name + anchore name: 'anchore_images', engineRetries: '700' } } - stage('Security Scan') { - writeFile file: 'anchore_images', text: name - anchore name: 'anchore_images', engineRetries: '700' + stage('Clean Local Image') { + clean_local_image(image_name) } - stage('Clean Up Local Image') { - withDockerServer([uri: PCIC_DOCKER]){ - sh "docker rmi ${name}" - } + stage('Clean Workspace') { + cleanWs() } } From 663a448ea654a77cef11531c2eeecd7fb7a39cfe Mon Sep 17 00:00:00 2001 From: Nikola Rados Date: Fri, 6 Dec 2019 12:47:22 -0800 Subject: [PATCH 6/7] Add missing method --- Jenkinsfile | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/Jenkinsfile b/Jenkinsfile index d30d7bc4..0cfc8818 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -103,6 +103,18 @@ def publish_image(image) { } +/** + * Clean up image on dev01 + * + * @param image_name name of the image to clean up + */ +def clean_local_image(image_name) { + withDockerServer([uri: PCIC_DOCKER]){ + sh "docker rmi ${image_name}" + } +} + + node { stage('Code Collection') { checkout scm From b197dba9f6dfcdfc12c119d90e93823080931c86 Mon Sep 17 00:00:00 2001 From: Nikola Rados Date: Fri, 6 Dec 2019 12:53:55 -0800 Subject: [PATCH 7/7] Move fetch call to second code collect --- Jenkinsfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Jenkinsfile b/Jenkinsfile index 0cfc8818..ab390597 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -118,7 +118,6 @@ def clean_local_image(image_name) { node { stage('Code Collection') { checkout scm - sh 'git fetch' } // Run the Python test suite @@ -130,6 +129,7 @@ node { stage('Re-collect Code') { checkout scm + sh 'git fetch' } // Define image items