[Backport 5.x] Docker Patch Library to support Docker re-release Auto…

…mation (#282) Signed-off-by: Divya Madala <[email protected]> Signed-off-by: Divya Madala <[email protected]> Signed-off-by: github-actions[bot] <github-actions[bot]@users.noreply.github.com> Signed-off-by: Sayali Gaikawad <[email protected]> Co-authored-by: github-actions[bot] <github-actions[bot]@users.noreply.github.com> Co-authored-by: Sayali Gaikawad <[email protected]>
opensearch-project · Aug 18, 2023 · 4d37717 · 4d37717
1 parent 8598bd9
commit 4d37717
Show file tree

Hide file tree

Showing 9 changed files with 296 additions and 16 deletions.
diff --git a/build.gradle b/build.gradle
@@ -120,7 +120,7 @@ jacocoTestReport {
     }
 }
 
-String version = '5.5.0'
+String version = '5.6.0'
 
 task updateVersion {
     doLast {

diff --git a/tests/jenkins/TestPatchDockerImage.groovy b/tests/jenkins/TestPatchDockerImage.groovy
@@ -0,0 +1,33 @@
+/*
+ * Copyright OpenSearch Contributors
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+import jenkins.tests.BuildPipelineTest
+import org.junit.Before
+import org.junit.Test
+
+
+class TestPatchDockerImage extends BuildPipelineTest {
+
+    @Before
+    void setUp() {
+        this.registerLibTester(new PatchDockerImageLibTester(
+            'opensearch',
+            '1',
+            'true'
+            )
+        )
+        super.setUp()
+    }
+
+    @Test
+    void testPatchDockerImage() {
+
+        super.testPipeline("tests/jenkins/jobs/PatchDockerImage_Jenkinsfile")
+    }
+}
diff --git a/tests/jenkins/jobs/BuildDockerImage_Jenkinsfile_builds_both.txt b/tests/jenkins/jobs/BuildDockerImage_Jenkinsfile_builds_both.txt
@@ -8,12 +8,12 @@
                   buildDockerImage.library({[email protected], retriever=null})
                   buildDockerImage.readYaml({file=tests/data/opensearch-1.3.0.yml})
                   InputManifest.asBoolean()
-                  buildDockerImage.echo(Trigger docker-build)
+                  buildDockerImage.echo(Triggering docker-build)
                   buildDockerImage.string({name=DOCKER_BUILD_GIT_REPOSITORY, value=https://github.com/opensearch-project/opensearch-build})
                   buildDockerImage.string({name=DOCKER_BUILD_GIT_REPOSITORY_REFERENCE, value=main})
                   buildDockerImage.string({name=DOCKER_BUILD_SCRIPT_WITH_COMMANDS, value=id && pwd && cd docker/release && curl -sSL opensearch.linux.x64 -o opensearch-x64.tgz && curl -sSL opensearch.linux.arm64 -o opensearch-arm64.tgz && bash build-image-multi-arch.sh -v 1.3.0 -f ./dockerfiles/opensearch.al2.dockerfile -p opensearch -a 'x64,arm64' -r opensearchstaging/opensearch -t 'opensearch-x64.tgz,opensearch-arm64.tgz' -n 33})
-                  buildDockerImage.build({job=docker-build, parameters=[null, null, null]})
-                  buildDockerImage.echo(Trigger docker create tag with build number)
-                  buildDockerImage.echo(Trigger docker-scan for opensearch version 1.3.0)
+                  buildDockerImage.build({job=docker-build, propagate=true, wait=true, parameters=[null, null, null]})
+                  buildDockerImage.echo(Triggering docker create tag with build number)
+                  buildDockerImage.echo(Triggering docker-scan for opensearch version 1.3.0)
                   buildDockerImage.string({name=IMAGE_FULL_NAME, value=opensearchstaging/opensearch:1.3.0})
-                  buildDockerImage.build({job=docker-scan, parameters=[null]})
+                  buildDockerImage.build({job=docker-scan, propagate=true, wait=true, parameters=[null]})
diff --git a/tests/jenkins/jobs/BuildDockerImage_Qualifier_Jenkinsfile_builds_both.txt b/tests/jenkins/jobs/BuildDockerImage_Qualifier_Jenkinsfile_builds_both.txt
@@ -8,12 +8,12 @@
                   buildDockerImage.library({[email protected], retriever=null})
                   buildDockerImage.readYaml({file=tests/data/opensearch-2.0.0.yml})
                   InputManifest.asBoolean()
-                  buildDockerImage.echo(Trigger docker-build)
+                  buildDockerImage.echo(Triggering docker-build)
                   buildDockerImage.string({name=DOCKER_BUILD_GIT_REPOSITORY, value=https://github.com/opensearch-project/opensearch-build})
                   buildDockerImage.string({name=DOCKER_BUILD_GIT_REPOSITORY_REFERENCE, value=main})
                   buildDockerImage.string({name=DOCKER_BUILD_SCRIPT_WITH_COMMANDS, value=id && pwd && cd docker/release && curl -sSL opensearch.linux.x64 -o opensearch-x64.tgz && curl -sSL opensearch.linux.arm64 -o opensearch-arm64.tgz && bash build-image-multi-arch.sh -v 2.0.0-alpha1 -f ./dockerfiles/opensearch.al2.dockerfile -p opensearch -a 'x64,arm64' -r opensearchstaging/opensearch -t 'opensearch-x64.tgz,opensearch-arm64.tgz' -n 33})
-                  buildDockerImage.build({job=docker-build, parameters=[null, null, null]})
-                  buildDockerImage.echo(Trigger docker create tag with build number)
-                  buildDockerImage.echo(Trigger docker-scan for opensearch version 2.0.0-alpha1)
+                  buildDockerImage.build({job=docker-build, propagate=true, wait=true, parameters=[null, null, null]})
+                  buildDockerImage.echo(Triggering docker create tag with build number)
+                  buildDockerImage.echo(Triggering docker-scan for opensearch version 2.0.0-alpha1)
                   buildDockerImage.string({name=IMAGE_FULL_NAME, value=opensearchstaging/opensearch:2.0.0-alpha1})
-                  buildDockerImage.build({job=docker-scan, parameters=[null]})
+                  buildDockerImage.build({job=docker-scan, propagate=true, wait=true, parameters=[null]})
diff --git a/tests/jenkins/jobs/PatchDockerImage_Jenkinsfile b/tests/jenkins/jobs/PatchDockerImage_Jenkinsfile
@@ -0,0 +1,25 @@
+/*
+ * Copyright OpenSearch Contributors
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+pipeline {
+    agent none
+    stages {
+        stage('Patch docker image') {
+            steps {
+                script {
+                    patchDockerImage(
+                        product: "opensearch",
+                        tag: "1",
+                        re_release: "true"
+                    )
+                }
+            }
+        }
+    }
+}
diff --git a/tests/jenkins/jobs/PatchDockerImage_Jenkinsfile.txt b/tests/jenkins/jobs/PatchDockerImage_Jenkinsfile.txt
@@ -0,0 +1,44 @@
+   PatchDockerImage_Jenkinsfile.run()
+      PatchDockerImage_Jenkinsfile.pipeline(groovy.lang.Closure)
+         PatchDockerImage_Jenkinsfile.echo(Executing on agent [label:none])
+         PatchDockerImage_Jenkinsfile.stage(Patch docker image, groovy.lang.Closure)
+            PatchDockerImage_Jenkinsfile.script(groovy.lang.Closure)
+               PatchDockerImage_Jenkinsfile.patchDockerImage({product=opensearch, tag=1, re_release=true})
+                  patchDockerImage.legacySCM(groovy.lang.Closure)
+                  patchDockerImage.library({identifier=jenkins@main, retriever=null})
+                  patchDockerImage.sh(#!/bin/bash
+    set -e
+    set +x
+    docker pull opensearchproject/opensearch:1
+    docker pull opensearchproject/opensearch:latest
+    )
+                  patchDockerImage.sh({script=docker inspect --format '{{ index .Config.Labels "org.label-schema.version"}}' opensearchproject/opensearch:1, returnStdout=true})
+                  patchDockerImage.sh({script=docker inspect --format '{{ index .Config.Labels "org.label-schema.build-date"}}' opensearchproject/opensearch:1, returnStdout=true})
+                  patchDockerImage.sh({script=docker inspect --format '{{ index .Config.Labels "org.label-schema.description"}}' opensearchproject/opensearch:1, returnStdout=true})
+                  patchDockerImage.sh({script=docker inspect --format '{{ index .Config.Labels "org.label-schema.version"}}' opensearchproject/opensearch:latest, returnStdout=true})
+                  patchDockerImage.readYaml({file=manifests/1.3.0/opensearch-1.3.0.yml})
+                  InputManifest.asBoolean()
+                  patchDockerImage.buildDockerImage({inputManifest=manifests/1.3.0/opensearch-1.3.0.yml, buildNumber=7756, buildDate=20230619, buildOption=re_release_docker_image, artifactUrlX64=https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/1.3.0/7756/linux/x64/tar/dist/opensearch/opensearch-1.3.0-linux-x64.tar.gz, artifactUrlArm64=https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/1.3.0/7756/linux/arm64/tar/dist/opensearch/opensearch-1.3.0-linux-arm64.tar.gz})
+                     buildDockerImage.legacySCM(groovy.lang.Closure)
+                     buildDockerImage.library({[email protected], retriever=null})
+                     buildDockerImage.readYaml({file=manifests/1.3.0/opensearch-1.3.0.yml})
+                     InputManifest.asBoolean()
+                     buildDockerImage.echo(Triggering docker-build)
+                     buildDockerImage.string({name=DOCKER_BUILD_GIT_REPOSITORY, value=https://github.com/opensearch-project/opensearch-build})
+                     buildDockerImage.string({name=DOCKER_BUILD_GIT_REPOSITORY_REFERENCE, value=main})
+                     buildDockerImage.string({name=DOCKER_BUILD_SCRIPT_WITH_COMMANDS, value=id && pwd && cd docker/release && curl -sSL https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/1.3.0/7756/linux/x64/tar/dist/opensearch/opensearch-1.3.0-linux-x64.tar.gz -o opensearch-x64.tgz && curl -sSL https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/1.3.0/7756/linux/arm64/tar/dist/opensearch/opensearch-1.3.0-linux-arm64.tar.gz -o opensearch-arm64.tgz && bash build-image-multi-arch.sh -v 1.3.0 -f ./dockerfiles/opensearch.al2.dockerfile -p opensearch -a 'x64,arm64' -r opensearchstaging/opensearch -t 'opensearch-x64.tgz,opensearch-arm64.tgz' -n 7756})
+                     buildDockerImage.build({job=docker-build, propagate=true, wait=true, parameters=[null, null, null]})
+                     buildDockerImage.echo(Triggering docker create tag with build number)
+                     buildDockerImage.string({name=SOURCE_IMAGE_REGISTRY, value=opensearchstaging})
+                     buildDockerImage.string({name=SOURCE_IMAGE, value=opensearch:1.3.0})
+                     buildDockerImage.string({name=DESTINATION_IMAGE_REGISTRY, value=opensearchstaging})
+                     buildDockerImage.string({name=DESTINATION_IMAGE, value=opensearch:1.3.0.7756.20230619})
+                     buildDockerImage.build({job=docker-copy, propagate=true, wait=true, parameters=[null, null, null, null]})
+                     buildDockerImage.echo(Triggering docker-scan for opensearch version 1.3.0)
+                     buildDockerImage.string({name=IMAGE_FULL_NAME, value=opensearchstaging/opensearch:1.3.0})
+                     buildDockerImage.build({job=docker-scan, propagate=true, wait=true, parameters=[null]})
+                  patchDockerImage.echo(Triggering docker-promotion)
+                  patchDockerImage.string({name=SOURCE_IMAGES, value=opensearch:1.3.0.7756.20230619})
+                  patchDockerImage.string({name=RELEASE_VERSION, value=1.3.0})
+                  patchDockerImage.booleanParam({name=TAG_LATEST, value=false})
+                  patchDockerImage.build({job=docker-promotion, propagate=true, wait=true, parameters=[null, null, null]})
diff --git a/tests/jenkins/lib-testers/PatchDockerImageLibtester.groovy b/tests/jenkins/lib-testers/PatchDockerImageLibtester.groovy
@@ -0,0 +1,62 @@
+/*
+ * Copyright OpenSearch Contributors
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+import static org.hamcrest.CoreMatchers.notNullValue
+import static org.hamcrest.MatcherAssert.assertThat
+import org.yaml.snakeyaml.Yaml
+
+
+
+class PatchDockerImageLibTester extends LibFunctionTester {
+
+    private String product
+    private String tag
+    private String re_release
+
+    public PatchDockerImageLibTester(product, tag, re_release){
+        this.product = product
+        this.tag = tag
+        this.re_release = re_release
+    }
+
+    void configure(helper, binding) {
+        def inputManifest = "tests/data/opensearch-1.3.0.yml"
+        binding.setVariable('MANIFEST', inputManifest)
+
+        helper.addShMock("""docker inspect --format '{{ index .Config.Labels "org.label-schema.version"}}' opensearchproject/opensearch:1""") { script ->
+            return [stdout: "1.3.0", exitValue: 0]
+        }
+        helper.addShMock("""docker inspect --format '{{ index .Config.Labels "org.label-schema.description"}}' opensearchproject/opensearch:1""") { script ->
+            return [stdout: "7756", exitValue: 0]
+        }
+        helper.addShMock("""docker inspect --format '{{ index .Config.Labels "org.label-schema.build-date"}}' opensearchproject/opensearch:1""") { script ->
+            return [stdout: "2023-06-19T19:12:59Z", exitValue: 0]
+        }
+        helper.addShMock("""docker inspect --format '{{ index .Config.Labels "org.label-schema.version"}}' opensearchproject/opensearch:latest""") { script ->
+            return [stdout: "2.5.0", exitValue: 0]
+        }
+        helper.registerAllowedMethod('readYaml', [Map.class], { args ->
+            return new Yaml().load((inputManifest as File).text)
+        })
+        helper.registerAllowedMethod("git", [Map])
+    }
+
+    void parameterInvariantsAssertions(call) {
+        assertThat(call.args.product.first(), notNullValue())
+        assertThat(call.args.tag.first(), notNullValue())
+    }
+
+    boolean expectedParametersMatcher(call) {
+        return call.args.product.first().toString().equals(this.product)
+                && call.args.tag.first().toString().equals(this.tag)
+    }
+
+    String libFunctionName() {
+        return 'patchDockerImage'
+    }
+}
diff --git a/vars/buildDockerImage.groovy b/vars/buildDockerImage.groovy
@@ -6,11 +6,26 @@
  * this file be licensed under the Apache-2.0 license or a
  * compatible open source license.
  */
+
+/**
+Library to build Docker Image with different Build Options
+@param Map[inputManifest] <Required> - Path to Input Manifest.
+@param Map[buildNumber] <Required> - Build number of the corresponding Artifact.
+@param Map[buildDate] <Optional> - Date on which the artifacts were built.
+@param Map[artifactUrlX64] <Required> - Url Path to X64 Tarball.
+@param Map[artifactUrlARM64] <Required> - Url Path to ARM64 Tarball.
+@param Map[buildOption] <Required> - Build Option for building the image with different options.
+*/
 void call(Map args = [:]) {
     def lib = library(identifier: '[email protected]', retriever: legacySCM(scm))
     def inputManifest = lib.jenkins.InputManifest.new(readYaml(file: args.inputManifest))
     def build_qualifier = inputManifest.build.qualifier
     def build_number = args.buildNumber ?: "${BUILD_NUMBER}"
+    String image_tag = ""
+
+    if (args.buildDate != null && args.buildDate != 'null'){
+        image_tag = "." + "${args.buildDate}"
+    }
 
     if (build_qualifier != null && build_qualifier != 'null') {
         build_qualifier = "-" + build_qualifier
@@ -23,9 +38,11 @@ void call(Map args = [:]) {
     if (args.artifactUrlX64 == null || args.artifactUrlArm64 ==  null) {
         echo 'Skipping docker build, one of x64 or arm64 artifacts was not built.'
     } else {
-        echo 'Trigger docker-build'
+        echo 'Triggering docker-build'
         dockerBuild: {
             build job: 'docker-build',
+            propagate: true,
+            wait: true,
             parameters: [
                 string(name: 'DOCKER_BUILD_GIT_REPOSITORY', value: 'https://github.com/opensearch-project/opensearch-build'),
                 string(name: 'DOCKER_BUILD_GIT_REPOSITORY_REFERENCE', value: 'main'),
@@ -50,22 +67,26 @@ void call(Map args = [:]) {
             ]
         }
 
-        echo 'Trigger docker create tag with build number'
-        if (args.buildOption == "build_docker_with_build_number_tag") {
+        echo 'Triggering docker create tag with build number'
+        if (args.buildOption == "build_docker_with_build_number_tag" || args.buildOption == "re_release_docker_image") {
             dockerCopy: {
                 build job: 'docker-copy',
+                propagate: true,
+                wait: true,
                 parameters: [
                     string(name: 'SOURCE_IMAGE_REGISTRY', value: 'opensearchstaging'),
                     string(name: 'SOURCE_IMAGE', value: "${filename}:${inputManifest.build.version}${build_qualifier}"),
                     string(name: 'DESTINATION_IMAGE_REGISTRY', value: 'opensearchstaging'),
-                    string(name: 'DESTINATION_IMAGE', value: "${filename}:${inputManifest.build.version}${build_qualifier}.${build_number}")
+                    string(name: 'DESTINATION_IMAGE', value: "${filename}:${inputManifest.build.version}${build_qualifier}.${build_number}${image_tag}")
                 ]
             }
         }
 
-        echo "Trigger docker-scan for ${filename} version ${inputManifest.build.version}${build_qualifier}"
+        echo "Triggering docker-scan for ${filename} version ${inputManifest.build.version}${build_qualifier}"
         dockerScan: {
             build job: 'docker-scan',
+            propagate: true,
+            wait: true,
             parameters: [
                 string(name: 'IMAGE_FULL_NAME', value: "opensearchstaging/${filename}:${inputManifest.build.version}${build_qualifier}")
             ]

diff --git a/vars/patchDockerImage.groovy b/vars/patchDockerImage.groovy
@@ -0,0 +1,95 @@
+/*
+ * Copyright OpenSearch Contributors
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+/**
+Library to support Docker Image Re-Release Automation
+@param Map[product] <Required> - Product type refers to opensearch or opensearch-dashboards.
+@param Map[tag] <Required> - Tag of the product that needs to be re-released.
+@param Map[re_release] <Optional> - This Build-Option can be checked to release the image after Docker-Build.
+*/
+void call(Map args = [:]) {
+    def lib = library(identifier: 'jenkins@main', retriever: legacySCM(scm))
+    String docker_image = "opensearchproject/${args.product}:${args.tag}"
+    String latest_docker_image = "opensearchproject/${args.product}:latest"
+    boolean tag_latest = false
+    String build_option = "build_docker_image"
+
+    sh """#!/bin/bash
+    set -e
+    set +x
+    docker pull ${docker_image}
+    docker pull ${latest_docker_image}
+    """
+
+    def version = sh (
+            script: """docker inspect --format '{{ index .Config.Labels "org.label-schema.version"}}' ${docker_image}""",
+            returnStdout: true
+    ).trim()
+    def build_time = sh (
+            script: """docker inspect --format '{{ index .Config.Labels "org.label-schema.build-date"}}' ${docker_image}""",
+            returnStdout: true
+    ).trim()
+    def build_number = sh (
+            script: """docker inspect --format '{{ index .Config.Labels "org.label-schema.description"}}' ${docker_image}""",
+            returnStdout: true
+    ).trim()
+    def latest_version = sh (
+            script: """docker inspect --format '{{ index .Config.Labels "org.label-schema.version"}}' ${latest_docker_image}""",
+            returnStdout: true
+    ).trim()
+
+    def inputManifest = lib.jenkins.InputManifest.new(readYaml(file: "manifests/${version}/${args.product}-${version}.yml"))
+
+    artifactUrlX64 = "https://ci.opensearch.org/ci/dbc/distribution-build-${args.product}/${version}/${build_number}/linux/x64/tar/dist/${args.product}/${args.product}-${version}-linux-x64.tar.gz"
+
+    artifactUrlARM64 = "https://ci.opensearch.org/ci/dbc/distribution-build-${args.product}/${version}/${build_number}/linux/arm64/tar/dist/${args.product}/${args.product}-${version}-linux-arm64.tar.gz"
+
+    //slice the build-date value (For Example: 2023-08-11T02:17:43Z -> 20230811)
+    build_date = build_time[0..3] + build_time[5..6] + build_time[8..9]
+
+    def build_qualifier = inputManifest.build.qualifier
+
+    if (build_qualifier != null && build_qualifier != 'null') {
+        build_qualifier = "-" + build_qualifier
+    }
+    else {
+        build_qualifier = ''
+    }
+
+    if (latest_version == version){
+        tag_latest = true
+    }
+
+    if (args.re_release){
+        build_option = "re_release_docker_image"
+    }
+
+    buildDockerImage(
+        inputManifest: "manifests/${version}/${args.product}-${version}.yml",
+        buildNumber: "${build_number}",
+        buildDate: "${build_date}",
+        buildOption: "${build_option}",
+        artifactUrlX64: "${artifactUrlX64}",
+        artifactUrlArm64: "${artifactUrlARM64}"
+    )
+
+    echo 'Triggering docker-promotion'
+    if(args.re_release){
+        dockerPromote: {
+            build job: 'docker-promotion',
+            propagate: true,
+            wait: true,
+            parameters: [
+                string(name: 'SOURCE_IMAGES', value: "${args.product}:${inputManifest.build.version}${build_qualifier}.${build_number}.${build_date}"),
+                string(name: 'RELEASE_VERSION', value: "${version}"),
+                booleanParam(name: 'TAG_LATEST', value: "${tag_latest}")
+            ]
+        }
+    }
+}