Add a new Jenkins job for the smoke test

jenkins/opensearch/smoke-test.jenkinsfile

@@ -0,0 +1,155 @@
+/*
+ * Copyright OpenSearch Contributors
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+lib = library(identifier: '[email protected]', retriever: modernSCM([
+    $class: 'GitSCMSource',
+    remote: 'https://github.com/opensearch-project/opensearch-build-libraries.git',
+]))
+
+def docker_images = [
+    'tar': 'opensearchstaging/ci-runner:ci-runner-al2-opensearch-build-v1',
+    'rpm': 'opensearchstaging/ci-runner:ci-runner-almalinux8-systemd-base-integtest-v1',
+    'deb': 'opensearchstaging/ci-runner:ci-runner-ubuntu2004-systemd-base-integtest-v3',
+    'zip': 'opensearchstaging/ci-runner:ci-runner-windows2019-opensearch-build-v1',
+]
+
+def docker_args = [
+    'tar': '-u 1000 --cpus 4 -m 16g',
+    'rpm': '--entrypoint=/usr/lib/systemd/systemd -u root --privileged -v /sys/fs/cgroup:/sys/fs/cgroup:rw --cgroupns=host --cpus 4 -m 16g',
+    'deb': '--entrypoint=/usr/lib/systemd/systemd -u root --privileged -v /sys/fs/cgroup:/sys/fs/cgroup:rw --cgroupns=host --cpus 4 -m 16g',
+    'zip': '-u ContainerAdministrator',
+]
+
+def agent_nodes = [
+    'linux_x64': 'Jenkins-Agent-AL2023-X64-M54xlarge-Docker-Host',
+    'linux_arm64': 'Jenkins-Agent-AL2023-Arm64-M6g4xlarge-Docker-Host',
+    'windows_x64': 'Jenkins-Agent-Windows2019-X64-M54xlarge-Docker-Host',
+]
+
+pipeline {
+    options {
+        timeout(time: 2, unit: 'HOURS')
+        buildDiscarder(logRotator(daysToKeepStr: '60'))
+    }
+    agent none
+    environment {
+        BUILD_MANIFEST = 'build-manifest.yml'
+        BUILD_JOB_NAME = 'distribution-build-opensearch'
+        ARTIFACT_BUCKET_NAME = credentials('jenkins-artifact-bucket-name')
+    }
+    parameters {
+        string(
+            name: 'TEST_MANIFEST',
+            description: 'Test manifest under the manifests folder, e.g. 2.19.0/opensearch-2.19.0-test.yml.',
+            trim: true
+        )
+        string(
+            name: 'BUILD_MANIFEST_URL',
+            description: 'The build manifest URL for OpenSearch, e.g. "https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/2.19.0/10545/linux/x64/tar/builds/opensearch/manifest.yml".',
+            trim: true
+        )
+    }
+    stages {
+        stage('verify-parameters') {
+            agent { label agent_nodes['linux_x64'] }
+            steps {
+                script {
+                    if (TEST_MANIFEST == '' || !fileExists("manifests/${TEST_MANIFEST}")) {
+                        currentBuild.result = 'ABORTED'
+                        error("Smoke Tests failed to start. Test manifest was not provided or not found in manifests/${TEST_MANIFEST}.")
+                    }
+
+                    if (BUILD_MANIFEST_URL == '') {
+                        currentBuild.result = 'ABORTED'
+                        error('Smoke Tests failed to start. Build manifest url was not provided.')
+                    }
+                    downloadBuildManifest(
+                        url: BUILD_MANIFEST_URL,
+                        path: BUILD_MANIFEST
+                    )
+
+                    def buildManifestObj = lib.jenkins.BuildManifest.new(readYaml(file: BUILD_MANIFEST))
+                    env.architecture = buildManifestObj.getArtifactArchitecture()
+                    env.buildId = buildManifestObj.getArtifactBuildId()
+                    env.distribution = buildManifestObj.getDistribution()
+                    env.version = buildManifestObj.build.version
+                    env.platform = buildManifestObj.build.platform
+                    env.artifactPath = buildManifestObj.getArtifactRoot(BUILD_JOB_NAME, buildId)
+                    env.AGENT_LABEL = agent_nodes["${env.platform}_${architecture}"]
+                }
+            }
+            post {
+                always {
+                    postCleanup()
+                }
+            }
+        }
+        stage('smoke-test') {
+            // Need to run this directly on agent node here in order to trigger stages with docker container and avoid docker within docker situation
+            // Can only be run in runner that is at least 50GB per container
+            agent { label AGENT_LABEL }
+            steps {
+                script {
+                    downloadBuildManifest(
+                        url: BUILD_MANIFEST_URL,
+                        path: BUILD_MANIFEST
+                    )
+
+                    def buildManifestObj = lib.jenkins.BuildManifest.new(readYaml(file: BUILD_MANIFEST))
+                    def testManifestObj = lib.jenkins.TestManifest.new(readYaml(file: "manifests/${TEST_MANIFEST}"))
+                    currentBuild.description = "$TEST_MANIFEST, $version, $architecture, $platform, $buildId, $distribution"
+
+                    // Stash the current working directory files, aka opensearch-build repo
+                    // Unstash later in each triggered stage to run integTest
+                    stash includes: '**', name: "smoketest-opensearch-$BUILD_NUMBER"
+
+                    timeout(time: 1, unit: 'HOURS') {
+                        node(AGENT_LABEL) {
+                            docker.withRegistry('https://public.ecr.aws/') {
+                                docker.image(docker_images["$distribution"]).inside(docker_args["$distribution"]) {
+                                    try {
+                                        stage("Smoke_tests") {
+                                            unstash "smoketest-opensearch-$BUILD_NUMBER"
+                                            sh('rm -rf test-results')
+                                            runSmokeTestScript(
+                                                jobName: "$BUILD_JOB_NAME",
+                                                buildManifest: "$BUILD_MANIFEST",
+                                                testManifest: "manifests/${TEST_MANIFEST}",
+                                                buildId: "${buildId}"
+                                            )
+                                        }
+                                    } catch (e) {
+                                        throw new Exception("Error running Smoke test", e)
+                                    } finally {
+                                        echo "Completed running smoke tests."
+                                        postCleanup()
+                                    }
+                                }
+                            }
+                        }
+                    }
+                }
+            }
+            post {
+                always {
+                    postCleanup()
+                }
+            }
+        }
+    }
+    post {
+        always {
+            node(AGENT_LABEL) {
+                script {
+                    postCleanup()
+                }
+            }
+        }
+    }
+}

tests/jenkins/TestSmokeTest.groovy

@@ -0,0 +1,93 @@
+/*
+ * Copyright OpenSearch Contributors
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+
+import jenkins.tests.BuildPipelineTest
+import org.junit.Before
+import org.junit.Test
+import org.yaml.snakeyaml.Yaml
+
+import static com.lesfurets.jenkins.unit.MethodCall.callArgsToString
+import static com.lesfurets.jenkins.unit.global.lib.GitSource.gitSource
+import static com.lesfurets.jenkins.unit.global.lib.LibraryConfiguration.library
+import static org.hamcrest.CoreMatchers.hasItem
+import static org.hamcrest.MatcherAssert.assertThat
+
+class TestSmokeTest extends BuildPipelineTest {
+
+    @Override
+    @Before
+    void setUp() {
+
+        helper.registerSharedLibrary(
+            library().name('jenkins')
+                .defaultVersion('8.1.0')
+                .allowOverride(true)
+                .implicit(true)
+                .targetPath('vars')
+                .retriever(gitSource('https://github.com/opensearch-project/opensearch-build-libraries.git'))
+                .build()
+            )
+
+        super.setUp()
+
+        def jobName = "dummy_job"
+        def testManifest = "tests/jenkins/data/opensearch-2.19.0-test.yml"
+        def buildManifest = "tests/jenkins/data/opensearch-2.19.0-build.yml"
+        def buildManifestUrl = "https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/2.19.0/10545/linux/x64/tar/dist/opensearch/opensearch-2.19.0-linux-x64.tar.gz"
+        def agentLabel = "Jenkins-Agent-AL2-X64-C54xlarge-Docker-Host"
+
+        binding.setVariable('env', ['BUILD_NUMBER': '234', 'PUBLIC_ARTIFACT_URL': 'DUMMY_PUBLIC_ARTIFACT_URL', 'JOB_NAME': 'dummy_job', 'DOCKER_AGENT':[image:'opensearchstaging/ci-runner:ci-runner-centos7-v1', args:'-e JAVA_HOME=/opt/java/openjdk-11']])
+        binding.setVariable('BUILD_JOB_NAME', 'dummy_job')
+        binding.setVariable('TEST_MANIFEST', testManifest)
+        binding.setVariable('BUILD_MANIFEST_URL', buildManifestUrl)
+        binding.setVariable('AGENT_LABEL', agentLabel)
+        binding.setVariable('BUILD_NUMBER', '234')
+        binding.setVariable('BUILD_MANIFEST', buildManifest)
+        helper.registerAllowedMethod("readYaml", [Map.class], { args ->
+            if (args.file == 'manifests/tests/jenkins/data/opensearch-2.19.0-test.yml') {
+                return new Yaml().load((testManifest as File).text)
+            } else if (args.file == 'tests/jenkins/data/opensearch-2.19.0-build.yml') {
+                return new Yaml().load((buildManifest as File).text)
+            } else {
+                println("Manifest not found ${args.file}")
+            }
+        })
+        helper.addFileExistsMock("manifests/${testManifest}", true)
+        helper.registerAllowedMethod('unstash', [String.class], null)
+    }
+
+    @Test
+    void smokeTests_runs() {
+        addParam('UPDATE_GITHUB_ISSUES', true)
+        super.testPipeline('jenkins/opensearch/smoke-test.jenkinsfile',
+                'tests/jenkins/jenkinsjob-regression-files/opensearch/smoke-test.jenkinsfile')
+        assertThat(getCommandExecutions('sh', 'test.sh'), hasItem('./test.sh smoke-test manifests/tests/jenkins/data/opensearch-2.19.0-test.yml --test-run-id 234 --paths opensearch=https://ci.opensearch.org/ci/dbc/dummy_job/2.19.0/10545/linux/x64/tar'))
+    }
+
+    @Test
+    void checkError() {
+        helper.addFileExistsMock('manifests/tests/jenkins/data/opensearch-2.19.0-test.yml', false)
+        runScript('jenkins/opensearch/smoke-test.jenkinsfile')
+        assertThat(getCommandExecutions('error', ''), hasItem('Smoke Tests failed to start. Test manifest was not provided or not found in manifests/tests/jenkins/data/opensearch-2.19.0-test.yml.'))
+        assertJobStatusFailure()
+    }
+
+    def getCommandExecutions(methodName, command) {
+        def shCommands = helper.callStack.findAll {
+            call ->
+                call.methodName == methodName
+        }.
+        collect {
+            call ->
+                callArgsToString(call)
+        }.findAll {
+            shCommand ->
+                shCommand.contains(command)
+        }
+
+        return shCommands
+    }
+}