Merge remote-tracking branch 'upstream/main' into opensearch-windows-…

…bundle-6 Signed-off-by: Peter Zhu <[email protected]>
opensearch-project · Oct 4, 2022 · 1d1a8fc · 1d1a8fc
2 parents 755cde6 + 077fc28
commit 1d1a8fc
Show file tree

Hide file tree

Showing 9 changed files with 544 additions and 49 deletions.
diff --git a/jenkins/data-prepper/release-data-prepper-all-artifacts.jenkinsfile b/jenkins/data-prepper/release-data-prepper-all-artifacts.jenkinsfile
@@ -11,6 +11,8 @@ pipeline {
     parameters {
         string(name: 'VERSION', description: 'The version of Data Prepper', trim: true)
         string(name: 'DATA_PREPPER_BUILD_NUMBER', description: 'The build number of the Data Prepper build from GitHub to release.', trim: true)
+        booleanParam(name: 'RELEASE_MAJOR_TAG', description: 'Whether to create major tag of docker image or not', defaultValue: false)
+        booleanParam(name: 'RELEASE_LATEST_TAG', description: 'Whether to create latest tag of docker image or not', defaultValue: false)
     }
     environment {
         DATA_PREPPER_ARTIFACT_STAGING_SITE = credentials('jenkins-data-prepper-artifact-staging-site')
@@ -92,6 +94,29 @@ pipeline {
                                     string(name: 'DESTINATION_IMAGE_REGISTRY', value: 'opensearchproject'),
                                     string(name: 'DESTINATION_IMAGE', value: "data-prepper:${VERSION}")
                                 ]
+
+                            if (RELEASE_MAJOR_TAG) {
+                                def majorVersion = VERSION.tokenize(".")[0].trim()
+                                def dockerCopyHubMajor =
+                                    build job: 'docker-copy',
+                                    parameters: [
+                                        string(name: 'SOURCE_IMAGE_REGISTRY', value: "${DATA_PREPPER_STAGING_CONTAINER_REPOSITORY}"),
+                                        string(name: 'SOURCE_IMAGE', value: "data-prepper:${VERSION}-${DATA_PREPPER_BUILD_NUMBER}"),
+                                        string(name: 'DESTINATION_IMAGE_REGISTRY', value: 'opensearchproject'),
+                                        string(name: 'DESTINATION_IMAGE', value: "data-prepper:${majorVersion}")
+                                    ]
+                            }
+
+                            if (RELEASE_LATEST_TAG) {
+                                def dockerCopyHubLatest =
+                                    build job: 'docker-copy',
+                                    parameters: [
+                                        string(name: 'SOURCE_IMAGE_REGISTRY', value: "${DATA_PREPPER_STAGING_CONTAINER_REPOSITORY}"),
+                                        string(name: 'SOURCE_IMAGE', value: "data-prepper:${VERSION}-${DATA_PREPPER_BUILD_NUMBER}"),
+                                        string(name: 'DESTINATION_IMAGE_REGISTRY', value: 'opensearchproject'),
+                                        string(name: 'DESTINATION_IMAGE', value: "data-prepper:latest")
+                                    ]
+                            }
                         }
                     }
                 }
@@ -106,6 +131,29 @@ pipeline {
                                     string(name: 'DESTINATION_IMAGE_REGISTRY', value: 'public.ecr.aws/opensearchproject'),
                                     string(name: 'DESTINATION_IMAGE', value: "data-prepper:${VERSION}")
                                 ]
+
+                            if (RELEASE_MAJOR_TAG) {
+                                def majorVersion = VERSION.tokenize(".")[0].trim()
+                                def dockerCopyECRMajor =
+                                    build job: 'docker-copy',
+                                    parameters: [
+                                        string(name: 'SOURCE_IMAGE_REGISTRY', value: "${DATA_PREPPER_STAGING_CONTAINER_REPOSITORY}"),
+                                        string(name: 'SOURCE_IMAGE', value: "data-prepper:${VERSION}-${DATA_PREPPER_BUILD_NUMBER}"),
+                                        string(name: 'DESTINATION_IMAGE_REGISTRY', value: 'public.ecr.aws/opensearchproject'),
+                                        string(name: 'DESTINATION_IMAGE', value: "data-prepper:${majorVersion}")
+                                    ]
+                            }
+
+                            if (RELEASE_LATEST_TAG) {
+                                def dockerCopyECRLatest =
+                                    build job: 'docker-copy',
+                                    parameters: [
+                                        string(name: 'SOURCE_IMAGE_REGISTRY', value: "${DATA_PREPPER_STAGING_CONTAINER_REPOSITORY}"),
+                                        string(name: 'SOURCE_IMAGE', value: "data-prepper:${VERSION}-${DATA_PREPPER_BUILD_NUMBER}"),
+                                        string(name: 'DESTINATION_IMAGE_REGISTRY', value: 'public.ecr.aws/opensearchproject'),
+                                        string(name: 'DESTINATION_IMAGE', value: "data-prepper:latest")
+                                    ]
+                            }
                         }
                     }
                 }

diff --git a/jenkins/opensearch/distribution-build.jenkinsfile b/jenkins/opensearch/distribution-build.jenkinsfile
@@ -43,8 +43,8 @@ pipeline {
         )
         string( // Note: need to update 'verify-parameters' entries if you add new platform(s)
             name: 'BUILD_PLATFORM',
-            description: "Build selected platform related artifacts, choices include 'all', 'linux', 'macos', 'windows'. Can combine multiple platforms with space in between (maven snapshot is only on linux)",
-            defaultValue: 'all',
+            description: "Build selected platform related artifacts, choices include 'linux', 'macos', 'windows'. Can combine multiple platforms with space in between (maven snapshot is only on linux)",
+            defaultValue: 'linux macos windows',
             trim: true
         )
         choice(
@@ -64,30 +64,31 @@ pipeline {
         )
     }
     stages {
-        stage('verify-parameters') {
+        stage('Initialize the job info yaml report') {
             agent {
                 docker {
                     label AGENT_X64
-                    image 'alpine:3'
+                    image dockerAgent.image
                     alwaysPull true
                 }
             }
             steps {
                 script {
-                    def build_platform_array = params.BUILD_PLATFORM.tokenize(' ')
-                    echo("User Entry Platforms: ${params.BUILD_PLATFORM}, ${build_platform_array}")
-                    def all_platforms = "all linux macos windows"
-                    echo("All Supported Platforms: ${all_platforms}")
-                    for (String plat : build_platform_array) {
-                        if (! all_platforms.contains(plat)) {
-                            currentBuild.result = 'ABORTED'
-                            error("Missing parameter: BUILD_PLATFORM (possible entries: ${all_platforms}).")
-                        }
-                    }
+                    buildInfoYaml(
+                        componentName: COMPONENT_NAME,
+                        inputManifest: "manifests/$INPUT_MANIFEST",
+                        status: "NOT_STARTED",
+                        stage: "INITIALIZE_STAGE"
+                    )
+                }
+            }
+            post {
+                always {
+                    postCleanup()
                 }
             }
         }
-        stage('detect docker image + args') {
+        stage('verify-parameters') {
             agent {
                 docker {
                     label AGENT_X64
@@ -97,32 +98,21 @@ pipeline {
             }
             steps {
                 script {
+                    echo("Detect Docker Images and Related Parameters")
                     dockerAgent = detectDockerAgent()
                     currentBuild.description = INPUT_MANIFEST
-                }
-            }
-        }
-        stage('Initialize the job info yaml report') {
-            agent {
-                docker {
-                    label AGENT_X64
-                    image dockerAgent.image
-                    alwaysPull true
-                }
-            }
-            steps {
-                script {
-                    buildInfoYaml(
-                        componentName: COMPONENT_NAME,
-                        inputManifest: "manifests/$INPUT_MANIFEST",
-                        status: "NOT_STARTED",
-                        stage: "INITIALIZE_STAGE"
-                    )
-                }
-            }
-            post {
-                always {
-                    postCleanup()
+
+                    echo("Verify Build Platforms")
+                    def build_platform_array = params.BUILD_PLATFORM.tokenize(' ')
+                    echo("User Entry Platforms: '${params.BUILD_PLATFORM}', ${build_platform_array}")
+                    def all_platforms = "linux macos windows"
+                    echo("All Supported Platforms: '${all_platforms}'")
+                    for (String plat : build_platform_array) {
+                        if (! all_platforms.contains(plat)) {
+                            currentBuild.result = 'ABORTED'
+                            error("Missing parameter: BUILD_PLATFORM (possible entries: ${all_platforms}).")
+                        }
+                    }
                 }
             }
         }
@@ -132,7 +122,7 @@ pipeline {
                     when {
                         beforeAgent true
                         expression{
-                            params.BUILD_PLATFORM.contains('all') || params.BUILD_PLATFORM.contains('linux')
+                            params.BUILD_PLATFORM.contains('linux')
                         }
                     }
                     environment {
@@ -183,7 +173,7 @@ pipeline {
                     when {
                         beforeAgent true
                         expression{
-                            params.BUILD_PLATFORM.contains('all') || params.BUILD_PLATFORM.contains('linux')
+                            params.BUILD_PLATFORM.contains('linux')
                         }
                     }
                     agent {
@@ -221,7 +211,7 @@ pipeline {
                     when {
                         beforeAgent true
                         expression{
-                            params.BUILD_PLATFORM.contains('all') || params.BUILD_PLATFORM.contains('macos')
+                            params.BUILD_PLATFORM.contains('macos')
                         }
                     }
                     agent {
@@ -259,7 +249,7 @@ pipeline {
                     when {
                         beforeAgent true
                         expression{
-                            params.BUILD_PLATFORM.contains('all') || params.BUILD_PLATFORM.contains('windows')
+                            params.BUILD_PLATFORM.contains('windows')
                         }
                     }
                     agent {
@@ -297,7 +287,7 @@ pipeline {
                     when {
                         beforeAgent true
                         expression{
-                            params.BUILD_PLATFORM.contains('all') || params.BUILD_PLATFORM.contains('linux')
+                            params.BUILD_PLATFORM.contains('linux')
                         }
                     }
                     agent {
@@ -402,7 +392,7 @@ pipeline {
                     when {
                         beforeAgent true
                         expression{
-                            params.BUILD_PLATFORM.contains('all') || params.BUILD_PLATFORM.contains('linux')
+                            params.BUILD_PLATFORM.contains('linux')
                         }
                     }
                     agent { label AGENT_X64 }
@@ -498,7 +488,7 @@ pipeline {
                     when {
                         beforeAgent true
                         expression{
-                            params.BUILD_PLATFORM.contains('all') || params.BUILD_PLATFORM.contains('linux')
+                            params.BUILD_PLATFORM.contains('linux')
                         }
                     }
                     agent {
@@ -602,7 +592,7 @@ pipeline {
                     when {
                         beforeAgent true
                         expression{
-                            params.BUILD_PLATFORM.contains('all') || params.BUILD_PLATFORM.contains('linux')
+                            params.BUILD_PLATFORM.contains('linux')
                         }
                     }
                     agent { label AGENT_X64 }
@@ -698,7 +688,7 @@ pipeline {
                     when {
                         beforeAgent true
                         expression{
-                            params.BUILD_PLATFORM.contains('all') || params.BUILD_PLATFORM.contains('windows')
+                            params.BUILD_PLATFORM.contains('windows')
                         }
                     }
                     agent {
@@ -768,7 +758,7 @@ pipeline {
                         params.BUILD_DOCKER != 'do_not_build_docker'
                     }
                     expression{
-                        params.BUILD_PLATFORM.contains('all') || params.BUILD_PLATFORM.contains('linux')
+                        params.BUILD_PLATFORM.contains('linux')
                     }
                 }
             }

diff --git a/manifests/1.3.6/opensearch-1.3.6-test.yml b/manifests/1.3.6/opensearch-1.3.6-test.yml
@@ -0,0 +1,71 @@
+---
+schema-version: '1.0'
+name: OpenSearch
+ci:
+  image:
+    name: opensearchstaging/ci-runner:ci-runner-centos7-opensearch-build-v2
+    args: -e JAVA_HOME=/opt/java/openjdk-11
+components:
+  - name: index-management
+    integ-test:
+      build-dependencies:
+        - job-scheduler
+      test-configs:
+        - with-security
+        - without-security
+      additional-cluster-configs:
+        path.repo: [/tmp]
+
+  - name: anomaly-detection
+    integ-test:
+      build-dependencies:
+        - job-scheduler
+      test-configs:
+        - with-security
+        - without-security
+
+  - name: asynchronous-search
+    integ-test:
+      test-configs:
+        - with-security
+        - without-security
+
+  - name: alerting
+    integ-test:
+      test-configs:
+        - with-security
+        - without-security
+      additional-cluster-configs:
+        plugins.destination.host.deny_list: [10.0.0.0/8, 127.0.0.1]
+
+  - name: sql
+    integ-test:
+      test-configs:
+        - with-security
+        - without-security
+      additional-cluster-configs:
+        script.context.field.max_compilations_rate: 1000/1m
+
+  - name: k-NN
+    integ-test:
+      test-configs:
+        - with-security
+        - without-security
+
+  - name: dashboards-reports
+    working-directory: reports-scheduler
+    integ-test:
+      test-configs:
+        - without-security
+
+  - name: observability
+    working-directory: opensearch-observability
+    integ-test:
+      test-configs:
+        - without-security
+
+  - name: ml-commons
+    integ-test:
+      test-configs:
+        - with-security
+        - without-security
diff --git a/manifests/1.3.6/opensearch-1.3.6.yml b/manifests/1.3.6/opensearch-1.3.6.yml
@@ -87,6 +87,7 @@ components:
     platforms:
       - linux
       - windows
+<<<<<<< HEAD
   - name: sql
     repository: https://github.com/opensearch-project/sql.git
     ref: '2.x'
@@ -96,3 +97,58 @@ components:
     checks:
       - gradle:properties:version
       - gradle:dependencies:opensearch.version: opensearch-sql-plugin
+=======
+  - name: k-NN
+    repository: https://github.com/opensearch-project/k-NN.git
+    ref: '1.3'
+    checks:
+      - gradle:properties:version
+      - gradle:dependencies:opensearch.version
+    platforms:
+      - linux
+      - windows
+  - name: asynchronous-search
+    repository: https://github.com/opensearch-project/asynchronous-search.git
+    ref: '1.3'
+    checks:
+      - gradle:properties:version
+      - gradle:dependencies:opensearch.version
+    platforms:
+      - linux
+      - windows
+  - name: anomaly-detection
+    repository: https://github.com/opensearch-project/anomaly-detection.git
+    ref: '1.3'
+    checks:
+      - gradle:properties:version
+      - gradle:dependencies:opensearch.version
+    platforms:
+      - linux
+      - windows
+  - name: index-management
+    repository: https://github.com/opensearch-project/index-management.git
+    ref: '1.3'
+    checks:
+      - gradle:properties:version
+      - gradle:dependencies:opensearch.version
+    platforms:
+      - linux
+      - windows
+  - name: cross-cluster-replication
+    repository: https://github.com/opensearch-project/cross-cluster-replication.git
+    ref: '1.3'
+    checks:
+      - gradle:properties:version
+      - gradle:dependencies:opensearch.version
+    platforms:
+      - linux
+      - windows
+  - name: sql
+    repository: https://github.com/opensearch-project/sql.git
+    ref: '1.3'
+    checks:
+      - gradle:properties:version
+      - gradle:dependencies:opensearch.version: plugin
+    platforms:
+      - linux
+      - windows