From 7506194bb7deabd39d5955ea88174e1dd3a1a4bf Mon Sep 17 00:00:00 2001 From: Peter Zhu Date: Fri, 27 Jan 2023 17:39:17 -0500 Subject: [PATCH] Add distribution selector for distribution builds on Jenkins (#3149) * Add distribution selector for distribution builds on Jenkins Signed-off-by: Peter Zhu * Add check for build entry to python lib Signed-off-by: Peter Zhu * Add check for build entry to python lib Signed-off-by: Peter Zhu --------- Signed-off-by: Peter Zhu --- jenkins/check-for-build.jenkinsfile | 22 ++- .../distribution-build.jenkinsfile | 119 ++++++++++--- .../opensearch/distribution-build.jenkinsfile | 163 ++++++++++++++---- src/manifests_workflow/input_manifests.py | 8 +- .../test_input_manifests.py | 2 +- 5 files changed, 246 insertions(+), 68 deletions(-) diff --git a/jenkins/check-for-build.jenkinsfile b/jenkins/check-for-build.jenkinsfile index 59771850af..7d9c90f83f 100644 --- a/jenkins/check-for-build.jenkinsfile +++ b/jenkins/check-for-build.jenkinsfile @@ -14,14 +14,14 @@ pipeline { } triggers { parameterizedCron ''' - H 1 * * * %INPUT_MANIFEST=2.6.0/opensearch-2.6.0.yml;TARGET_JOB_NAME=distribution-build-opensearch;BUILD_PLATFORM=linux windows - H 1 * * * %INPUT_MANIFEST=2.6.0/opensearch-dashboards-2.6.0.yml;TARGET_JOB_NAME=distribution-build-opensearch-dashboards;BUILD_PLATFORM=linux windows - H 1 * * * %INPUT_MANIFEST=2.4.2/opensearch-dashboards-2.4.2.yml;TARGET_JOB_NAME=distribution-build-opensearch-dashboards;BUILD_PLATFORM=linux windows - H 1 * * * %INPUT_MANIFEST=2.4.2/opensearch-2.4.2.yml;TARGET_JOB_NAME=distribution-build-opensearch;BUILD_PLATFORM=linux macos windows - H 1 * * * %INPUT_MANIFEST=1.4.0/opensearch-1.4.0.yml;TARGET_JOB_NAME=distribution-build-opensearch;BUILD_PLATFORM=linux macos windows - H 1 * * * %INPUT_MANIFEST=1.4.0/opensearch-dashboards-1.4.0.yml;TARGET_JOB_NAME=distribution-build-opensearch-dashboards;BUILD_PLATFORM=linux windows - H 1 * * * %INPUT_MANIFEST=3.0.0/opensearch-3.0.0.yml;TARGET_JOB_NAME=distribution-build-opensearch;BUILD_PLATFORM=linux macos windows - H 1 * * * %INPUT_MANIFEST=3.0.0/opensearch-dashboards-3.0.0.yml;TARGET_JOB_NAME=distribution-build-opensearch-dashboards;BUILD_PLATFORM=linux windows + H 1 * * * %INPUT_MANIFEST=2.6.0/opensearch-2.6.0.yml;TARGET_JOB_NAME=distribution-build-opensearch;BUILD_PLATFORM=linux windows;BUILD_DISTRIBUTION=tar rpm deb zip + H 1 * * * %INPUT_MANIFEST=2.6.0/opensearch-dashboards-2.6.0.yml;TARGET_JOB_NAME=distribution-build-opensearch-dashboards;BUILD_PLATFORM=linux windows;BUILD_DISTRIBUTION=tar rpm deb zip + H 1 * * * %INPUT_MANIFEST=2.4.2/opensearch-dashboards-2.4.2.yml;TARGET_JOB_NAME=distribution-build-opensearch-dashboards;BUILD_PLATFORM=linux windows;BUILD_DISTRIBUTION=tar rpm deb zip + H 1 * * * %INPUT_MANIFEST=2.4.2/opensearch-2.4.2.yml;TARGET_JOB_NAME=distribution-build-opensearch;BUILD_PLATFORM=linux macos windows;BUILD_DISTRIBUTION=tar rpm deb zip + H 1 * * * %INPUT_MANIFEST=1.4.0/opensearch-1.4.0.yml;TARGET_JOB_NAME=distribution-build-opensearch;BUILD_PLATFORM=linux macos windows;BUILD_DISTRIBUTION=tar rpm zip + H 1 * * * %INPUT_MANIFEST=1.4.0/opensearch-dashboards-1.4.0.yml;TARGET_JOB_NAME=distribution-build-opensearch-dashboards;BUILD_PLATFORM=linux windows;BUILD_DISTRIBUTION=tar rpm zip + H 1 * * * %INPUT_MANIFEST=3.0.0/opensearch-3.0.0.yml;TARGET_JOB_NAME=distribution-build-opensearch;BUILD_PLATFORM=linux macos windows;BUILD_DISTRIBUTION=tar rpm deb zip + H 1 * * * %INPUT_MANIFEST=3.0.0/opensearch-dashboards-3.0.0.yml;TARGET_JOB_NAME=distribution-build-opensearch-dashboards;BUILD_PLATFORM=linux windows;BUILD_DISTRIBUTION=tar rpm deb zip ''' } parameters { @@ -45,6 +45,11 @@ pipeline { description: 'Platform to build', trim: true ) + string( + name: 'BUILD_DISTRIBUTION', + description: 'Distribution to build', + trim: true + ) } stages { stage('detect docker image + args') { @@ -88,6 +93,7 @@ pipeline { string(name: 'INPUT_MANIFEST', value: "${INPUT_MANIFEST}"), string(name: 'TEST_MANIFEST', value: "${TEST_MANIFEST}"), string(name: 'BUILD_PLATFORM', value: "${BUILD_PLATFORM}") + string(name: 'BUILD_DISTRIBUTION', value: "${BUILD_DISTRIBUTION}") ], wait: true echo "Build succeeded, uploading build SHA for that job" diff --git a/jenkins/opensearch-dashboards/distribution-build.jenkinsfile b/jenkins/opensearch-dashboards/distribution-build.jenkinsfile index ff44774e2d..7cc12b31ee 100644 --- a/jenkins/opensearch-dashboards/distribution-build.jenkinsfile +++ b/jenkins/opensearch-dashboards/distribution-build.jenkinsfile @@ -44,10 +44,16 @@ pipeline { ) string( // Note: need to update 'verify-parameters' entries if you add new platform(s) name: 'BUILD_PLATFORM', - description: "Build selected platform related artifacts, choices include 'linux windows'. Can combine multiple platforms with space in between", + description: "Build selected platform related artifacts, choices include 'linux windows'. Can combine multiple platforms with space in between (docker is only available on linux)", defaultValue: 'linux windows', trim: true ) + string( // Note: need to update 'verify-parameters' entries if you add new distribution(s) + name: 'BUILD_DISTRIBUTION', + description: "Build selected distribution related artifacts, choices include 'tar', 'rpm', 'deb', 'zip'. Can combine multiple distributions with space in between (docker is only available on tar)", + defaultValue: 'tar rpm deb zip', + trim: true + ) choice( name: 'BUILD_DOCKER', description: 'Build docker image or not with options.', @@ -101,6 +107,25 @@ pipeline { error("Missing parameter: BUILD_PLATFORM (possible entries: ${all_platforms}).") } } + + echo("Verify Build Distributions") + def build_distribution_array = params.BUILD_DISTRIBUTION.tokenize(' ') + echo("User Entry Distributions: '${params.BUILD_DISTRIBUTION}', ${build_distribution_array}") + def all_distributions = "tar rpm deb zip" + echo("All Supported Platforms: '${all_distributions}'") + + if (params.BUILD_DISTRIBUTION == null || params.BUILD_DISTRIBUTION == '') { + currentBuild.result = 'ABORTED' + error("Missing parameter: BUILD_DISTRIBUTION (possible entries: ${all_distributions}).") + } + + for (String plat : build_distribution_array) { + if (! all_distributions.contains(plat)) { + currentBuild.result = 'ABORTED' + error("Missing parameter: BUILD_DISTRIBUTION (possible entries: ${all_distributions}).") + } + } + } } } @@ -133,8 +158,13 @@ pipeline { stage('build-and-test-linux-x64-tar') { when { beforeAgent true - expression{ - params.BUILD_PLATFORM.contains('linux') + allOf{ + expression { + params.BUILD_PLATFORM.contains('linux') + } + expression{ + params.BUILD_DISTRIBUTION.contains('tar') + } } } agent { @@ -237,8 +267,13 @@ pipeline { stage('build-and-test-linux-x64-rpm') { when { beforeAgent true - expression{ - params.BUILD_PLATFORM.contains('linux') + allOf{ + expression { + params.BUILD_PLATFORM.contains('linux') + } + expression{ + params.BUILD_DISTRIBUTION.contains('rpm') + } } } agent { label AGENT_X64 } @@ -332,8 +367,13 @@ pipeline { stage('build-and-test-linux-x64-deb') { when { beforeAgent true - expression{ - params.BUILD_PLATFORM.contains('linux') + allOf{ + expression { + params.BUILD_PLATFORM.contains('linux') + } + expression{ + params.BUILD_DISTRIBUTION.contains('deb') + } } } agent { label AGENT_X64 } @@ -412,8 +452,13 @@ pipeline { stage('build-and-test-linux-arm64-tar') { when { beforeAgent true - expression{ - params.BUILD_PLATFORM.contains('linux') + allOf{ + expression { + params.BUILD_PLATFORM.contains('linux') + } + expression{ + params.BUILD_DISTRIBUTION.contains('tar') + } } } agent { label AGENT_X64 } @@ -554,8 +599,13 @@ pipeline { stage('build-and-test-linux-arm64-rpm') { when { beforeAgent true - expression{ - params.BUILD_PLATFORM.contains('linux') + allOf{ + expression { + params.BUILD_PLATFORM.contains('linux') + } + expression{ + params.BUILD_DISTRIBUTION.contains('rpm') + } } } agent { label AGENT_X64 } @@ -654,8 +704,13 @@ pipeline { stage('build-and-test-linux-arm64-deb') { when { beforeAgent true - expression{ - params.BUILD_PLATFORM.contains('linux') + allOf{ + expression { + params.BUILD_PLATFORM.contains('linux') + } + expression{ + params.BUILD_DISTRIBUTION.contains('deb') + } } } agent { label AGENT_ARM64 } @@ -734,8 +789,13 @@ pipeline { stage('build-and-test-windows-x64-zip') { when { beforeAgent true - expression{ - params.BUILD_PLATFORM.contains('windows') + allOf{ + expression { + params.BUILD_PLATFORM.contains('windows') + } + expression{ + params.BUILD_DISTRIBUTION.contains('zip') + } } } agent { @@ -805,6 +865,9 @@ pipeline { expression{ params.BUILD_PLATFORM.contains('linux') } + expression{ + params.BUILD_DISTRIBUTION.contains('tar') + } } } agent { @@ -851,17 +914,25 @@ pipeline { script { List stages = [] if (params.BUILD_PLATFORM.contains('linux')) { - stages = [ - 'build-and-test-linux-x64-tar', - 'assemble-archive-and-test-linux-x64-rpm', - 'assemble-archive-and-test-linux-x64-deb', - 'assemble-archive-and-test-linux-arm64-tar', - 'assemble-archive-and-test-linux-arm64-rpm', - 'assemble-archive-and-test-linux-arm64-deb', - ] + if (params.BUILD_DISTRIBUTION.contains('tar')) { + stages = [ + 'build-and-test-linux-x64-tar', + 'assemble-archive-and-test-linux-arm64-tar', + ] + } + if (params.BUILD_DISTRIBUTION.contains('rpm')) { + stages.add('assemble-archive-and-test-linux-x64-rpm') + stages.add('assemble-archive-and-test-linux-arm64-rpm') + } + if (params.BUILD_DISTRIBUTION.contains('deb')) { + stages.add('assemble-archive-and-test-linux-x64-deb') + stages.add('assemble-archive-and-test-linux-arm64-deb') + } } if (params.BUILD_PLATFORM.contains('windows')){ - stages.add('build-and-test-windows-x64-zip') + if (params.BUILD_DISTRIBUTION.contains('zip')) { + stages.add('build-and-test-windows-x64-zip') + } } if (params.PUBLISH_NOTIFICATION) { def stashed = lib.jenkins.Messages.new(this).get(stages) diff --git a/jenkins/opensearch/distribution-build.jenkinsfile b/jenkins/opensearch/distribution-build.jenkinsfile index 91b9c5b6af..9ea708504d 100644 --- a/jenkins/opensearch/distribution-build.jenkinsfile +++ b/jenkins/opensearch/distribution-build.jenkinsfile @@ -44,10 +44,16 @@ pipeline { ) string( // Note: need to update 'verify-parameters' entries if you add new platform(s) name: 'BUILD_PLATFORM', - description: "Build selected platform related artifacts, choices include 'linux', 'macos', 'windows'. Can combine multiple platforms with space in between (maven snapshot is only on linux)", + description: "Build selected platform related artifacts, choices include 'linux', 'macos', 'windows'. Can combine multiple platforms with space in between (maven snapshot/docker is only available on linux)", defaultValue: 'linux macos windows', trim: true ) + string( // Note: need to update 'verify-parameters' entries if you add new distribution(s) + name: 'BUILD_DISTRIBUTION', + description: "Build selected distribution related artifacts, choices include 'tar', 'rpm', 'deb', 'zip'. Can combine multiple distributions with space in between (docker is only available on tar)", + defaultValue: 'tar rpm deb zip', + trim: true + ) choice( name: 'BUILD_DOCKER', description: 'Build docker image or not with options.', @@ -101,6 +107,24 @@ pipeline { error("Missing parameter: BUILD_PLATFORM (possible entries: ${all_platforms}).") } } + + echo("Verify Build Distributions") + def build_distribution_array = params.BUILD_DISTRIBUTION.tokenize(' ') + echo("User Entry Distributions: '${params.BUILD_DISTRIBUTION}', ${build_distribution_array}") + def all_distributions = "tar rpm deb zip" + echo("All Supported Platforms: '${all_distributions}'") + + if (params.BUILD_DISTRIBUTION == null || params.BUILD_DISTRIBUTION == '') { + currentBuild.result = 'ABORTED' + error("Missing parameter: BUILD_DISTRIBUTION (possible entries: ${all_distributions}).") + } + + for (String plat : build_distribution_array) { + if (! all_distributions.contains(plat)) { + currentBuild.result = 'ABORTED' + error("Missing parameter: BUILD_DISTRIBUTION (possible entries: ${all_distributions}).") + } + } } } } @@ -133,8 +157,13 @@ pipeline { stage('build-distribution-snapshot-linux-x64-tar') { when { beforeAgent true - expression{ - params.BUILD_PLATFORM.contains('linux') + allOf{ + expression { + params.BUILD_PLATFORM.contains('linux') + } + expression{ + params.BUILD_DISTRIBUTION.contains('tar') + } } } environment { @@ -179,8 +208,13 @@ pipeline { stage('build-opensearch-snapshot-linux-x64-tar') { when { beforeAgent true - expression{ - params.BUILD_PLATFORM.contains('linux') + allOf{ + expression { + params.BUILD_PLATFORM.contains('linux') + } + expression{ + params.BUILD_DISTRIBUTION.contains('tar') + } } } environment { @@ -220,8 +254,13 @@ pipeline { stage('build-opensearch-snapshot-linux-arm64-tar') { when { beforeAgent true - expression{ - params.BUILD_PLATFORM.contains('linux') + allOf{ + expression { + params.BUILD_PLATFORM.contains('linux') + } + expression{ + params.BUILD_DISTRIBUTION.contains('tar') + } } } agent { @@ -258,8 +297,13 @@ pipeline { stage('build-opensearch-snapshot-macos-x64-tar') { when { beforeAgent true - expression{ - params.BUILD_PLATFORM.contains('macos') + allOf{ + expression { + params.BUILD_PLATFORM.contains('macos') + } + expression{ + params.BUILD_DISTRIBUTION.contains('tar') + } } } agent { @@ -296,8 +340,13 @@ pipeline { stage('build-opensearch-snapshot-windows-x64-zip') { when { beforeAgent true - expression{ - params.BUILD_PLATFORM.contains('windows') + allOf{ + expression { + params.BUILD_PLATFORM.contains('windows') + } + expression{ + params.BUILD_DISTRIBUTION.contains('zip') + } } } agent { @@ -334,8 +383,13 @@ pipeline { stage('build-and-test-linux-x64-tar') { when { beforeAgent true - expression{ - params.BUILD_PLATFORM.contains('linux') + allOf{ + expression { + params.BUILD_PLATFORM.contains('linux') + } + expression{ + params.BUILD_DISTRIBUTION.contains('tar') + } } } agent { @@ -439,8 +493,13 @@ pipeline { stage('build-and-test-linux-x64-rpm') { when { beforeAgent true - expression{ - params.BUILD_PLATFORM.contains('linux') + allOf{ + expression { + params.BUILD_PLATFORM.contains('linux') + } + expression{ + params.BUILD_DISTRIBUTION.contains('rpm') + } } } agent { label AGENT_X64 } @@ -535,8 +594,13 @@ pipeline { stage('build-and-test-linux-x64-deb') { when { beforeAgent true - expression{ - params.BUILD_PLATFORM.contains('linux') + allOf{ + expression { + params.BUILD_PLATFORM.contains('linux') + } + expression{ + params.BUILD_DISTRIBUTION.contains('deb') + } } } agent { label AGENT_X64 } @@ -615,8 +679,13 @@ pipeline { stage('build-and-test-linux-arm64-tar') { when { beforeAgent true - expression{ - params.BUILD_PLATFORM.contains('linux') + allOf{ + expression { + params.BUILD_PLATFORM.contains('linux') + } + expression{ + params.BUILD_DISTRIBUTION.contains('tar') + } } } agent { @@ -719,8 +788,13 @@ pipeline { stage('build-and-test-linux-arm64-rpm') { when { beforeAgent true - expression{ - params.BUILD_PLATFORM.contains('linux') + allOf{ + expression { + params.BUILD_PLATFORM.contains('linux') + } + expression{ + params.BUILD_DISTRIBUTION.contains('rpm') + } } } agent { label AGENT_X64 } @@ -815,8 +889,13 @@ pipeline { stage('build-and-test-linux-arm64-deb') { when { beforeAgent true - expression{ - params.BUILD_PLATFORM.contains('linux') + allOf{ + expression { + params.BUILD_PLATFORM.contains('linux') + } + expression{ + params.BUILD_DISTRIBUTION.contains('deb') + } } } agent { label AGENT_ARM64 } @@ -895,8 +974,13 @@ pipeline { stage('build-and-test-windows-x64-zip') { when { beforeAgent true - expression{ - params.BUILD_PLATFORM.contains('windows') + allOf{ + expression { + params.BUILD_PLATFORM.contains('windows') + } + expression{ + params.BUILD_DISTRIBUTION.contains('zip') + } } } agent { @@ -970,6 +1054,9 @@ pipeline { expression{ params.BUILD_PLATFORM.contains('linux') } + expression{ + params.BUILD_DISTRIBUTION.contains('tar') + } } } agent { @@ -1016,17 +1103,25 @@ pipeline { script { List stages = [] if (params.BUILD_PLATFORM.contains('linux')) { - stages = [ - 'build-and-test-linux-x64-tar', - 'assemble-archive-and-test-linux-x64-rpm', - 'assemble-archive-and-test-linux-x64-deb', - 'build-and-test-linux-arm64-tar', - 'assemble-archive-and-test-linux-arm64-rpm', - 'assemble-archive-and-test-linux-arm64-deb', - ] + if (params.BUILD_DISTRIBUTION.contains('tar')) { + stages = [ + 'build-and-test-linux-x64-tar', + 'build-and-test-linux-arm64-tar', + ] + } + if (params.BUILD_DISTRIBUTION.contains('rpm')) { + stages.add('assemble-archive-and-test-linux-x64-rpm') + stages.add('assemble-archive-and-test-linux-arm64-rpm') + } + if (params.BUILD_DISTRIBUTION.contains('deb')) { + stages.add('assemble-archive-and-test-linux-x64-deb') + stages.add('assemble-archive-and-test-linux-arm64-deb') + } } if (params.BUILD_PLATFORM.contains('windows')){ - stages.add('build-and-test-windows-x64-zip') + if (params.BUILD_DISTRIBUTION.contains('zip')) { + stages.add('build-and-test-windows-x64-zip') + } } if (params.PUBLISH_NOTIFICATION) { diff --git a/src/manifests_workflow/input_manifests.py b/src/manifests_workflow/input_manifests.py index 853c27105a..cf8c58fd08 100644 --- a/src/manifests_workflow/input_manifests.py +++ b/src/manifests_workflow/input_manifests.py @@ -28,6 +28,10 @@ class InputManifests(Manifests): "opensearch": "linux macos windows", "opensearch-dashboards": "linux windows" } + BUILD_DISTRIBUTION = { + "opensearch": "tar rpm deb zip", + "opensearch-dashboards": "tar rpm deb zip" + } def __init__(self, name: str) -> None: self.name = name @@ -170,10 +174,12 @@ def add_to_cron(self, version: str) -> None: data = f.read() build_platform = self.BUILD_PLATFORM.get(self.prefix, "linux") + build_distribution = self.BUILD_DISTRIBUTION.get(self.prefix, "tar") cron_entry = f"H 1 * * * %INPUT_MANIFEST={version}/{self.prefix}-{version}.yml;" \ f"TARGET_JOB_NAME=distribution-build-{self.prefix};" \ - f"BUILD_PLATFORM={build_platform}\n" + f"BUILD_PLATFORM={build_platform};" \ + f"BUILD_DISTRIBUTION={build_distribution}\n" if cron_entry in data: raise ValueError(f"{jenkinsfile} already contains an entry for {self.prefix} {version}") diff --git a/tests/tests_manifests_workflow/test_input_manifests.py b/tests/tests_manifests_workflow/test_input_manifests.py index b985ac4d04..cac9501db0 100644 --- a/tests/tests_manifests_workflow/test_input_manifests.py +++ b/tests/tests_manifests_workflow/test_input_manifests.py @@ -126,7 +126,7 @@ def test_add_to_cron(self, mock_open: MagicMock) -> None: mock_open.assert_has_calls([call(InputManifests.cron_jenkinsfile(), 'r')]) mock_open().write.assert_called_once_with( f"parameterizedCron '''\n{' ' * 12}H 1 * * * %INPUT_MANIFEST=0.1.2/test-0.1.2.yml;" - f"TARGET_JOB_NAME=distribution-build-test;BUILD_PLATFORM=linux\n" + f"TARGET_JOB_NAME=distribution-build-test;BUILD_PLATFORM=linux;BUILD_DISTRIBUTION=tar\n" ) def test_versionincrement_workflow(self) -> None: