From 8babdde32e2262b9e2de3e3672daa198bef74c4f Mon Sep 17 00:00:00 2001 From: dblock Date: Wed, 9 Mar 2022 18:18:30 +0000 Subject: [PATCH] Added version qualifier support for OpenSearch. Signed-off-by: dblock --- manifests/2.0.0/opensearch-2.0.0.yml | 1 + scripts/components/OpenSearch/build.sh | 28 ++++++----- scripts/default/install.sh | 5 +- .../default/opensearch-dashboards/build.sh | 5 +- scripts/default/opensearch/build.sh | 5 +- src/build_workflow/build_target.py | 34 ++++++++++--- src/build_workflow/builder_from_source.py | 1 + src/manifests/input_manifest.py | 3 ++ src/run_build.py | 1 + .../tests_build_workflow/test_build_target.py | 48 +++++++++++++++++-- .../test_builder_from_source.py | 24 ++++++++++ 11 files changed, 130 insertions(+), 25 deletions(-) diff --git a/manifests/2.0.0/opensearch-2.0.0.yml b/manifests/2.0.0/opensearch-2.0.0.yml index 316eca4456..faa3753971 100644 --- a/manifests/2.0.0/opensearch-2.0.0.yml +++ b/manifests/2.0.0/opensearch-2.0.0.yml @@ -7,6 +7,7 @@ ci: build: name: OpenSearch version: 2.0.0 + qualifier: alpha1 components: - name: OpenSearch ref: main diff --git a/scripts/components/OpenSearch/build.sh b/scripts/components/OpenSearch/build.sh index 039d1d1fee..7cd340d1a1 100755 --- a/scripts/components/OpenSearch/build.sh +++ b/scripts/components/OpenSearch/build.sh @@ -13,6 +13,7 @@ function usage() { echo "" echo "Arguments:" echo -e "-v VERSION\t[Required] OpenSearch version." + echo -e "-q QUALIFIER\t[Optional] Version qualifier." echo -e "-s SNAPSHOT\t[Optional] Build a snapshot, default is 'false'." echo -e "-p PLATFORM\t[Optional] Platform, default is 'uname -s'." echo -e "-a ARCHITECTURE\t[Optional] Build architecture, default is 'uname -m'." @@ -21,7 +22,7 @@ function usage() { echo -e "-h help" } -while getopts ":h:v:s:o:p:a:d:" arg; do +while getopts ":h:v:q:s:o:p:a:d:" arg; do case $arg in h) usage @@ -30,6 +31,9 @@ while getopts ":h:v:s:o:p:a:d:" arg; do v) VERSION=$OPTARG ;; + q) + QUALIFIER=$OPTARG + ;; s) SNAPSHOT=$OPTARG ;; @@ -68,10 +72,10 @@ fi mkdir -p $OUTPUT/maven/org/opensearch # Build project and publish to maven local. -./gradlew publishToMavenLocal -Dbuild.snapshot=$SNAPSHOT +./gradlew publishToMavenLocal -Dbuild.snapshot=$SNAPSHOT -Dbuild.version_qualifier=$QUALIFIER # Publish to existing test repo, using this to stage release versions of the artifacts that can be released from the same build. -./gradlew publishNebulaPublicationToTestRepository -Dbuild.snapshot=$SNAPSHOT +./gradlew publishNebulaPublicationToTestRepository -Dbuild.snapshot=$SNAPSHOT -Dbuild.version_qualifier=$QUALIFIER # Copy maven publications to be promoted cp -r ./build/local-test-repo/org/opensearch "${OUTPUT}"/maven/org @@ -89,42 +93,42 @@ case $PLATFORM-$DISTRIBUTION-$ARCHITECTURE in EXT="tar.gz" TYPE="archives" TARGET="$PLATFORM-$PACKAGE" - QUALIFIER="$PLATFORM-x64" + SUFFIX="$PLATFORM-x64" ;; linux-tar-arm64|darwin-tar-arm64) PACKAGE="tar" EXT="tar.gz" TYPE="archives" TARGET="$PLATFORM-arm64-$PACKAGE" - QUALIFIER="$PLATFORM-arm64" + SUFFIX="$PLATFORM-arm64" ;; linux-rpm-x64) PACKAGE="rpm" EXT="rpm" TYPE="packages" TARGET="rpm" - QUALIFIER="x86_64" + SUFFIX="x86_64" ;; linux-rpm-arm64) PACKAGE="rpm" EXT="rpm" TYPE="packages" TARGET="arm64-rpm" - QUALIFIER="aarch64" + SUFFIX="aarch64" ;; windows-zip-x64) PACKAGE="zip" EXT="zip" TYPE="archives" TARGET="$PLATFORM-$PACKAGE" - QUALIFIER="$PLATFORM-x64" + SUFFIX="$PLATFORM-x64" ;; windows-zip-arm64) PACKAGE="zip" EXT="zip" TYPE="archives" TARGET="$PLATFORM-arm64-$PACKAGE" - QUALIFIER="$PLATFORM-arm64" + SUFFIX="$PLATFORM-arm64" ;; *) echo "Unsupported platform-distribution-architecture combination: $PLATFORM-$DISTRIBUTION-$ARCHITECTURE" @@ -134,18 +138,18 @@ esac echo "Building OpenSearch for $PLATFORM-$DISTRIBUTION-$ARCHITECTURE" -./gradlew :distribution:$TYPE:$TARGET:assemble -Dbuild.snapshot=$SNAPSHOT +./gradlew :distribution:$TYPE:$TARGET:assemble -Dbuild.snapshot=$SNAPSHOT -Dbuild.version_qualifier=$QUALIFIER # Copy artifact to dist folder in bundle build output [[ "$SNAPSHOT" == "true" ]] && IDENTIFIER="-SNAPSHOT" -ARTIFACT_BUILD_NAME=`ls distribution/$TYPE/$TARGET/build/distributions/ | grep "opensearch-min.*$QUALIFIER.$EXT"` +ARTIFACT_BUILD_NAME=`ls distribution/$TYPE/$TARGET/build/distributions/ | grep "opensearch-min.*$SUFFIX.$EXT"` mkdir -p "${OUTPUT}/dist" cp distribution/$TYPE/$TARGET/build/distributions/$ARTIFACT_BUILD_NAME "${OUTPUT}"/dist/$ARTIFACT_BUILD_NAME echo "Building core plugins..." mkdir -p "${OUTPUT}/core-plugins" cd plugins -../gradlew assemble -Dbuild.snapshot="$SNAPSHOT" +../gradlew assemble -Dbuild.snapshot="$SNAPSHOT" -Dbuild.version_qualifier=$QUALIFIER cd .. for plugin in plugins/*; do PLUGIN_NAME=$(basename "$plugin") diff --git a/scripts/default/install.sh b/scripts/default/install.sh index 4d1a554a97..b3f1c51df3 100755 --- a/scripts/default/install.sh +++ b/scripts/default/install.sh @@ -21,7 +21,7 @@ function usage() { echo -e "-h help" } -while getopts ":h:v:s:o:p:a:f:" arg; do +while getopts ":h:v:q:s:o:p:a:f:" arg; do case $arg in h) usage @@ -30,6 +30,9 @@ while getopts ":h:v:s:o:p:a:f:" arg; do v) VERSION=$OPTARG ;; + q) + QUALIFIER=$OPTARG + ;; s) SNAPSHOT=$OPTARG ;; diff --git a/scripts/default/opensearch-dashboards/build.sh b/scripts/default/opensearch-dashboards/build.sh index 18bf7187d9..31c4ae0996 100755 --- a/scripts/default/opensearch-dashboards/build.sh +++ b/scripts/default/opensearch-dashboards/build.sh @@ -20,7 +20,7 @@ function usage() { echo -e "-h help" } -while getopts ":h:v:s:o:p:a:" arg; do +while getopts ":h:v:q:s:o:p:a:" arg; do case $arg in h) usage @@ -29,6 +29,9 @@ while getopts ":h:v:s:o:p:a:" arg; do v) VERSION=$OPTARG ;; + q) + QUALIFIER=$OPTARG + ;; s) SNAPSHOT=$OPTARG ;; diff --git a/scripts/default/opensearch/build.sh b/scripts/default/opensearch/build.sh index b6292541d3..11007f1361 100755 --- a/scripts/default/opensearch/build.sh +++ b/scripts/default/opensearch/build.sh @@ -20,7 +20,7 @@ function usage() { echo -e "-h help" } -while getopts ":h:v:s:o:p:a:" arg; do +while getopts ":h:v:q:s:o:p:a:" arg; do case $arg in h) usage @@ -29,6 +29,9 @@ while getopts ":h:v:s:o:p:a:" arg; do v) VERSION=$OPTARG ;; + q) + QUALIFIER=$OPTARG + ;; s) SNAPSHOT=$OPTARG ;; diff --git a/src/build_workflow/build_target.py b/src/build_workflow/build_target.py index 7b3bbc4788..9687e44c5e 100644 --- a/src/build_workflow/build_target.py +++ b/src/build_workflow/build_target.py @@ -15,6 +15,7 @@ class BuildTarget: build_id: str name: str version: str + qualifier: str platform: str architecture: str distribution: str @@ -24,6 +25,7 @@ class BuildTarget: def __init__( self, version: str, + qualifier: str = None, patches: List[str] = [], platform: str = None, architecture: str = None, @@ -36,6 +38,7 @@ def __init__( self.build_id = os.getenv("BUILD_NUMBER") or build_id or uuid.uuid4().hex self.name = name self.version = version + self.qualifier = qualifier self.patches = patches self.snapshot = snapshot self.architecture = architecture or current_architecture() @@ -45,27 +48,35 @@ def __init__( @property def opensearch_version(self) -> str: - return self.version + "-SNAPSHOT" if self.snapshot else self.version + return BuildTarget.__qualify_version( + self.version, + self.qualifier, + self.snapshot + ) @property def compatible_opensearch_versions(self) -> List[str]: return ( - [self.version + "-SNAPSHOT" if self.snapshot else self.version] + [BuildTarget.__qualify_version(self.version, self.qualifier, self.snapshot)] + self.patches - + list(map(lambda version: version + "-SNAPSHOT", self.patches)) + + list(map(lambda version: BuildTarget.__qualify_version(version, self.qualifier, True), self.patches)) ) @property def component_version(self) -> str: # BUG: the 4th digit is dictated by the component, it's not .0, this will break for 1.1.0.1 - return self.version + ".0-SNAPSHOT" if self.snapshot else f"{self.version}.0" + return BuildTarget.__qualify_version( + self.version + ".0", + self.qualifier, + self.snapshot + ) @property def compatible_component_versions(self) -> List[str]: return ( - [self.version + ".0-SNAPSHOT" if self.snapshot else f"{self.version}.0"] - + list(map(lambda version: version + ".0", self.patches)) - + list(map(lambda version: version + ".0-SNAPSHOT", self.patches)) + [BuildTarget.__qualify_version(self.version + ".0", self.qualifier, self.snapshot)] + + list(map(lambda version: BuildTarget.__qualify_version(version + ".0", self.qualifier, False), self.patches)) + + list(map(lambda version: BuildTarget.__qualify_version(version + ".0", self.qualifier, True), self.patches)) ) @property @@ -73,3 +84,12 @@ def compatible_versions(self) -> List[str]: versions = [self.version] versions.extend(self.patches) return versions + + @classmethod + def __qualify_version(cls, unqualified_version: str, qualifier: str = None, snapshot: bool = False) -> str: + version = unqualified_version + if qualifier: + version += f"-{qualifier}" + if snapshot: + version += "-SNAPSHOT" + return version diff --git a/src/build_workflow/builder_from_source.py b/src/build_workflow/builder_from_source.py index 7c209655e4..0191eec6d0 100644 --- a/src/build_workflow/builder_from_source.py +++ b/src/build_workflow/builder_from_source.py @@ -42,6 +42,7 @@ def build(self, build_recorder: BuildRecorder) -> None: "bash", build_script, f"-v {self.target.version}", + f"-q {self.target.qualifier}" if self.target.qualifier else None, f"-p {self.target.platform}", f"-a {self.target.architecture}", f"-d {self.target.distribution}" if self.target.distribution and (self.component.name in DISTRIBUTION_SUPPORTED_COMPONENTS) else None, diff --git a/src/manifests/input_manifest.py b/src/manifests/input_manifest.py index 19f4fb688b..102fc41ae3 100644 --- a/src/manifests/input_manifest.py +++ b/src/manifests/input_manifest.py @@ -54,6 +54,7 @@ class InputManifest(ComponentManifest['InputManifest', 'InputComponents']): "schema": { "name": {"required": True, "type": "string"}, "version": {"required": True, "type": "string"}, + "qualifier": {"type": "string"}, "patches": {"type": "list", "schema": {"type": "string"}}, "platform": {"type": "string"}, "architecture": {"type": "string"}, @@ -143,6 +144,7 @@ class Build: def __init__(self, data: Any): self.name: str = data["name"] self.version = data["version"] + self.qualifier = data.get("qualifier", None) self.platform = data.get("platform", None) self.architecture = data.get("architecture", None) self.snapshot = data.get("snapshot", None) @@ -152,6 +154,7 @@ def __to_dict__(self) -> dict: return { "name": self.name, "version": self.version, + "qualifier": self.qualifier, "patches": self.patches, "platform": self.platform, "architecture": self.architecture, diff --git a/src/run_build.py b/src/run_build.py index d221e61f95..bdbb0dcc44 100755 --- a/src/run_build.py +++ b/src/run_build.py @@ -46,6 +46,7 @@ def main(): target = BuildTarget( name=manifest.build.name, version=manifest.build.version, + qualifier=manifest.build.qualifier, patches=manifest.build.patches, snapshot=args.snapshot if args.snapshot is not None else manifest.build.snapshot, output_dir=output_dir, diff --git a/tests/tests_build_workflow/test_build_target.py b/tests/tests_build_workflow/test_build_target.py index 401bef1740..0d166a5c41 100644 --- a/tests/tests_build_workflow/test_build_target.py +++ b/tests/tests_build_workflow/test_build_target.py @@ -34,22 +34,46 @@ def test_opensearch_version(self) -> None: "1.1.0", ) + def test_opensearch_version_snapshot(self) -> None: + self.assertEqual( + BuildTarget(version="1.1.0", architecture="x86", snapshot=True).opensearch_version, + "1.1.0-SNAPSHOT", + ) + + def test_opensearch_version_qualifier(self) -> None: + self.assertEqual( + BuildTarget(version="1.1.0", architecture="x86", snapshot=False, qualifier="alpha1").opensearch_version, + "1.1.0-alpha1", + ) + + def test_opensearch_version_snapshot_qualifier(self) -> None: + self.assertEqual( + BuildTarget(version="1.1.0", architecture="x86", snapshot=True, qualifier="alpha1").opensearch_version, + "1.1.0-alpha1-SNAPSHOT", + ) + def test_compatible_opensearch_versions(self) -> None: self.assertEqual( BuildTarget(version="1.1.2", architecture="x86", patches=["1.1.0", "1.1.1"], snapshot=False).compatible_opensearch_versions, ['1.1.2', '1.1.0', '1.1.1', '1.1.0-SNAPSHOT', '1.1.1-SNAPSHOT'], ) + def test_compatible_opensearch_versions_qualifier(self) -> None: + self.assertEqual( + BuildTarget(version="1.1.2", architecture="x86", patches=["1.1.0", "1.1.1"], snapshot=False, qualifier="alpha1").compatible_opensearch_versions, + ['1.1.2-alpha1', '1.1.0', '1.1.1', '1.1.0-alpha1-SNAPSHOT', '1.1.1-alpha1-SNAPSHOT'], + ) + def test_compatible_opensearch_versions_snapshot(self) -> None: self.assertEqual( BuildTarget(version="1.1.2", architecture="x86", patches=["1.1.0", "1.1.1"], snapshot=True).compatible_opensearch_versions, ['1.1.2-SNAPSHOT', '1.1.0', '1.1.1', '1.1.0-SNAPSHOT', '1.1.1-SNAPSHOT'], ) - def test_opensearch_version_snapshot(self) -> None: + def test_compatible_opensearch_versions_snapshot_qualifier(self) -> None: self.assertEqual( - BuildTarget(version="1.1.0", architecture="x86", snapshot=True).opensearch_version, - "1.1.0-SNAPSHOT", + BuildTarget(version="1.1.2", architecture="x86", patches=["1.1.0", "1.1.1"], snapshot=True, qualifier="alpha1").compatible_opensearch_versions, + ['1.1.2-alpha1-SNAPSHOT', '1.1.0', '1.1.1', '1.1.0-alpha1-SNAPSHOT', '1.1.1-alpha1-SNAPSHOT'], ) def test_component_version(self) -> None: @@ -58,18 +82,36 @@ def test_component_version(self) -> None: "1.1.0.0", ) + def test_component_version_qualifier(self) -> None: + self.assertEqual( + BuildTarget(version="1.1.0", architecture="x86", qualifier="alpha1", snapshot=False).component_version, + "1.1.0.0-alpha1", + ) + def test_compatible_component_versions(self) -> None: self.assertEqual( BuildTarget(version="1.1.2", architecture="x86", patches=["1.1.0", "1.1.1"], snapshot=False).compatible_component_versions, ['1.1.2.0', '1.1.0.0', '1.1.1.0', '1.1.0.0-SNAPSHOT', '1.1.1.0-SNAPSHOT'], ) + def test_compatible_component_versions_qualifier(self) -> None: + self.assertEqual( + BuildTarget(version="1.1.2", architecture="x86", patches=["1.1.0", "1.1.1"], snapshot=False, qualifier="alpha1").compatible_component_versions, + ['1.1.2.0-alpha1', '1.1.0.0-alpha1', '1.1.1.0-alpha1', '1.1.0.0-alpha1-SNAPSHOT', '1.1.1.0-alpha1-SNAPSHOT'], + ) + def test_compatible_component_versions_snapshot(self) -> None: self.assertEqual( BuildTarget(version="1.1.2", architecture="x86", patches=["1.1.0", "1.1.1"], snapshot=True).compatible_component_versions, ['1.1.2.0-SNAPSHOT', '1.1.0.0', '1.1.1.0', '1.1.0.0-SNAPSHOT', '1.1.1.0-SNAPSHOT'], ) + def test_compatible_component_versions_snapshot_qualifier(self) -> None: + self.assertEqual( + BuildTarget(version="1.1.2", architecture="x86", patches=["1.1.0", "1.1.1"], snapshot=True, qualifier="alpha1").compatible_component_versions, + ['1.1.2.0-alpha1-SNAPSHOT', '1.1.0.0-alpha1', '1.1.1.0-alpha1', '1.1.0.0-alpha1-SNAPSHOT', '1.1.1.0-alpha1-SNAPSHOT'], + ) + def test_component_version_snapshot(self) -> None: self.assertEqual( BuildTarget(version="1.1.0", architecture="x86", snapshot=True).component_version, diff --git a/tests/tests_build_workflow/test_builder_from_source.py b/tests/tests_build_workflow/test_builder_from_source.py index 76f4bb3f36..7bee93560d 100644 --- a/tests/tests_build_workflow/test_builder_from_source.py +++ b/tests/tests_build_workflow/test_builder_from_source.py @@ -141,6 +141,30 @@ def test_build_snapshot(self, mock_git_repo: Mock) -> None: ) build_recorder.record_component.assert_called_with("common-utils", self.builder.git_repo) + @patch("build_workflow.builder_from_source.GitRepository") + def test_build_snapshot_qualiier(self, mock_git_repo: Mock) -> None: + self.builder.target.snapshot = True + self.builder.target.qualifier = "alpha1" + mock_git_repo.return_value = MagicMock(working_directory="dir") + build_recorder = MagicMock() + self.builder.checkout("dir") + self.builder.build(build_recorder) + mock_git_repo.return_value.execute.assert_called_with( + " ".join( + [ + "bash", + os.path.realpath(os.path.join(ScriptFinder.component_scripts_path, "common-utils", "build.sh")), + "-v 1.1.0", + "-q alpha1", + "-p linux", + "-a x64", + "-s true", + "-o builds", + ] + ) + ) + build_recorder.record_component.assert_called_with("common-utils", self.builder.git_repo) + def mock_os_walk(self, artifact_path: str) -> List[Any]: if artifact_path.endswith(os.path.join("dir", "builds", "core-plugins")): return [["core-plugins", [], ["plugin1.zip"]]]