From 923c161c25bac363f6959e8b53eb1965d2f64d33 Mon Sep 17 00:00:00 2001 From: Peter Zhu Date: Tue, 15 Feb 2022 22:43:19 +0000 Subject: [PATCH] Revert tests and also add changes based on comments Signed-off-by: Peter Zhu --- scripts/components/OpenSearch/build.sh | 120 +++++++----------- src/build_workflow/build_recorder.py | 2 +- src/build_workflow/builder_from_source.py | 21 +-- tests/data/opensearch-build-1.1.0.yml | 1 - .../opensearch-dashboards-build-1.2.0.yml | 1 - .../data/opensearch-build-linux-1.1.0.yml | 1 - .../data/opensearch-build-linux-1.1.1.yml | 1 - .../data/opensearch-build-windows-1.1.0.yml | 1 - .../opensearch-dashboards-build-1.1.0.yml | 1 - tests/tests_assemble_workflow/test_bundles.py | 3 +- .../data/opensearch-build-windows-1.1.0.yml | 1 - .../test_build_recorder.py | 2 - .../opensearch-build-schema-version-1.2.yml | 1 - .../data/opensearch-build-1.1.0.yml | 1 - .../data/opensearch-build-1.2.0.yml | 1 - .../data/opensearch-build-1.1.0.yml | 1 - .../data/local/builds/opensearch/manifest.yml | 1 - .../remote/builds/opensearch/manifest.yml | 1 - .../bwc_test/data/build_manifest.yml | 1 - .../integ_test/data/build_manifest.yml | 1 - .../build_manifest_missing_components.yml | 1 - 21 files changed, 52 insertions(+), 112 deletions(-) diff --git a/scripts/components/OpenSearch/build.sh b/scripts/components/OpenSearch/build.sh index a12e6a8c9f..939fcae142 100755 --- a/scripts/components/OpenSearch/build.sh +++ b/scripts/components/OpenSearch/build.sh @@ -16,7 +16,7 @@ function usage() { echo -e "-s SNAPSHOT\t[Optional] Build a snapshot, default is 'false'." echo -e "-p PLATFORM\t[Optional] Platform, default is 'uname -s'." echo -e "-a ARCHITECTURE\t[Optional] Build architecture, default is 'uname -m'." - echo -e "-d DISTRIBUTION\t[Optional] Distribution, default is 'None'." + echo -e "-d DISTRIBUTION\t[Optional] Distribution, default is 'tar'." echo -e "-o OUTPUT\t[Optional] Output path, default is 'artifacts'." echo -e "-h help" } @@ -81,93 +81,59 @@ cp -r ./build/local-test-repo/org/opensearch "${OUTPUT}"/maven/org [ -z "$PLATFORM" ] && PLATFORM=$(uname -s | awk '{print tolower($0)}') [ -z "$ARCHITECTURE" ] && ARCHITECTURE=`uname -m` -[ -z "$DISTRIBUTION" ] && DISTRIBUTION="None" - -case $PLATFORM in - linux*) - case $DISTRIBUTION in - tar|None) - PACKAGE="tar" - EXT="tar.gz" - TYPE="archives" - ;; - rpm) - PACKAGE="rpm" - EXT="rpm" - TYPE="packages" - ;; - *) - echo "Unsupported platform and distribution combo: $PLATFORM-$DISTRIBUTION" - exit 1 - ;; - esac +[ -z "$DISTRIBUTION" ] && DISTRIBUTION="tar" + +case $PLATFORM-$DISTRIBUTION-$ARCHITECTURE in + linux-tar-x64|darwin-tar-x64) + PACKAGE="tar" + EXT="tar.gz" + TYPE="archives" + TARGET="$PLATFORM-$PACKAGE" + QUALIFIER="$PLATFORM-x64" ;; - - darwin*) - case $DISTRIBUTION in - tar|None) - PACKAGE="tar" - EXT="tar.gz" - TYPE="archives" - ;; - *) - echo "Unsupported platform and distribution combo: $PLATFORM-$DISTRIBUTION" - exit 1 - ;; - esac + linux-tar-arm64|darwin-tar-arm64) + PACKAGE="tar" + EXT="tar.gz" + TYPE="archives" + TARGET="$PLATFORM-arm64-$PACKAGE" + QUALIFIER="$PLATFORM-arm64" ;; - - windows*) - case $DISTRIBUTION in - zip|None) - PACKAGE="zip" - EXT="zip" - TYPE="archives" - ;; - *) - echo "Unsupported platform and distribution combo: $PLATFORM-$DISTRIBUTION" - exit 1 - ;; - esac + linux-rpm-x64) + PACKAGE="rpm" + EXT="rpm" + TYPE="packages" + TARGET="rpm" + QUALIFIER="x86_64" ;; - *) - echo "Unsupported platform: $PLATFORM" - exit 1 + linux-rpm-arm64) + PACKAGE="rpm" + EXT="rpm" + TYPE="packages" + TARGET="arm64-rpm" + QUALIFIER="aarch64" ;; -esac - -case $ARCHITECTURE in - x64) - case $DISTRIBUTION in - rpm) - TARGET="$DISTRIBUTION" - QUALIFIER="x86_64" - ;; - *) - TARGET="$PLATFORM-$PACKAGE" - QUALIFIER="$PLATFORM-x64" - ;; - esac + windows-zip-x64) + PACKAGE="zip" + EXT="zip" + TYPE="archives" + TARGET="$PLATFORM-$PACKAGE" + QUALIFIER="$PLATFORM-x64" ;; - - arm64) - case $DISTRIBUTION in - rpm) - TARGET="$DISTRIBUTION" - QUALIFIER="aarch64" - ;; - *) - TARGET="$PLATFORM-arm64-$PACKAGE" - QUALIFIER="$PLATFORM-arm64" - ;; - esac + windows-zip-arm64) + PACKAGE="zip" + EXT="zip" + TYPE="archives" + TARGET="$PLATFORM-arm64-$PACKAGE" + QUALIFIER="$PLATFORM-arm64" ;; *) - echo "Unsupported architecture: $ARCHITECTURE" + echo "Unsupported platform-distribution-architecture combo: $PLATFORM-$DISTRIBUTION-$ARCHITECTURE" exit 1 ;; esac +echo "Building OpenSearch for $PLATFORM-$DISTRIBUTION-$ARCHITECTURE" + ./gradlew :distribution:$TYPE:$TARGET:assemble -Dbuild.snapshot=$SNAPSHOT # Copy artifact to dist folder in bundle build output diff --git a/src/build_workflow/build_recorder.py b/src/build_workflow/build_recorder.py index 2eb6251990..3e6fcee5e5 100644 --- a/src/build_workflow/build_recorder.py +++ b/src/build_workflow/build_recorder.py @@ -60,7 +60,7 @@ def __init__(self, target: BuildTarget): self.data["build"]["version"] = target.opensearch_version self.data["build"]["platform"] = target.platform self.data["build"]["architecture"] = target.architecture - self.data["build"]["distribution"] = target.distribution if target.distribution else "None" + self.data["build"]["distribution"] = target.distribution if target.distribution else "tar" self.data["schema-version"] = "1.2" self.components_hash: Dict[str, Dict[str, Any]] = {} diff --git a/src/build_workflow/builder_from_source.py b/src/build_workflow/builder_from_source.py index 3fe7a99391..54caffcd4a 100644 --- a/src/build_workflow/builder_from_source.py +++ b/src/build_workflow/builder_from_source.py @@ -29,25 +29,18 @@ def checkout(self, work_dir: str) -> None: def build(self, build_recorder: BuildRecorder) -> None: build_script = ScriptFinder.find_build_script(self.target.name, self.component.name, self.git_repo.working_directory) - distribution_parameter = "-d" + " " + self.target.distribution if self.target.distribution else None build_command = " ".join( - filter( - None, + filter(None, [ "bash", build_script, - "-v", - self.target.version, - "-p", - self.target.platform, - "-a", - self.target.architecture, - distribution_parameter, - "-s", - str(self.target.snapshot).lower(), - "-o", - self.output_path, + f"-v {self.target.version}", + f"-p {self.target.platform}", + f"-a {self.target.architecture}", + f"-d {self.target.distribution}" if self.target.distribution else None, + f"-s {str(self.target.snapshot).lower()}", + f"-o {self.output_path}", ] ) ) diff --git a/tests/data/opensearch-build-1.1.0.yml b/tests/data/opensearch-build-1.1.0.yml index b9fe5465f3..eaf25f9a81 100644 --- a/tests/data/opensearch-build-1.1.0.yml +++ b/tests/data/opensearch-build-1.1.0.yml @@ -2,7 +2,6 @@ build: platform: linux architecture: x64 - distribution: None id: c3ff7a232d25403fa8cc14c97799c323 name: OpenSearch version: 1.1.0 diff --git a/tests/data/opensearch-dashboards-build-1.2.0.yml b/tests/data/opensearch-dashboards-build-1.2.0.yml index 9bb35d4592..2cbaf2461e 100644 --- a/tests/data/opensearch-dashboards-build-1.2.0.yml +++ b/tests/data/opensearch-dashboards-build-1.2.0.yml @@ -1,7 +1,6 @@ --- build: architecture: x64 - distribution: None id: 25b38c278cdd45efa583765d8ba76346 name: OpenSearch Dashboards platform: linux diff --git a/tests/tests_assemble_workflow/data/opensearch-build-linux-1.1.0.yml b/tests/tests_assemble_workflow/data/opensearch-build-linux-1.1.0.yml index b9fe5465f3..eaf25f9a81 100644 --- a/tests/tests_assemble_workflow/data/opensearch-build-linux-1.1.0.yml +++ b/tests/tests_assemble_workflow/data/opensearch-build-linux-1.1.0.yml @@ -2,7 +2,6 @@ build: platform: linux architecture: x64 - distribution: None id: c3ff7a232d25403fa8cc14c97799c323 name: OpenSearch version: 1.1.0 diff --git a/tests/tests_assemble_workflow/data/opensearch-build-linux-1.1.1.yml b/tests/tests_assemble_workflow/data/opensearch-build-linux-1.1.1.yml index 0e05c2c990..93ce620d6b 100644 --- a/tests/tests_assemble_workflow/data/opensearch-build-linux-1.1.1.yml +++ b/tests/tests_assemble_workflow/data/opensearch-build-linux-1.1.1.yml @@ -1,7 +1,6 @@ --- build: architecture: x64 - distribution: None id: f49ffcf2c1b341d1b29e147093689c6d name: OpenSearch platform: linux diff --git a/tests/tests_assemble_workflow/data/opensearch-build-windows-1.1.0.yml b/tests/tests_assemble_workflow/data/opensearch-build-windows-1.1.0.yml index 64b8285e34..41a130da36 100644 --- a/tests/tests_assemble_workflow/data/opensearch-build-windows-1.1.0.yml +++ b/tests/tests_assemble_workflow/data/opensearch-build-windows-1.1.0.yml @@ -1,7 +1,6 @@ --- build: platform: windows - distribution: None architecture: x64 id: c3ff7a232d25403fa8cc14c97799c323 name: OpenSearch diff --git a/tests/tests_assemble_workflow/data/opensearch-dashboards-build-1.1.0.yml b/tests/tests_assemble_workflow/data/opensearch-dashboards-build-1.1.0.yml index 0d59e28cd6..ccc51fc6ef 100644 --- a/tests/tests_assemble_workflow/data/opensearch-dashboards-build-1.1.0.yml +++ b/tests/tests_assemble_workflow/data/opensearch-dashboards-build-1.1.0.yml @@ -2,7 +2,6 @@ build: platform: linux architecture: x64 - distribution: None id: c94ebec444a94ada86a230c9297b1d73 name: OpenSearch Dashboards version: 1.1.0 diff --git a/tests/tests_assemble_workflow/test_bundles.py b/tests/tests_assemble_workflow/test_bundles.py index 52ac60a682..4f8ee58bff 100644 --- a/tests/tests_assemble_workflow/test_bundles.py +++ b/tests/tests_assemble_workflow/test_bundles.py @@ -1,4 +1,4 @@ -# SPDX-License-Identifier: Apache-1.0 +# SPDX-License-Identifier: Apache-2.0 # # The OpenSearch Contributors require contributions made to # this file be licensed under the Apache-2.0 license or a @@ -42,7 +42,6 @@ def test_bundle_opensearch_invalid(self) -> None: "name": "invalid", "platform": "linux", "architecture": "x86", - "distribution": "None", "id": "id", "version": "1.0.0", }, diff --git a/tests/tests_build_workflow/data/opensearch-build-windows-1.1.0.yml b/tests/tests_build_workflow/data/opensearch-build-windows-1.1.0.yml index 64b8285e34..41a130da36 100644 --- a/tests/tests_build_workflow/data/opensearch-build-windows-1.1.0.yml +++ b/tests/tests_build_workflow/data/opensearch-build-windows-1.1.0.yml @@ -1,7 +1,6 @@ --- build: platform: windows - distribution: None architecture: x64 id: c3ff7a232d25403fa8cc14c97799c323 name: OpenSearch diff --git a/tests/tests_build_workflow/test_build_recorder.py b/tests/tests_build_workflow/test_build_recorder.py index dc599675af..3d6ad6a7ea 100644 --- a/tests/tests_build_workflow/test_build_recorder.py +++ b/tests/tests_build_workflow/test_build_recorder.py @@ -56,7 +56,6 @@ def test_record_component_and_artifact(self, mock_makedirs: Mock, mock_copyfile: "build": { "platform": "linux", "architecture": "x64", - "distribution": "None", "id": "1", "name": "OpenSearch", "version": "1.1.0", @@ -135,7 +134,6 @@ def test_get_manifest(self) -> None: "build": { "platform": "linux", "architecture": "x64", - "distribution": "None", "id": "1", "name": "OpenSearch", "version": "1.1.0", diff --git a/tests/tests_manifests/data/build/opensearch-build-schema-version-1.2.yml b/tests/tests_manifests/data/build/opensearch-build-schema-version-1.2.yml index c0d5486753..fe14999355 100644 --- a/tests/tests_manifests/data/build/opensearch-build-schema-version-1.2.yml +++ b/tests/tests_manifests/data/build/opensearch-build-schema-version-1.2.yml @@ -2,7 +2,6 @@ build: platform: linux architecture: x64 - distribution: None id: c3ff7a232d25403fa8cc14c97799c323 name: OpenSearch version: 1.1.0 diff --git a/tests/tests_manifests/data/opensearch-build-1.1.0.yml b/tests/tests_manifests/data/opensearch-build-1.1.0.yml index b9fe5465f3..eaf25f9a81 100644 --- a/tests/tests_manifests/data/opensearch-build-1.1.0.yml +++ b/tests/tests_manifests/data/opensearch-build-1.1.0.yml @@ -2,7 +2,6 @@ build: platform: linux architecture: x64 - distribution: None id: c3ff7a232d25403fa8cc14c97799c323 name: OpenSearch version: 1.1.0 diff --git a/tests/tests_manifests/data/opensearch-build-1.2.0.yml b/tests/tests_manifests/data/opensearch-build-1.2.0.yml index 0bfbfe986c..e8978b85ad 100644 --- a/tests/tests_manifests/data/opensearch-build-1.2.0.yml +++ b/tests/tests_manifests/data/opensearch-build-1.2.0.yml @@ -2,7 +2,6 @@ build: platform: linux architecture: x64 - distribution: None id: c3ff7a232d25403fa8cc14c97799c323 name: OpenSearch version: 1.2.0 diff --git a/tests/tests_sign_workflow/data/opensearch-build-1.1.0.yml b/tests/tests_sign_workflow/data/opensearch-build-1.1.0.yml index 4d1e1d93dc..be06b06b94 100644 --- a/tests/tests_sign_workflow/data/opensearch-build-1.1.0.yml +++ b/tests/tests_sign_workflow/data/opensearch-build-1.1.0.yml @@ -2,7 +2,6 @@ build: platform: linux architecture: x64 - distribution: None id: c3ff7a232d25403fa8cc14c97799c323 name: OpenSearch version: 1.1.0 diff --git a/tests/tests_test_workflow/data/local/builds/opensearch/manifest.yml b/tests/tests_test_workflow/data/local/builds/opensearch/manifest.yml index 73c2897641..6508a80568 100644 --- a/tests/tests_test_workflow/data/local/builds/opensearch/manifest.yml +++ b/tests/tests_test_workflow/data/local/builds/opensearch/manifest.yml @@ -1,7 +1,6 @@ --- build: architecture: x64 - distribution: None id: a8f93473cab545bbaac3e2ce599b5434 name: OpenSearch platform: linux diff --git a/tests/tests_test_workflow/data/remote/builds/opensearch/manifest.yml b/tests/tests_test_workflow/data/remote/builds/opensearch/manifest.yml index 73c2897641..6508a80568 100644 --- a/tests/tests_test_workflow/data/remote/builds/opensearch/manifest.yml +++ b/tests/tests_test_workflow/data/remote/builds/opensearch/manifest.yml @@ -1,7 +1,6 @@ --- build: architecture: x64 - distribution: None id: a8f93473cab545bbaac3e2ce599b5434 name: OpenSearch platform: linux diff --git a/tests/tests_test_workflow/test_bwc_workflow/bwc_test/data/build_manifest.yml b/tests/tests_test_workflow/test_bwc_workflow/bwc_test/data/build_manifest.yml index c021b97740..eaf25f9a81 100644 --- a/tests/tests_test_workflow/test_bwc_workflow/bwc_test/data/build_manifest.yml +++ b/tests/tests_test_workflow/test_bwc_workflow/bwc_test/data/build_manifest.yml @@ -1,7 +1,6 @@ --- build: platform: linux - distribution: None architecture: x64 id: c3ff7a232d25403fa8cc14c97799c323 name: OpenSearch diff --git a/tests/tests_test_workflow/test_integ_workflow/integ_test/data/build_manifest.yml b/tests/tests_test_workflow/test_integ_workflow/integ_test/data/build_manifest.yml index c021b97740..eaf25f9a81 100644 --- a/tests/tests_test_workflow/test_integ_workflow/integ_test/data/build_manifest.yml +++ b/tests/tests_test_workflow/test_integ_workflow/integ_test/data/build_manifest.yml @@ -1,7 +1,6 @@ --- build: platform: linux - distribution: None architecture: x64 id: c3ff7a232d25403fa8cc14c97799c323 name: OpenSearch diff --git a/tests/tests_test_workflow/test_integ_workflow/integ_test/data/build_manifest_missing_components.yml b/tests/tests_test_workflow/test_integ_workflow/integ_test/data/build_manifest_missing_components.yml index b19697a463..8067daf0b4 100644 --- a/tests/tests_test_workflow/test_integ_workflow/integ_test/data/build_manifest_missing_components.yml +++ b/tests/tests_test_workflow/test_integ_workflow/integ_test/data/build_manifest_missing_components.yml @@ -2,7 +2,6 @@ build: platform: linux architecture: x64 - distribution: None id: c3ff7a232d25403fa8cc14c97799c323 name: OpenSearch version: 1.1.0