(Re-open) Update build workflow to build based on distribution (OpenS…

…earch) (#1629)

* Update build workflow to build based on distribution (OpenSearch)

Signed-off-by: Peter Zhu <[email protected]>

* Fix all existing tests to run

Signed-off-by: Peter Zhu <[email protected]>

* More changes based on the style and outputs

Signed-off-by: Peter Zhu <[email protected]>

* Add 3 more tests to the PR

Signed-off-by: Peter Zhu <[email protected]>

* Add more test cases and readme contents

Signed-off-by: Peter Zhu <[email protected]>

scripts/components/OpenSearch/build.sh

@@ -16,11 +16,12 @@ function usage() {
     echo -e "-s SNAPSHOT\t[Optional] Build a snapshot, default is 'false'."
     echo -e "-p PLATFORM\t[Optional] Platform, default is 'uname -s'."
     echo -e "-a ARCHITECTURE\t[Optional] Build architecture, default is 'uname -m'."
+    echo -e "-d DISTRIBUTION\t[Optional] Distribution, default is 'tar'."
     echo -e "-o OUTPUT\t[Optional] Output path, default is 'artifacts'."
     echo -e "-h help"
 }
 
-while getopts ":h:v:s:o:p:a:" arg; do
+while getopts ":h:v:s:o:p:a:d:" arg; do
     case $arg in
         h)
             usage
@@ -41,6 +42,9 @@ while getopts ":h:v:s:o:p:a:" arg; do
         a)
             ARCHITECTURE=$OPTARG
             ;;
+        d)
+            DISTRIBUTION=$OPTARG
+            ;;
         :)
             echo "Error: -${OPTARG} requires an argument"
             usage
@@ -77,48 +81,66 @@ cp -r ./build/local-test-repo/org/opensearch "${OUTPUT}"/maven/org
 
 [ -z "$PLATFORM" ] && PLATFORM=$(uname -s | awk '{print tolower($0)}')
 [ -z "$ARCHITECTURE" ] && ARCHITECTURE=`uname -m`
+[ -z "$DISTRIBUTION" ] && DISTRIBUTION="tar"
 
-case $PLATFORM in
-    linux*)
+case $PLATFORM-$DISTRIBUTION-$ARCHITECTURE in
+    linux-tar-x64|darwin-tar-x64)
         PACKAGE="tar"
         EXT="tar.gz"
+        TYPE="archives"
+        TARGET="$PLATFORM-$PACKAGE"
+        QUALIFIER="$PLATFORM-x64"
         ;;
-    darwin*)
+    linux-tar-arm64|darwin-tar-arm64)
         PACKAGE="tar"
         EXT="tar.gz"
+        TYPE="archives"
+        TARGET="$PLATFORM-arm64-$PACKAGE"
+        QUALIFIER="$PLATFORM-arm64"
         ;;
-    windows*)
-        PACKAGE="zip"
-        EXT="zip"
+    linux-rpm-x64)
+        PACKAGE="rpm"
+        EXT="rpm"
+        TYPE="packages"
+        TARGET="rpm"
+        QUALIFIER="x86_64"
         ;;
-    *)
-        echo "Unsupported platform: $PLATFORM"
-        exit 1
+    linux-rpm-arm64)
+        PACKAGE="rpm"
+        EXT="rpm"
+        TYPE="packages"
+        TARGET="arm64-rpm"
+        QUALIFIER="aarch64"
         ;;
-esac
-
-case $ARCHITECTURE in
-    x64)
+    windows-zip-x64)
+        PACKAGE="zip"
+        EXT="zip"
+        TYPE="archives"
         TARGET="$PLATFORM-$PACKAGE"
         QUALIFIER="$PLATFORM-x64"
         ;;
-    arm64)
+    windows-zip-arm64)
+        PACKAGE="zip"
+        EXT="zip"
+        TYPE="archives"
         TARGET="$PLATFORM-arm64-$PACKAGE"
         QUALIFIER="$PLATFORM-arm64"
         ;;
     *)
-        echo "Unsupported architecture: $ARCHITECTURE"
+        echo "Unsupported platform-distribution-architecture combination: $PLATFORM-$DISTRIBUTION-$ARCHITECTURE"
         exit 1
         ;;
 esac
 
-./gradlew :distribution:archives:$TARGET:assemble -Dbuild.snapshot=$SNAPSHOT
+echo "Building OpenSearch for $PLATFORM-$DISTRIBUTION-$ARCHITECTURE"
+
+./gradlew :distribution:$TYPE:$TARGET:assemble -Dbuild.snapshot=$SNAPSHOT
 
 # Copy artifact to dist folder in bundle build output
 [[ "$SNAPSHOT" == "true" ]] && IDENTIFIER="-SNAPSHOT"
-ARTIFACT_BUILD_NAME=`ls distribution/archives/$TARGET/build/distributions/ | grep "opensearch-min.*$QUALIFIER.$EXT"`
+ARTIFACT_BUILD_NAME=`ls distribution/$TYPE/$TARGET/build/distributions/ | grep "opensearch-min.*$QUALIFIER.$EXT"`
 mkdir -p "${OUTPUT}/dist"
-cp distribution/archives/$TARGET/build/distributions/$ARTIFACT_BUILD_NAME "${OUTPUT}"/dist/$ARTIFACT_BUILD_NAME
+cp distribution/$TYPE/$TARGET/build/distributions/$ARTIFACT_BUILD_NAME "${OUTPUT}"/dist/$ARTIFACT_BUILD_NAME
 
 echo "Building core plugins..."
 mkdir -p "${OUTPUT}/core-plugins"

src/build_workflow/README.md

@@ -65,7 +65,10 @@ The following options are available in `build.sh`.
 
 | name               | description                                                             |
 |--------------------|-------------------------------------------------------------------------|
-| --snapshot         | Build a snapshot instead of a release artifact, default is `false`.     |
+| -s, --snapshot     | Build a snapshot instead of a release artifact, default is `false`.     |
+| -a, --architecture | Specify architecture to build, default is architecture of build system. |
+| -d, --distribution | Specify distribution to build, default is `tar`.                        |
+| -p, --platform     | Specify platform to build, default is platform of build system.         |
 | --component [name] | Rebuild a single component by name, e.g. `--component common-utils`.    |
 | --keep             | Do not delete the temporary working directory on both success or error. |
 | -l, --lock         | Generate a stable reference manifest.                                   |

src/build_workflow/build_args.py

@@ -16,13 +16,15 @@ class BuildArgs:
         "x64",
         "arm64",
     ]
+    SUPPORTED_DISTRIBUTIONS = ["tar", "zip", "rpm"]
 
     manifest: IO
     snapshot: bool
     component: str
     keep: bool
     platform: str
     architecture: str
+    distribution: str
 
     def __init__(self) -> None:
         parser = argparse.ArgumentParser(description="Build an OpenSearch Bundle")
@@ -60,6 +62,15 @@ def __init__(self) -> None:
             const=logging.DEBUG,
             dest="logging_level",
         )
+        parser.add_argument(
+            "-d",
+            "--distribution",
+            type=str,
+            choices=self.SUPPORTED_DISTRIBUTIONS,
+            help="Distribution to build.",
+            default=None,
+            dest="distribution"
+        )
 
         args = parser.parse_args()
         self.logging_level = args.logging_level
@@ -70,6 +81,7 @@ def __init__(self) -> None:
         self.keep = args.keep
         self.platform = args.platform
         self.architecture = args.architecture
+        self.distribution = args.distribution
         self.script_path = sys.argv[0].replace("/src/run_build.py", "/build.sh")
 
     def component_command(self, name: str) -> str:

src/build_workflow/build_recorder.py

@@ -60,6 +60,7 @@ def __init__(self, target: BuildTarget):
             self.data["build"]["version"] = target.opensearch_version
             self.data["build"]["platform"] = target.platform
             self.data["build"]["architecture"] = target.architecture
+            self.data["build"]["distribution"] = target.distribution if target.distribution else "tar"
             self.data["schema-version"] = "1.2"
             self.components_hash: Dict[str, Dict[str, Any]] = {}
 

src/build_workflow/build_target.py

@@ -17,6 +17,7 @@ class BuildTarget:
     version: str
     platform: str
     architecture: str
+    distribution: str
     snapshot: bool
     output_dir: str
 
@@ -26,6 +27,7 @@ def __init__(
         patches: List[str] = [],
         platform: str = None,
         architecture: str = None,
+        distribution: str = None,
         name: str = None,
         snapshot: bool = True,
         build_id: str = None,
@@ -37,6 +39,7 @@ def __init__(
         self.patches = patches
         self.snapshot = snapshot
         self.architecture = architecture or current_architecture()
+        self.distribution = distribution
         self.platform = platform or current_platform()
         self.output_dir = output_dir
 

src/build_workflow/builder_from_source.py

@@ -31,20 +31,19 @@ def build(self, build_recorder: BuildRecorder) -> None:
         build_script = ScriptFinder.find_build_script(self.target.name, self.component.name, self.git_repo.working_directory)
 
         build_command = " ".join(
-            [
-                "bash",
-                build_script,
-                "-v",
-                self.target.version,
-                "-p",
-                self.target.platform,
-                "-a",
-                self.target.architecture,
-                "-s",
-                str(self.target.snapshot).lower(),
-                "-o",
-                self.output_path,
-            ]
+            filter(
+                None,
+                [
+                    "bash",
+                    build_script,
+                    f"-v {self.target.version}",
+                    f"-p {self.target.platform}",
+                    f"-a {self.target.architecture}",
+                    f"-d {self.target.distribution}" if self.target.distribution else None,
+                    f"-s {str(self.target.snapshot).lower()}",
+                    f"-o {self.output_path}",
+                ]
+            )
         )
 
         self.git_repo.execute(build_command)

src/manifests/build_manifest.py

@@ -55,6 +55,7 @@ class BuildManifest(ComponentManifest['BuildManifest', 'BuildComponents']):
             "schema": {
                 "platform": {"required": True, "type": "string"},  # added in 1.2
                 "architecture": {"required": True, "type": "string"},
+                "distribution": {"type": "string"},
                 "id": {"required": True, "type": "string"},
                 "name": {"required": True, "type": "string"},
                 "version": {"required": True, "type": "string"},
@@ -104,6 +105,7 @@ def __init__(self, data: Any):
             self.version: str = data["version"]
             self.platform: str = data["platform"]
             self.architecture: str = data["architecture"]
+            self.distribution: str = data.get('distribution', None)
             self.id: str = data["id"]
 
         def __to_dict__(self) -> dict:
@@ -112,6 +114,7 @@ def __to_dict__(self) -> dict:
                 "version": self.version,
                 "platform": self.platform,
                 "architecture": self.architecture,
+                "distribution": self.distribution,
                 "id": self.id
             }
 

src/run_build.py

@@ -49,6 +49,7 @@ def main():
             patches=manifest.build.patches,
             snapshot=args.snapshot if args.snapshot is not None else manifest.build.snapshot,
             output_dir=output_dir,
+            distribution=args.distribution,
             platform=args.platform or manifest.build.platform,
             architecture=args.architecture or manifest.build.architecture,
         )

tests/tests_build_workflow/test_build_args.py

@@ -81,6 +81,14 @@ def test_architecture_default(self) -> None:
     def test_architecture(self) -> None:
         self.assertEqual(BuildArgs().architecture, "arm64")
 
+    @patch("argparse._sys.argv", [BUILD_PY, OPENSEARCH_MANIFEST])
+    def test_distribution_default(self) -> None:
+        self.assertIsNone(BuildArgs().distribution)
+
+    @patch("argparse._sys.argv", [BUILD_PY, OPENSEARCH_MANIFEST, "--distribution", "rpm"])
+    def test_distribution(self) -> None:
+        self.assertEqual(BuildArgs().distribution, "rpm")
+
     @patch("argparse._sys.argv", [BUILD_PY, OPENSEARCH_MANIFEST, "--component", "xyz"])
     def test_script_path(self) -> None:
         self.assertEqual(BuildArgs().script_path, self.BUILD_SH)

tests/tests_build_workflow/test_build_recorder.py

@@ -25,13 +25,27 @@ def __mock(self, snapshot: bool = True) -> BuildRecorder:
                 build_id="1",
                 output_dir="output_dir",
                 name="OpenSearch",
-                version="1.1.0",
+                version="1.3.0",
                 platform="linux",
                 architecture="x64",
                 snapshot=snapshot,
             )
         )
 
+    def __mock_distribution(self, snapshot: bool = True) -> BuildRecorder:
+        return BuildRecorder(
+            BuildTarget(
+                build_id="1",
+                output_dir="output_dir",
+                name="OpenSearch",
+                version="1.3.0",
+                platform="linux",
+                architecture="x64",
+                distribution="rpm",
+                snapshot=snapshot,
+            )
+        )
+
     @patch("shutil.copyfile")
     @patch("os.makedirs")
     def test_record_component_and_artifact(self, mock_makedirs: Mock, mock_copyfile: Mock) -> None:
@@ -56,9 +70,10 @@ def test_record_component_and_artifact(self, mock_makedirs: Mock, mock_copyfile:
                 "build": {
                     "platform": "linux",
                     "architecture": "x64",
+                    "distribution": "tar",
                     "id": "1",
                     "name": "OpenSearch",
-                    "version": "1.1.0",
+                    "version": "1.3.0",
                 },
                 "components": [
                     {
@@ -67,7 +82,7 @@ def test_record_component_and_artifact(self, mock_makedirs: Mock, mock_copyfile:
                         "name": "common-utils",
                         "ref": "main",
                         "repository": "https://github.com/opensearch-project/common-utils",
-                        "version": "1.1.0.0",
+                        "version": "1.3.0.0",
                     }
                 ],
                 "schema-version": "1.2",
@@ -134,9 +149,28 @@ def test_get_manifest(self) -> None:
                 "build": {
                     "platform": "linux",
                     "architecture": "x64",
+                    "distribution": "tar",
+                    "id": "1",
+                    "name": "OpenSearch",
+                    "version": "1.3.0",
+                },
+                "schema-version": "1.2",
+            },
+        )
+
+    def test_get_manifest_distribution(self) -> None:
+        manifest = self.__mock_distribution(snapshot=False).get_manifest()
+        self.assertIs(type(manifest), BuildManifest)
+        self.assertEqual(
+            manifest.to_dict(),
+            {
+                "build": {
+                    "platform": "linux",
+                    "architecture": "x64",
+                    "distribution": "rpm",
                     "id": "1",
                     "name": "OpenSearch",
-                    "version": "1.1.0",
+                    "version": "1.3.0",
                 },
                 "schema-version": "1.2",
             },
@@ -166,10 +200,10 @@ def test_record_artifact_check_plugin_version_properties(self, mock_plugin_check
                 sha="3913d7097934cbfe1fdcf919347f22a597d00b76",
             ),
         )
-        mock.record_artifact("security", "plugins", "../file1.zip", "valid-1.1.0.0.zip")
+        mock.record_artifact("security", "plugins", "../file1.zip", "valid-1.3.0.0.zip")
         manifest_dict = mock.get_manifest().to_dict()
-        self.assertEqual(manifest_dict["build"]["version"], "1.1.0")
-        self.assertEqual(manifest_dict["components"][0]["version"], "1.1.0.0")
+        self.assertEqual(manifest_dict["build"]["version"], "1.3.0")
+        self.assertEqual(manifest_dict["components"][0]["version"], "1.3.0.0")
         mock_plugin_check.assert_called()
         mock_copyfile.assert_called()
         mock_makedirs.assert_called()
@@ -187,10 +221,10 @@ def test_record_artifact_check_plugin_version_properties_snapshot(self, mock_plu
                 sha="3913d7097934cbfe1fdcf919347f22a597d00b76",
             ),
         )
-        mock.record_artifact("security", "plugins", "../file1.zip", "valid-1.1.0.0-SNAPSHOT.zip")
+        mock.record_artifact("security", "plugins", "../file1.zip", "valid-1.3.0.0-SNAPSHOT.zip")
         manifest_dict = mock.get_manifest().to_dict()
-        self.assertEqual(manifest_dict["build"]["version"], "1.1.0-SNAPSHOT")
-        self.assertEqual(manifest_dict["components"][0]["version"], "1.1.0.0-SNAPSHOT")
+        self.assertEqual(manifest_dict["build"]["version"], "1.3.0-SNAPSHOT")
+        self.assertEqual(manifest_dict["components"][0]["version"], "1.3.0.0-SNAPSHOT")
         mock_plugin_check.assert_called()
         mock_copyfile.assert_called()
         mock_makedirs.assert_called()

tests/tests_build_workflow/test_build_target.py

@@ -25,6 +25,9 @@ def test_build_id_from_env(self) -> None:
     def test_build_id_from_arg(self) -> None:
         self.assertEqual(BuildTarget(version="1.1.0", architecture="x86", build_id="id").build_id, "id")
 
+    def test_distribution_from_arg(self) -> None:
+        self.assertEqual(BuildTarget(version="1.3.0", architecture="x86", distribution="rpm").distribution, "rpm")
+
     def test_opensearch_version(self) -> None:
         self.assertEqual(
             BuildTarget(version="1.1.0", architecture="x86", snapshot=False).opensearch_version,