Added platform to manifests. (#751)

Signed-off-by: dblock <[email protected]>

src/assemble_workflow/bundle_recorder.py

@@ -23,12 +23,18 @@ def __init__(self, build, output_dir, artifacts_dir):
             build.id,
             build.name,
             build.version,
+            build.platform,
             build.architecture,
             self.__get_tar_location(),
         )
 
     def __get_tar_name(self, build):
-        parts = [build.name.lower().replace(" ", "-"), build.version, "linux", build.architecture]
+        parts = [
+            build.name.lower().replace(" ", "-"),
+            build.version,
+            build.platform,
+            build.architecture,
+        ]
         return "-".join(parts) + ".tar.gz"
 
     def __get_public_url_path(self, folder, rel_path):
@@ -70,15 +76,16 @@ def write_manifest(self, folder):
         self.get_manifest().to_file(manifest_path)
 
     class BundleManifestBuilder:
-        def __init__(self, build_id, name, version, arch, location):
+        def __init__(self, build_id, name, version, platform, arch, location):
             self.data = {}
             self.data["build"] = {}
             self.data["build"]["id"] = build_id
             self.data["build"]["name"] = name
             self.data["build"]["version"] = str(version)
+            self.data["build"]["platform"] = platform
             self.data["build"]["architecture"] = arch
             self.data["build"]["location"] = location
-            self.data["schema-version"] = "1.0"
+            self.data["schema-version"] = "1.1"
             # We need to store components as a hash so that we can append artifacts by component name
             # When we convert to a BundleManifest this will get converted back into a list
             self.data["components"] = []

src/build_workflow/build_recorder.py

@@ -61,8 +61,9 @@ def __init__(self, target):
             self.data["build"]["id"] = target.build_id
             self.data["build"]["name"] = target.name
             self.data["build"]["version"] = target.opensearch_version
+            self.data["build"]["platform"] = target.platform
             self.data["build"]["architecture"] = target.arch
-            self.data["schema-version"] = "1.1"
+            self.data["schema-version"] = "1.2"
             self.components_hash = {}
 
         def append_component(self, name, version, repository_url, ref, commit_id):

src/manifests/build_manifest.py

@@ -14,11 +14,12 @@
 The manifest contains information about the product that was built (in the `build` section),
 and the components that made up the build in the `components` section.
 
-The format for schema version 1.0 is:
-schema-version: "1.0"
+The format for schema version 1.2 is:
+schema-version: "1.2"
 build:
   name: string
   version: string
+  platform: linux or darwin
   architecture: x64 or arm64
 components:
   - name: string
@@ -47,13 +48,14 @@ class BuildManifest(Manifest):
             "required": True,
             "type": "dict",
             "schema": {
+                "platform": {"required": True, "type": "string"},
                 "architecture": {"required": True, "type": "string"},
                 "id": {"required": True, "type": "string"},
                 "name": {"required": True, "type": "string"},
                 "version": {"required": True, "type": "string"},
             },
         },
-        "schema-version": {"required": True, "type": "string", "allowed": ["1.1"]},
+        "schema-version": {"required": True, "type": "string", "allowed": ["1.2"]},
         "components": {
             "type": "list",
             "schema": {
@@ -89,7 +91,7 @@ def __init__(self, data):
 
     def __to_dict__(self):
         return {
-            "schema-version": "1.1",
+            "schema-version": "1.2",
             "build": self.build.__to_dict__(),
             "components": list(
                 map(lambda component: component.__to_dict__(), self.components)
@@ -109,15 +111,16 @@ def get_component(self, component_name):
 
     @staticmethod
     def get_build_manifest_relative_location(
-        build_id, opensearch_version, architecture
+        build_id, opensearch_version, platform, architecture
     ):
+        # TODO: use platform, https://github.com/opensearch-project/opensearch-build/issues/669
         return f"builds/{opensearch_version}/{build_id}/{architecture}/manifest.yml"
 
     @staticmethod
-    def from_s3(bucket_name, build_id, opensearch_version, architecture, work_dir=None):
+    def from_s3(bucket_name, build_id, opensearch_version, platform, architecture, work_dir=None):
         work_dir = work_dir if not None else str(os.getcwd())
         manifest_s3_path = BuildManifest.get_build_manifest_relative_location(
-            build_id, opensearch_version, architecture
+            build_id, opensearch_version, platform, architecture
         )
         S3Bucket(bucket_name).download_file(manifest_s3_path, work_dir)
         build_manifest = BuildManifest.from_path("manifest.yml")
@@ -131,13 +134,15 @@ class Build:
         def __init__(self, data):
             self.name = data["name"]
             self.version = data["version"]
+            self.platform = data["platform"]
             self.architecture = data["architecture"]
             self.id = data["id"]
 
         def __to_dict__(self):
             return {
                 "name": self.name,
                 "version": self.version,
+                "platform": self.platform,
                 "architecture": self.architecture,
                 "id": self.id,
             }

src/manifests/bundle_manifest.py

@@ -16,11 +16,12 @@ class BundleManifest(Manifest):
     The manifest contains information about the bundle that was built (in the `assemble` section),
     and the components that made up the bundle in the `components` section.
 
-    The format for schema version 1.0 is:
-        schema-version: "1.0"
+    The format for schema version 1.1 is:
+        schema-version: "1.1"
         build:
           name: string
           version: string
+          platform: linux or darwin
           architecture: x64 or arm64
           location: /relative/path/to/tarball
         components:
@@ -36,14 +37,15 @@ class BundleManifest(Manifest):
             "required": True,
             "type": "dict",
             "schema": {
+                "platform": {"required": True, "type": "string"},
                 "architecture": {"required": True, "type": "string"},
                 "id": {"required": True, "type": "string"},
                 "location": {"required": True, "type": "string"},
                 "name": {"required": True, "type": "string"},
                 "version": {"required": True, "type": "string"},
             },
         },
-        "schema-version": {"required": True, "type": "string", "allowed": ["1.0"]},
+        "schema-version": {"required": True, "type": "string", "allowed": ["1.1"]},
         "components": {
             "required": True,
             "type": "list",
@@ -69,42 +71,51 @@ def __init__(self, data):
 
     def __to_dict__(self):
         return {
-            "schema-version": "1.0",
+            "schema-version": "1.1",
             "build": self.build.__to_dict__(),
             "components": list(
                 map(lambda component: component.__to_dict__(), self.components)
             ),
         }
 
     @staticmethod
-    def from_s3(bucket_name, build_id, opensearch_version, architecture, work_dir=None):
+    def from_s3(
+        bucket_name, build_id, opensearch_version, platform, architecture, work_dir=None
+    ):
         work_dir = work_dir if not None else str(os.getcwd())
         manifest_s3_path = BundleManifest.get_bundle_manifest_relative_location(
-            build_id, opensearch_version, architecture
+            build_id, opensearch_version, platform, architecture
         )
         S3Bucket(bucket_name).download_file(manifest_s3_path, work_dir)
-        bundle_manifest = BundleManifest.from_path(os.path.join(work_dir, 'manifest.yml'))
+        bundle_manifest = BundleManifest.from_path(
+            os.path.join(work_dir, "manifest.yml")
+        )
         os.remove(os.path.realpath(os.path.join(work_dir, "manifest.yml")))
         return bundle_manifest
 
     @staticmethod
-    def get_tarball_relative_location(build_id, opensearch_version, architecture):
-        return f"bundles/{opensearch_version}/{build_id}/{architecture}/opensearch-{opensearch_version}-linux-{architecture}.tar.gz"
+    def get_tarball_relative_location(
+        build_id, opensearch_version, platform, architecture
+    ):
+        # TODO: use platform, https://github.com/opensearch-project/opensearch-build/issues/669
+        return f"bundles/{opensearch_version}/{build_id}/{architecture}/opensearch-{opensearch_version}-{platform}-{architecture}.tar.gz"
 
     @staticmethod
-    def get_tarball_name(opensearch_version, architecture):
-        return f"opensearch-{opensearch_version}-linux-{architecture}.tar.gz"
+    def get_tarball_name(opensearch_version, platform, architecture):
+        return f"opensearch-{opensearch_version}-{platform}-{architecture}.tar.gz"
 
     @staticmethod
     def get_bundle_manifest_relative_location(
-        build_id, opensearch_version, architecture
+        build_id, opensearch_version, platform, architecture
     ):
+        # TODO: use platform, https://github.com/opensearch-project/opensearch-build/issues/669
         return f"bundles/{opensearch_version}/{build_id}/{architecture}/manifest.yml"
 
     class Build:
         def __init__(self, data):
             self.name = data["name"]
             self.version = data["version"]
+            self.platform = data["platform"]
             self.architecture = data["architecture"]
             self.location = data["location"]
             self.id = data["id"]
@@ -113,6 +124,7 @@ def __to_dict__(self):
             return {
                 "name": self.name,
                 "version": self.version,
+                "platform": self.platform,
                 "architecture": self.architecture,
                 "location": self.location,
                 "id": self.id,

src/run_assemble.py

@@ -55,7 +55,7 @@ def main():
 
     with tempfile.TemporaryDirectory() as work_dir:
         logging.info(
-            f"Bundling {build.name} ({build.architecture}) into {output_dir} ..."
+            f"Bundling {build.name} ({build.architecture}) on {build.platform} into {output_dir} ..."
         )
 
         os.chdir(work_dir)

src/run_integ_test.py

@@ -34,7 +34,9 @@ def pull_build_repo(work_dir):
 def main():
     args = TestArgs()
     console.configure(level=args.logging_level)
-    test_manifest_path = os.path.join(os.path.dirname(__file__), 'test_workflow/config/test_manifest.yml')
+    test_manifest_path = os.path.join(
+        os.path.dirname(__file__), "test_workflow/config/test_manifest.yml"
+    )
     test_manifest = TestManifest.from_path(test_manifest_path)
     integ_test_config = dict()
     for component in test_manifest.components:
@@ -45,9 +47,21 @@ def main():
         test_recorder = TestRecorder(args.test_run_id, "integ-test", work_dir)
         os.chdir(work_dir)
         bundle_manifest = BundleManifest.from_s3(
-            args.s3_bucket, args.build_id, args.opensearch_version, args.architecture, work_dir)
+            args.s3_bucket,
+            args.build_id,
+            args.opensearch_version,
+            args.platform,
+            args.architecture,
+            work_dir,
+        )
         build_manifest = BuildManifest.from_s3(
-            args.s3_bucket, args.build_id, args.opensearch_version, args.architecture, work_dir)
+            args.s3_bucket,
+            args.build_id,
+            args.opensearch_version,
+            args.platform,
+            args.architecture,
+            work_dir,
+        )
         pull_build_repo(work_dir)
         DependencyInstaller(build_manifest.build).install_all_maven_dependencies()
         all_results = TestSuiteResults()
@@ -60,7 +74,7 @@ def main():
                     build_manifest,
                     work_dir,
                     args.s3_bucket,
-                    test_recorder
+                    test_recorder,
                 )
                 test_results = test_suite.execute()
                 all_results.append(component.name, test_results)

src/test_workflow/dependency_installer.py

@@ -20,14 +20,11 @@ class DependencyInstaller:
     def __init__(self, build):
         self.build_id = build.id
         self.version = build.version
-        self.arch = build.architecture
+        self.platform = build.platform
+        self.architecture = build.architecture
         self.s3_bucket = S3Bucket(self.ARTIFACT_S3_BUCKET)
-        self.s3_maven_location = (
-            f"builds/{self.version}/{self.build_id}/{self.arch}/maven/org/opensearch"
-        )
-        self.s3_build_location = (
-            f"builds/{self.version}/{self.build_id}/{self.arch}/plugins"
-        )
+        self.s3_maven_location = f"builds/{self.version}/{self.build_id}/{self.platform}/{self.architecture}/maven/org/opensearch"
+        self.s3_build_location = f"builds/{self.version}/{self.build_id}/{self.platform}/{self.architecture}/plugins"
         self.maven_local_path = os.path.join(
             os.path.expanduser("~"), ".m2/repository/org/opensearch/"
         )