Added version qualifier support for OpenSearch. (#1731)

Signed-off-by: dblock <[email protected]>

manifests/2.0.0/opensearch-2.0.0.yml

@@ -7,6 +7,7 @@ ci:
 build:
   name: OpenSearch
   version: 2.0.0
+  qualifier: alpha1
 components:
   - name: OpenSearch
     ref: main

scripts/components/OpenSearch/build.sh

@@ -13,6 +13,7 @@ function usage() {
     echo ""
     echo "Arguments:"
     echo -e "-v VERSION\t[Required] OpenSearch version."
+    echo -e "-q QUALIFIER\t[Optional] Version qualifier."
     echo -e "-s SNAPSHOT\t[Optional] Build a snapshot, default is 'false'."
     echo -e "-p PLATFORM\t[Optional] Platform, default is 'uname -s'."
     echo -e "-a ARCHITECTURE\t[Optional] Build architecture, default is 'uname -m'."
@@ -21,7 +22,7 @@ function usage() {
     echo -e "-h help"
 }
 
-while getopts ":h:v:s:o:p:a:d:" arg; do
+while getopts ":h:v:q:s:o:p:a:d:" arg; do
     case $arg in
         h)
             usage
@@ -30,6 +31,9 @@ while getopts ":h:v:s:o:p:a:d:" arg; do
         v)
             VERSION=$OPTARG
             ;;
+        q)
+            QUALIFIER=$OPTARG
+            ;;
         s)
             SNAPSHOT=$OPTARG
             ;;
@@ -68,10 +72,10 @@ fi
 mkdir -p $OUTPUT/maven/org/opensearch
 
 # Build project and publish to maven local.
-./gradlew publishToMavenLocal -Dbuild.snapshot=$SNAPSHOT
+./gradlew publishToMavenLocal -Dbuild.snapshot=$SNAPSHOT -Dbuild.version_qualifier=$QUALIFIER
 
 # Publish to existing test repo, using this to stage release versions of the artifacts that can be released from the same build.
-./gradlew publishNebulaPublicationToTestRepository -Dbuild.snapshot=$SNAPSHOT
+./gradlew publishNebulaPublicationToTestRepository -Dbuild.snapshot=$SNAPSHOT -Dbuild.version_qualifier=$QUALIFIER
 
 # Copy maven publications to be promoted
 cp -r ./build/local-test-repo/org/opensearch "${OUTPUT}"/maven/org
@@ -89,42 +93,42 @@ case $PLATFORM-$DISTRIBUTION-$ARCHITECTURE in
         EXT="tar.gz"
         TYPE="archives"
         TARGET="$PLATFORM-$PACKAGE"
-        QUALIFIER="$PLATFORM-x64"
+        SUFFIX="$PLATFORM-x64"
         ;;
     linux-tar-arm64|darwin-tar-arm64)
         PACKAGE="tar"
         EXT="tar.gz"
         TYPE="archives"
         TARGET="$PLATFORM-arm64-$PACKAGE"
-        QUALIFIER="$PLATFORM-arm64"
+        SUFFIX="$PLATFORM-arm64"
         ;;
     linux-rpm-x64)
         PACKAGE="rpm"
         EXT="rpm"
         TYPE="packages"
         TARGET="rpm"
-        QUALIFIER="x86_64"
+        SUFFIX="x86_64"
         ;;
     linux-rpm-arm64)
         PACKAGE="rpm"
         EXT="rpm"
         TYPE="packages"
         TARGET="arm64-rpm"
-        QUALIFIER="aarch64"
+        SUFFIX="aarch64"
         ;;
     windows-zip-x64)
         PACKAGE="zip"
         EXT="zip"
         TYPE="archives"
         TARGET="$PLATFORM-$PACKAGE"
-        QUALIFIER="$PLATFORM-x64"
+        SUFFIX="$PLATFORM-x64"
         ;;
     windows-zip-arm64)
         PACKAGE="zip"
         EXT="zip"
         TYPE="archives"
         TARGET="$PLATFORM-arm64-$PACKAGE"
-        QUALIFIER="$PLATFORM-arm64"
+        SUFFIX="$PLATFORM-arm64"
         ;;
     *)
         echo "Unsupported platform-distribution-architecture combination: $PLATFORM-$DISTRIBUTION-$ARCHITECTURE"
@@ -134,18 +138,18 @@ esac
 
 echo "Building OpenSearch for $PLATFORM-$DISTRIBUTION-$ARCHITECTURE"
 
-./gradlew :distribution:$TYPE:$TARGET:assemble -Dbuild.snapshot=$SNAPSHOT
+./gradlew :distribution:$TYPE:$TARGET:assemble -Dbuild.snapshot=$SNAPSHOT -Dbuild.version_qualifier=$QUALIFIER
 
 # Copy artifact to dist folder in bundle build output
 [[ "$SNAPSHOT" == "true" ]] && IDENTIFIER="-SNAPSHOT"
-ARTIFACT_BUILD_NAME=`ls distribution/$TYPE/$TARGET/build/distributions/ | grep "opensearch-min.*$QUALIFIER.$EXT"`
+ARTIFACT_BUILD_NAME=`ls distribution/$TYPE/$TARGET/build/distributions/ | grep "opensearch-min.*$SUFFIX.$EXT"`
 mkdir -p "${OUTPUT}/dist"
 cp distribution/$TYPE/$TARGET/build/distributions/$ARTIFACT_BUILD_NAME "${OUTPUT}"/dist/$ARTIFACT_BUILD_NAME
 
 echo "Building core plugins..."
 mkdir -p "${OUTPUT}/core-plugins"
 cd plugins
-../gradlew assemble -Dbuild.snapshot="$SNAPSHOT"
+../gradlew assemble -Dbuild.snapshot="$SNAPSHOT" -Dbuild.version_qualifier=$QUALIFIER
 cd ..
 for plugin in plugins/*; do
   PLUGIN_NAME=$(basename "$plugin")

scripts/default/install.sh

@@ -21,7 +21,7 @@ function usage() {
     echo -e "-h help"
 }
 
-while getopts ":h:v:s:o:p:a:f:" arg; do
+while getopts ":h:v:q:s:o:p:a:f:" arg; do
     case $arg in
         h)
             usage
@@ -30,6 +30,9 @@ while getopts ":h:v:s:o:p:a:f:" arg; do
         v)
             VERSION=$OPTARG
             ;;
+        q)
+            QUALIFIER=$OPTARG
+            ;;
         s)
             SNAPSHOT=$OPTARG
             ;;

scripts/default/opensearch-dashboards/build.sh

@@ -20,7 +20,7 @@ function usage() {
     echo -e "-h help"
 }
 
-while getopts ":h:v:s:o:p:a:" arg; do
+while getopts ":h:v:q:s:o:p:a:" arg; do
     case $arg in
         h)
             usage
@@ -29,6 +29,9 @@ while getopts ":h:v:s:o:p:a:" arg; do
         v)
             VERSION=$OPTARG
             ;;
+        q)
+            QUALIFIER=$OPTARG
+            ;;
         s)
             SNAPSHOT=$OPTARG
             ;;

scripts/default/opensearch/build.sh

@@ -20,7 +20,7 @@ function usage() {
     echo -e "-h help"
 }
 
-while getopts ":h:v:s:o:p:a:" arg; do
+while getopts ":h:v:q:s:o:p:a:" arg; do
     case $arg in
         h)
             usage
@@ -29,6 +29,9 @@ while getopts ":h:v:s:o:p:a:" arg; do
         v)
             VERSION=$OPTARG
             ;;
+        q)
+            QUALIFIER=$OPTARG
+            ;;
         s)
             SNAPSHOT=$OPTARG
             ;;

src/build_workflow/build_target.py

@@ -15,6 +15,7 @@ class BuildTarget:
     build_id: str
     name: str
     version: str
+    qualifier: str
     platform: str
     architecture: str
     distribution: str
@@ -24,6 +25,7 @@ class BuildTarget:
     def __init__(
         self,
         version: str,
+        qualifier: str = None,
         patches: List[str] = [],
         platform: str = None,
         architecture: str = None,
@@ -36,6 +38,7 @@ def __init__(
         self.build_id = os.getenv("BUILD_NUMBER") or build_id or uuid.uuid4().hex
         self.name = name
         self.version = version
+        self.qualifier = qualifier
         self.patches = patches
         self.snapshot = snapshot
         self.architecture = architecture or current_architecture()
@@ -45,31 +48,48 @@ def __init__(
 
     @property
     def opensearch_version(self) -> str:
-        return self.version + "-SNAPSHOT" if self.snapshot else self.version
+        return BuildTarget.__qualify_version(
+            self.version,
+            self.qualifier,
+            self.snapshot
+        )
 
     @property
     def compatible_opensearch_versions(self) -> List[str]:
         return (
-            [self.version + "-SNAPSHOT" if self.snapshot else self.version]
+            [BuildTarget.__qualify_version(self.version, self.qualifier, self.snapshot)]
             + self.patches
-            + list(map(lambda version: version + "-SNAPSHOT", self.patches))
+            + list(map(lambda version: BuildTarget.__qualify_version(version, self.qualifier, True), self.patches))
         )
 
     @property
     def component_version(self) -> str:
         # BUG: the 4th digit is dictated by the component, it's not .0, this will break for 1.1.0.1
-        return self.version + ".0-SNAPSHOT" if self.snapshot else f"{self.version}.0"
+        return BuildTarget.__qualify_version(
+            self.version + ".0",
+            self.qualifier,
+            self.snapshot
+        )
 
     @property
     def compatible_component_versions(self) -> List[str]:
         return (
-            [self.version + ".0-SNAPSHOT" if self.snapshot else f"{self.version}.0"]
-            + list(map(lambda version: version + ".0", self.patches))
-            + list(map(lambda version: version + ".0-SNAPSHOT", self.patches))
+            [BuildTarget.__qualify_version(self.version + ".0", self.qualifier, self.snapshot)]
+            + list(map(lambda version: BuildTarget.__qualify_version(version + ".0", self.qualifier, False), self.patches))
+            + list(map(lambda version: BuildTarget.__qualify_version(version + ".0", self.qualifier, True), self.patches))
         )
 
     @property
     def compatible_versions(self) -> List[str]:
         versions = [self.version]
         versions.extend(self.patches)
         return versions
+
+    @classmethod
+    def __qualify_version(cls, unqualified_version: str, qualifier: str = None, snapshot: bool = False) -> str:
+        version = unqualified_version
+        if qualifier:
+            version += f"-{qualifier}"
+        if snapshot:
+            version += "-SNAPSHOT"
+        return version

src/build_workflow/builder_from_source.py

@@ -42,6 +42,7 @@ def build(self, build_recorder: BuildRecorder) -> None:
                     "bash",
                     build_script,
                     f"-v {self.target.version}",
+                    f"-q {self.target.qualifier}" if self.target.qualifier else None,
                     f"-p {self.target.platform}",
                     f"-a {self.target.architecture}",
                     f"-d {self.target.distribution}" if self.target.distribution and (self.component.name in DISTRIBUTION_SUPPORTED_COMPONENTS) else None,

src/manifests/input_manifest.py

@@ -54,6 +54,7 @@ class InputManifest(ComponentManifest['InputManifest', 'InputComponents']):
             "schema": {
                 "name": {"required": True, "type": "string"},
                 "version": {"required": True, "type": "string"},
+                "qualifier": {"type": "string"},
                 "patches": {"type": "list", "schema": {"type": "string"}},
                 "platform": {"type": "string"},
                 "architecture": {"type": "string"},
@@ -143,6 +144,7 @@ class Build:
         def __init__(self, data: Any):
             self.name: str = data["name"]
             self.version = data["version"]
+            self.qualifier = data.get("qualifier", None)
             self.platform = data.get("platform", None)
             self.architecture = data.get("architecture", None)
             self.snapshot = data.get("snapshot", None)
@@ -152,6 +154,7 @@ def __to_dict__(self) -> dict:
             return {
                 "name": self.name,
                 "version": self.version,
+                "qualifier": self.qualifier,
                 "patches": self.patches,
                 "platform": self.platform,
                 "architecture": self.architecture,

src/run_build.py

@@ -46,6 +46,7 @@ def main():
         target = BuildTarget(
             name=manifest.build.name,
             version=manifest.build.version,
+            qualifier=manifest.build.qualifier,
             patches=manifest.build.patches,
             snapshot=args.snapshot if args.snapshot is not None else manifest.build.snapshot,
             output_dir=output_dir,

tests/tests_build_workflow/test_build_target.py

@@ -34,22 +34,46 @@ def test_opensearch_version(self) -> None:
             "1.1.0",
         )
 
+    def test_opensearch_version_snapshot(self) -> None:
+        self.assertEqual(
+            BuildTarget(version="1.1.0", architecture="x86", snapshot=True).opensearch_version,
+            "1.1.0-SNAPSHOT",
+        )
+
+    def test_opensearch_version_qualifier(self) -> None:
+        self.assertEqual(
+            BuildTarget(version="1.1.0", architecture="x86", snapshot=False, qualifier="alpha1").opensearch_version,
+            "1.1.0-alpha1",
+        )
+
+    def test_opensearch_version_snapshot_qualifier(self) -> None:
+        self.assertEqual(
+            BuildTarget(version="1.1.0", architecture="x86", snapshot=True, qualifier="alpha1").opensearch_version,
+            "1.1.0-alpha1-SNAPSHOT",
+        )
+
     def test_compatible_opensearch_versions(self) -> None:
         self.assertEqual(
             BuildTarget(version="1.1.2", architecture="x86", patches=["1.1.0", "1.1.1"], snapshot=False).compatible_opensearch_versions,
             ['1.1.2', '1.1.0', '1.1.1', '1.1.0-SNAPSHOT', '1.1.1-SNAPSHOT'],
         )
 
+    def test_compatible_opensearch_versions_qualifier(self) -> None:
+        self.assertEqual(
+            BuildTarget(version="1.1.2", architecture="x86", patches=["1.1.0", "1.1.1"], snapshot=False, qualifier="alpha1").compatible_opensearch_versions,
+            ['1.1.2-alpha1', '1.1.0', '1.1.1', '1.1.0-alpha1-SNAPSHOT', '1.1.1-alpha1-SNAPSHOT'],
+        )
+
     def test_compatible_opensearch_versions_snapshot(self) -> None:
         self.assertEqual(
             BuildTarget(version="1.1.2", architecture="x86", patches=["1.1.0", "1.1.1"], snapshot=True).compatible_opensearch_versions,
             ['1.1.2-SNAPSHOT', '1.1.0', '1.1.1', '1.1.0-SNAPSHOT', '1.1.1-SNAPSHOT'],
         )
 
-    def test_opensearch_version_snapshot(self) -> None:
+    def test_compatible_opensearch_versions_snapshot_qualifier(self) -> None:
         self.assertEqual(
-            BuildTarget(version="1.1.0", architecture="x86", snapshot=True).opensearch_version,
-            "1.1.0-SNAPSHOT",
+            BuildTarget(version="1.1.2", architecture="x86", patches=["1.1.0", "1.1.1"], snapshot=True, qualifier="alpha1").compatible_opensearch_versions,
+            ['1.1.2-alpha1-SNAPSHOT', '1.1.0', '1.1.1', '1.1.0-alpha1-SNAPSHOT', '1.1.1-alpha1-SNAPSHOT'],
         )
 
     def test_component_version(self) -> None:
@@ -58,18 +82,36 @@ def test_component_version(self) -> None:
             "1.1.0.0",
         )
 
+    def test_component_version_qualifier(self) -> None:
+        self.assertEqual(
+            BuildTarget(version="1.1.0", architecture="x86", qualifier="alpha1", snapshot=False).component_version,
+            "1.1.0.0-alpha1",
+        )
+
     def test_compatible_component_versions(self) -> None:
         self.assertEqual(
             BuildTarget(version="1.1.2", architecture="x86", patches=["1.1.0", "1.1.1"], snapshot=False).compatible_component_versions,
             ['1.1.2.0', '1.1.0.0', '1.1.1.0', '1.1.0.0-SNAPSHOT', '1.1.1.0-SNAPSHOT'],
         )
 
+    def test_compatible_component_versions_qualifier(self) -> None:
+        self.assertEqual(
+            BuildTarget(version="1.1.2", architecture="x86", patches=["1.1.0", "1.1.1"], snapshot=False, qualifier="alpha1").compatible_component_versions,
+            ['1.1.2.0-alpha1', '1.1.0.0-alpha1', '1.1.1.0-alpha1', '1.1.0.0-alpha1-SNAPSHOT', '1.1.1.0-alpha1-SNAPSHOT'],
+        )
+
     def test_compatible_component_versions_snapshot(self) -> None:
         self.assertEqual(
             BuildTarget(version="1.1.2", architecture="x86", patches=["1.1.0", "1.1.1"], snapshot=True).compatible_component_versions,
             ['1.1.2.0-SNAPSHOT', '1.1.0.0', '1.1.1.0', '1.1.0.0-SNAPSHOT', '1.1.1.0-SNAPSHOT'],
         )
 
+    def test_compatible_component_versions_snapshot_qualifier(self) -> None:
+        self.assertEqual(
+            BuildTarget(version="1.1.2", architecture="x86", patches=["1.1.0", "1.1.1"], snapshot=True, qualifier="alpha1").compatible_component_versions,
+            ['1.1.2.0-alpha1-SNAPSHOT', '1.1.0.0-alpha1', '1.1.1.0-alpha1', '1.1.0.0-alpha1-SNAPSHOT', '1.1.1.0-alpha1-SNAPSHOT'],
+        )
+
     def test_component_version_snapshot(self) -> None:
         self.assertEqual(
             BuildTarget(version="1.1.0", architecture="x86", snapshot=True).component_version,