Add qualifier support for ci workflows in build repo

src/ci_workflow/ci_check_gradle_dependencies.py

@@ -23,13 +23,17 @@ def __init__(self, component: InputComponent, git_repo: GitRepository, target: C
 
     def __get_dependencies(self) -> PropertiesFile:
         cmd = " ".join(
-            [
-                f"./gradlew {self.gradle_project or ''}:dependencies",
-                f"-Dopensearch.version={self.target.opensearch_version}",
-                f"-Dbuild.snapshot={str(self.target.snapshot).lower()}",
-                "--configuration compileOnly",
-                '| grep -e "---"',
-            ]
+            filter(
+                None,
+                [
+                    f"./gradlew {self.gradle_project or ''}:dependencies",
+                    f"-Dopensearch.version={self.target.opensearch_version}",
+                    f"-Dbuild.snapshot={str(self.target.snapshot).lower()}",
+                    f"-Dbuild.version_qualifier={str(self.target.qualifier)}" if self.target.qualifier else None,
+                    "--configuration compileOnly",
+                    '| grep -e "---"',
+                ]
+            )
         )
 
         lines = self.git_repo.output(cmd)

src/ci_workflow/ci_check_gradle_properties.py

@@ -20,11 +20,15 @@ def __init__(self, component: Component, git_repo: GitRepository, target: CiTarg
 
     def __get_properties(self) -> PropertiesFile:
         cmd = " ".join(
-            [
-                "./gradlew properties",
-                f"-Dopensearch.version={self.target.opensearch_version}",
-                f"-Dbuild.snapshot={str(self.target.snapshot).lower()}",
-            ]
+            filter(
+                None,
+                [
+                    "./gradlew properties",
+                    f"-Dopensearch.version={self.target.opensearch_version}",
+                    f"-Dbuild.snapshot={str(self.target.snapshot).lower()}",
+                    f"-Dbuild.version_qualifier={str(self.target.qualifier)}" if self.target.qualifier else None,
+                ]
+            )
         )
 
         return PropertiesFile(self.git_repo.output(cmd))

src/ci_workflow/ci_check_gradle_publish_to_maven_local.py

@@ -10,11 +10,15 @@
 class CiCheckGradlePublishToMavenLocal(CiCheckSource):
     def check(self) -> None:
         cmd = " ".join(
-            [
-                "./gradlew publishToMavenLocal",
-                f"-Dopensearch.version={self.target.opensearch_version}",
-                f"-Dbuild.snapshot={str(self.target.snapshot).lower()}",
-            ]
+            filter(
+                None,
+                [
+                    "./gradlew publishToMavenLocal",
+                    f"-Dopensearch.version={self.target.opensearch_version}",
+                    f"-Dbuild.snapshot={str(self.target.snapshot).lower()}",
+                    f"-Dbuild.version_qualifier={str(self.target.qualifier)}" if self.target.qualifier else None,
+                ]
+            )
         )
 
         self.git_repo.execute(cmd)

src/ci_workflow/ci_check_npm_package_version.py

@@ -13,9 +13,9 @@ class CiCheckNpmPackageVersion(CiCheckPackage):
     @property
     def checked_version(self) -> str:
         if self.component.name == "OpenSearch-Dashboards":
-            return self.target.opensearch_version.replace("-SNAPSHOT", "")
+            return self.target.opensearch_version.split("-")[0]
         else:
-            return self.target.component_version.replace("-SNAPSHOT", "")
+            return self.target.component_version.split("-")[0]
 
     def check(self) -> None:
         self.properties.check_value("version", self.checked_version)

src/ci_workflow/ci_input_manifest.py

@@ -21,7 +21,12 @@ def __init__(self, file: TextIOWrapper, args: CiArgs) -> None:
 
     def __check__(self) -> None:
 
-        target = CiTarget(version=self.manifest.build.version, name=self.manifest.build.filename, snapshot=self.args.snapshot)
+        target = CiTarget(
+            version=self.manifest.build.version,
+            name=self.manifest.build.filename,
+            qualifier=self.manifest.build.qualifier if self.manifest.build.qualifier else None,
+            snapshot=self.args.snapshot
+        )
 
         with TemporaryDirectory(keep=self.args.keep, chdir=True) as work_dir:
             logging.info(f"Sanity-testing in {work_dir.name}")

src/ci_workflow/ci_target.py

@@ -10,16 +10,19 @@ class CiTarget:
     name: str
     snapshot: bool
 
-    def __init__(self, version: str, name: str, snapshot: bool = True) -> None:
+    def __init__(self, version: str, name: str, qualifier: str, snapshot: bool = True) -> None:
         self.version = version
         self.name = name
+        self.qualifier = qualifier
         self.snapshot = snapshot
 
     @property
     def opensearch_version(self) -> str:
-        return self.version + "-SNAPSHOT" if self.snapshot else self.version
+        os_version = self.version + f"-{self.qualifier}" if self.qualifier else self.version
+        return os_version + "-SNAPSHOT" if self.snapshot else os_version
 
     @property
     def component_version(self) -> str:
         # BUG: the 4th digit is dictated by the component, it's not .0, this will break for 1.1.0.1
-        return self.version + ".0-SNAPSHOT" if self.snapshot else f"{self.version}.0"
+        comp_version = f"{self.version}.0" + f"-{self.qualifier}" if self.qualifier else f"{self.version}.0"
+        return comp_version + "-SNAPSHOT" if self.snapshot else comp_version

tests/tests_ci_workflow/test_ci_check_gradle_dependencies.py

@@ -18,14 +18,14 @@ class DummyDependencies(CiCheckGradleDependencies):
         def check(self) -> None:
             pass
 
-    def __mock_dependencies(self, props: str = "", snapshot: bool = False, gradle_project: Any = None) -> DummyDependencies:
+    def __mock_dependencies(self, props: str = "", qualifier: Any = None, snapshot: bool = False, gradle_project: Any = None) -> DummyDependencies:
         git_repo = MagicMock()
         git_repo.output.return_value = props
 
         return TestCiCheckGradleDependencies.DummyDependencies(
             component=MagicMock(),
             git_repo=git_repo,
-            target=CiTarget(version="1.1.0", name="opensearch", snapshot=snapshot),
+            target=CiTarget(version="1.1.0", name="opensearch", qualifier=None, snapshot=snapshot),
             args=gradle_project,
         )
 
@@ -41,13 +41,27 @@ def test_executes_gradle_dependencies_snapshot(self) -> None:
             './gradlew :dependencies -Dopensearch.version=1.1.0-SNAPSHOT -Dbuild.snapshot=true --configuration compileOnly | grep -e "---"'
         )
 
+    def test_executes_gradle_dependencies_qualifier_snapshot(self) -> None:
+        check = self.__mock_dependencies(qualifier="alpha1", snapshot=True)
+        output = unittest.mock.create_autospec(check.git_repo.output)
+        output.assert_called_once_with(
+            './gradlew :dependencies -Dopensearch.version=1.1.0-alpha1-SNAPSHOT -Dbuild.snapshot=true -Dbuild.version_qualifier=alpha1 --configuration compileOnly | grep -e "---"'
+        )
+
     def test_executes_gradle_dependencies_project(self) -> None:
         check = self.__mock_dependencies(snapshot=True, gradle_project="project")
         output = unittest.mock.create_autospec(check.git_repo.output)
         output.assert_called_once_with(
             './gradlew project:dependencies -Dopensearch.version=1.1.0-SNAPSHOT -Dbuild.snapshot=true --configuration compileOnly | grep -e "---"'
         )
 
+    def test_executes_gradle_dependencies_project_qualifier(self) -> None:
+        check = self.__mock_dependencies(qualifier="alpha1", snapshot=True, gradle_project="project")
+        output = unittest.mock.create_autospec(check.git_repo.output)
+        output.assert_called_once_with(
+            './gradlew project:dependencies -Dopensearch.version=1.1.0-alpha1-SNAPSHOT -Dbuild.snapshot=true -Dbuild.version_qualifier=alpha1 --configuration compileOnly | grep -e "---"'
+        )
+
     def test_loads_tree(self) -> None:
         data_path = os.path.join(os.path.dirname(__file__), "data", "job_scheduler_dependencies.txt")
         with open(data_path) as f:

tests/tests_ci_workflow/test_ci_check_gradle_dependencies_opensearch.py

@@ -23,7 +23,7 @@ def __mock_check(self, props: Any = None) -> CiCheckGradleDependenciesOpenSearch
             return CiCheckGradleDependenciesOpenSearchVersion(
                 component=MagicMock(),
                 git_repo=MagicMock(),
-                target=CiTarget(version="1.1.0", name="opensearch", snapshot=True),
+                target=CiTarget(version="1.1.0", name="opensearch", qualifier=None, snapshot=True),
                 args=None,
             )
 
@@ -53,22 +53,46 @@ def test_executes_gradle_command(self) -> None:
         check = CiCheckGradleDependenciesOpenSearchVersion(
             component=MagicMock(),
             git_repo=MagicMock(),
-            target=CiTarget(version="1.1.0", name="opensearch", snapshot=True),
+            target=CiTarget(version="1.1.0", name="opensearch", qualifier=None, snapshot=True),
             args=None,
         )
         output = unittest.mock.create_autospec(check.git_repo.output)
         output.assert_called_once_with(
             './gradlew :dependencies -Dopensearch.version=1.1.0-SNAPSHOT -Dbuild.snapshot=true --configuration compileOnly | grep -e "---"'
         )
 
+    def test_executes_gradle_command_qualifier(self) -> None:
+        check = CiCheckGradleDependenciesOpenSearchVersion(
+            component=MagicMock(),
+            git_repo=MagicMock(),
+            target=CiTarget(version="2.0.0", name="opensearch", qualifier="alpha1", snapshot=True),
+            args=None,
+        )
+        output = unittest.mock.create_autospec(check.git_repo.output)
+        output.assert_called_once_with(
+            './gradlew :dependencies -Dopensearch.version=2.0.0-alpha1-SNAPSHOT -Dbuild.snapshot=true -Dbuild.version_qualifier=alpha1 --configuration compileOnly | grep -e "---"'
+        )
+
     def test_executes_gradle_command_with_arg(self) -> None:
         check = CiCheckGradleDependenciesOpenSearchVersion(
             component=MagicMock(),
             git_repo=MagicMock(),
-            target=CiTarget(version="1.1.0", name="opensearch", snapshot=True),
+            target=CiTarget(version="1.1.0", name="opensearch", qualifier=None, snapshot=True),
             args="plugin",
         )
         output = unittest.mock.create_autospec(check.git_repo.output)
         output.assert_called_once_with(
             './gradlew plugin:dependencies -Dopensearch.version=1.1.0-SNAPSHOT -Dbuild.snapshot=true --configuration compileOnly | grep -e "---"'
         )
+
+    def test_executes_gradle_command_qualifier_with_arg(self) -> None:
+        check = CiCheckGradleDependenciesOpenSearchVersion(
+            component=MagicMock(),
+            git_repo=MagicMock(),
+            target=CiTarget(version="2.0.0", name="opensearch", qualifier="alpha1", snapshot=True),
+            args="plugin",
+        )
+        output = unittest.mock.create_autospec(check.git_repo.output)
+        output.assert_called_once_with(
+            './gradlew plugin:dependencies -Dopensearch.version=2.0.0-alpha1-SNAPSHOT -Dbuild.snapshot=true -Dbuild.version_qualifier=alpha1 --configuration compileOnly | grep -e "---"'
+        )

tests/tests_ci_workflow/test_ci_check_gradle_properties.py

@@ -23,7 +23,7 @@ def test_executes_gradle_properties(self) -> None:
         TestCiCheckGradleProperties.DummyProperties(
             component=MagicMock(),
             git_repo=git_repo,
-            target=CiTarget(version="1.1.0", name="opensearch", snapshot=False),
+            target=CiTarget(version="1.1.0", name="opensearch", qualifier=None, snapshot=False),
         )
 
         git_repo.output.assert_called_once_with("./gradlew properties -Dopensearch.version=1.1.0 -Dbuild.snapshot=false")
@@ -35,7 +35,19 @@ def test_executes_gradle_properties_snapshot(self) -> None:
         TestCiCheckGradleProperties.DummyProperties(
             component=MagicMock(),
             git_repo=git_repo,
-            target=CiTarget(version="1.1.0", name="opensearch", snapshot=True),
+            target=CiTarget(version="1.1.0", name="opensearch", qualifier=None, snapshot=True),
         )
 
         git_repo.output.assert_called_once_with("./gradlew properties -Dopensearch.version=1.1.0-SNAPSHOT -Dbuild.snapshot=true")
+
+    def test_executes_gradle_properties_qualifier_snapshot(self) -> None:
+        git_repo = MagicMock()
+        git_repo.output.return_value = ""
+
+        TestCiCheckGradleProperties.DummyProperties(
+            component=MagicMock(),
+            git_repo=git_repo,
+            target=CiTarget(version="2.0.0", name="opensearch", qualifier="alpha1", snapshot=True),
+        )
+
+        git_repo.output.assert_called_once_with("./gradlew properties -Dopensearch.version=2.0.0-alpha1-SNAPSHOT -Dbuild.snapshot=true -Dbuild.version_qualifier=alpha1")

tests/tests_ci_workflow/test_ci_check_gradle_properties_version.py

@@ -21,56 +21,56 @@ def __mock_check(self, props: Any = None, component: Component = None, snapshot:
             return CiCheckGradlePropertiesVersion(
                 component=component or MagicMock(),
                 git_repo=MagicMock(),
-                target=CiTarget(version="1.1.0", name="opensearch", snapshot=snapshot),
+                target=CiTarget(version="2.0.0", name="opensearch", qualifier="alpha1", snapshot=snapshot),
             )
 
     def test_has_version(self) -> None:
-        self.__mock_check({"version": "1.1.0.0-SNAPSHOT"}).check()
+        self.__mock_check({"version": "2.0.0.0-alpha1-SNAPSHOT"}).check()
 
     def test_missing_version(self) -> None:
         with self.assertRaises(PropertiesFile.UnexpectedKeyValueError) as err:
             self.__mock_check().check()
         self.assertEqual(
             str(err.exception),
-            "Expected to have version='1.1.0.0-SNAPSHOT', but none was found.",
+            "Expected to have version='2.0.0.0-alpha1-SNAPSHOT', but none was found.",
         )
 
     def test_invalid_version(self) -> None:
         with self.assertRaises(PropertiesFile.UnexpectedKeyValueError) as err:
-            self.__mock_check({"version": "1.2.0-SNAPSHOT"}).check()
+            self.__mock_check({"version": "2.1.0-SNAPSHOT"}).check()
         self.assertEqual(
             str(err.exception),
-            "Expected to have version='1.1.0.0-SNAPSHOT', but was '1.2.0-SNAPSHOT'.",
+            "Expected to have version='2.0.0.0-alpha1-SNAPSHOT', but was '2.1.0-SNAPSHOT'.",
         )
 
     def test_component_version_opensearch(self) -> None:
         check = self.__mock_check(
-            props={"version": "1.1.0.0-SNAPSHOT"},
+            props={"version": "2.0.0.0-alpha1-SNAPSHOT"},
             component=Component({"name": "OpenSearch", "repository": "", "ref": ""}),
         )
 
-        self.assertEqual(check.checked_version, "1.1.0-SNAPSHOT")
+        self.assertEqual(check.checked_version, "2.0.0-alpha1-SNAPSHOT")
 
         with self.assertRaises(PropertiesFile.UnexpectedKeyValueError) as err:
             check.check()
 
         self.assertEqual(
             str(err.exception),
-            "Expected to have version='1.1.0-SNAPSHOT', but was '1.1.0.0-SNAPSHOT'.",
+            "Expected to have version='2.0.0-alpha1-SNAPSHOT', but was '2.0.0.0-alpha1-SNAPSHOT'.",
         )
 
     def test_component_version(self) -> None:
         check = self.__mock_check(
-            props={"version": "1.1.0-SNAPSHOT"},
+            props={"version": "2.0.0-alpha1-SNAPSHOT"},
             component=Component({"name": "Plugin", "repository": "", "ref": ""}),
         )
 
-        self.assertEqual(check.checked_version, "1.1.0.0-SNAPSHOT")
+        self.assertEqual(check.checked_version, "2.0.0.0-alpha1-SNAPSHOT")
 
         with self.assertRaises(PropertiesFile.UnexpectedKeyValueError) as err:
             check.check()
 
         self.assertEqual(
             str(err.exception),
-            "Expected to have version='1.1.0.0-SNAPSHOT', but was '1.1.0-SNAPSHOT'.",
+            "Expected to have version='2.0.0.0-alpha1-SNAPSHOT', but was '2.0.0-alpha1-SNAPSHOT'.",
         )

tests/tests_ci_workflow/test_ci_check_gradle_publish_to_maven_local.py

@@ -16,7 +16,7 @@ def test_executes_gradle_command(self) -> None:
         check = CiCheckGradlePublishToMavenLocal(
             component=MagicMock(),
             git_repo=MagicMock(),
-            target=CiTarget(version="1.1.0", name="opensearch", snapshot=False),
+            target=CiTarget(version="1.1.0", name="opensearch", qualifier=None, snapshot=False),
         )
         check.check()
         exec_command = unittest.mock.create_autospec(check.git_repo.execute)
@@ -26,8 +26,18 @@ def test_executes_gradle_command_snapshot(self) -> None:
         check = CiCheckGradlePublishToMavenLocal(
             component=MagicMock(),
             git_repo=MagicMock(),
-            target=CiTarget(version="1.1.0", name="opensearch", snapshot=True),
+            target=CiTarget(version="1.1.0", name="opensearch", qualifier=None, snapshot=True),
         )
         check.check()
         exec_command = unittest.mock.create_autospec(check.git_repo.execute)
         exec_command.assert_called_once_with("./gradlew publishToMavenLocal -Dopensearch.version=1.1.0-SNAPSHOT -Dbuild.snapshot=true")
+
+    def test_executes_gradle_command_qualifier_snapshot(self) -> None:
+        check = CiCheckGradlePublishToMavenLocal(
+            component=MagicMock(),
+            git_repo=MagicMock(),
+            target=CiTarget(version="2.0.0", name="opensearch", qualifier="alpha1", snapshot=True),
+        )
+        check.check()
+        exec_command = unittest.mock.create_autospec(check.git_repo.execute)
+        exec_command.assert_called_once_with("./gradlew publishToMavenLocal -Dopensearch.version=2.0.0-alpha1-SNAPSHOT -Dbuild.snapshot=true -Dbuild.version_qualifier=alpha1")

tests/tests_ci_workflow/test_ci_check_lists_dist.py

@@ -23,14 +23,14 @@ class TestCiCheckListsDist(unittest.TestCase):
     def test_check(self, mock_manifest_from_url: Mock, find_build_root: Mock) -> None:
         mock_manifest_from_url.return_value = BuildManifest.from_path(self.BUILD_MANIFEST)
         component = InputComponentFromDist({"name": "common-utils", "dist": "url", "checks": ["manifest:component"]})
-        list = CiCheckListDist(component, CiTarget(version="1.1.0", name="opensearch", snapshot=True))
+        list = CiCheckListDist(component, CiTarget(version="1.1.0", name="opensearch", qualifier=None, snapshot=True))
         list.check()
         mock_manifest_from_url.assert_called()
         find_build_root.assert_called()
 
     def test_invalid_check(self, *mocks: Mock) -> None:
         component = InputComponentFromDist({"name": "common-utils", "dist": "url", "checks": ["invalid:check"]})
-        list = CiCheckListDist(component, CiTarget(version="1.1.0", name="opensearch", snapshot=True))
+        list = CiCheckListDist(component, CiTarget(version="1.1.0", name="opensearch", qualifier=None, snapshot=True))
         list.checkout("path")
         with self.assertRaises(CiCheckListDist.InvalidCheckError) as ctx:
             list.check()

tests/tests_ci_workflow/test_ci_check_manifest_component.py

@@ -23,7 +23,7 @@ class TestCiCheckManifestComponent(unittest.TestCase):
     def test_retrieves_manifests(self, mock_manifest: Mock, find_build_root: Mock) -> None:
         find_build_root.return_value = "url/linux/ARCH/builds/opensearch"
         check = CiCheckManifestComponent(
-            InputComponentFromDist({"name": "common-utils", "dist": "url"}), CiTarget(version="1.1.0", name="opensearch", snapshot=True)
+            InputComponentFromDist({"name": "common-utils", "dist": "url"}), CiTarget(version="1.1.0", name="opensearch", qualifier=None, snapshot=True)
         )
 
         mock_manifest.from_url.return_value = BuildManifest.from_path(self.BUILD_MANIFEST)
@@ -47,7 +47,7 @@ def test_retrieves_manifests(self, mock_manifest: Mock, find_build_root: Mock) -
     def test_missing_component(self, mock_manifest: Mock, find_build_root: Mock) -> None:
         find_build_root.return_value = "url/linux/x64/builds/opensearch"
         check = CiCheckManifestComponent(
-            InputComponentFromDist({"name": "does-not-exist", "dist": "url"}), CiTarget(version="1.1.0", name="opensearch", snapshot=True)
+            InputComponentFromDist({"name": "does-not-exist", "dist": "url"}), CiTarget(version="1.1.0", name="opensearch", qualifier=None, snapshot=True)
         )
 
         mock_manifest.from_url.return_value = BuildManifest.from_path(self.BUILD_MANIFEST)