Add pants-plugins/uses_services to check before running tests for required services (like mongo)

.github/workflows/test.yaml

@@ -39,6 +39,12 @@ jobs:
             python-version-short: '3.8'
             python-version: '3.8.10'
 
+    services:
+      mongo:
+        image: mongo:4.4
+        ports:
+          - 27017:27017
+
     env:
       COLUMNS: '120'
 

CHANGELOG.rst

@@ -14,7 +14,7 @@ Added
   working on StackStorm, improve our security posture, and improve CI reliability thanks in part
   to pants' use of PEX lockfiles. This is not a user-facing addition.
   #5778 #5789 #5817 #5795 #5830 #5833 #5834 #5841 #5840 #5838 #5842 #5837 #5849 #5850
-  #5846 #5853 #5848 #5847 #5858 #5857 #5860 #5868 #5871
+  #5846 #5853 #5848 #5847 #5858 #5857 #5860 #5868 #5871 #5864
   Contributed by @cognifloyd
 
 * Added a joint index to solve the problem of slow mongo queries for scheduled executions. #5805

pants-plugins/README.md

@@ -8,6 +8,9 @@ The plugins here add custom goals or other logic into pants.
 
 To see available goals, do "./pants help goals" and "./pants help $goal".
 
+These plugins might be useful outside of the StackStorm project:
+- `uses_services`
+
 These StackStorm-specific plugins might be useful in other StackStorm-related repos.
 - `pack_metadata`
 
@@ -66,3 +69,13 @@ the `fmt` goal (eg `./pants fmt contrib/schemas::`), the schemas will
 be regenerated if any of the files used to generate them have changed.
 Also, running the `lint` goal will fail if the schemas need to be
 regenerated.
+
+### `uses_seevices` plugin
+
+This plugin validates that services are running if required. For example, some tests
+need mongo, so this plugin can ensure mongo is running. If it is not running, then
+an error with instructions on how to run it are given to the user.
+
+`uses_services` has some StackStorm-specific assumptions in it, but it might be
+generalizable. There are several other StackStorm-specific plugins, but some of
+them are only useful in the st2 repo.

pants-plugins/uses_services/BUILD

@@ -0,0 +1,19 @@
+python_sources(
+    sources=["*.py", "!*_test.py", "!data_fixtures.py"],
+)
+
+python_test_utils(
+    name="test_utils",
+    sources=["data_fixtures.py"],
+)
+
+python_tests(
+    name="tests",
+    # Do not enable `uses` for these tests.
+    # These tests still need the running services, but if the plugin
+    # is somehow broken, then any failures would prevent these tests
+    # from running to tell us what is wrong.
+    # overrides={
+    #    "mongo_rules_test.py": {"uses": ["mongo"]},
+    # },
+)

pants-plugins/uses_services/data_fixtures.py

@@ -0,0 +1,144 @@
+# Copyright 2023 The StackStorm Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+from __future__ import annotations
+
+from .platform_rules import Platform
+
+
+def platform(
+    arch="",
+    os="",
+    distro="",
+    distro_name="",
+    distro_codename="",
+    distro_like="",
+    distro_major_version="",
+    distro_version="",
+    mac_release="",
+    win_release="",
+) -> Platform:
+    """Create a Platform with all values defaulted to the empty string."""
+    return Platform(
+        arch=arch,
+        os=os,
+        distro=distro,
+        distro_name=distro_name,
+        distro_codename=distro_codename,
+        distro_like=distro_like,
+        distro_major_version=distro_major_version,
+        distro_version=distro_version,
+        mac_release=mac_release,
+        win_release=win_release,
+    )
+
+
+platform_samples = (
+    platform(),  # empty
+    # EL distros ##################
+    platform(
+        arch="x86_64",
+        os="Linux",
+        distro="centos",
+        distro_name="Centos Linux",
+        distro_codename="Core",
+        distro_like="rhel fedora",
+        distro_major_version="7",
+        distro_version="7",
+    ),
+    platform(
+        arch="x86_64",
+        os="Linux",
+        distro="rocky",
+        distro_name="Rocky Linux",
+        distro_codename="Green Obsidian",
+        distro_like="rhel centos fedora",
+        distro_major_version="8",
+        distro_version="8.7",
+    ),
+    # debian distros ##############
+    platform(
+        arch="x86_64",
+        os="Linux",
+        distro="ubuntu",
+        distro_name="Ubuntu",
+        distro_codename="xenial",
+        distro_like="debian",
+        distro_major_version="16",
+        distro_version="16.04",
+    ),
+    platform(
+        arch="x86_64",
+        os="Linux",
+        distro="ubuntu",
+        distro_name="Ubuntu",
+        distro_codename="bionic",
+        distro_like="debian",
+        distro_major_version="18",
+        distro_version="18.04",
+    ),
+    platform(
+        arch="x86_64",
+        os="Linux",
+        distro="ubuntu",
+        distro_name="Ubuntu",
+        distro_codename="focal",
+        distro_like="debian",
+        distro_major_version="20",
+        distro_version="20.04",
+    ),
+    # other Linux distros #########
+    platform(
+        arch="x86_64",
+        os="Linux",
+        distro="gentoo",
+        distro_name="Gentoo",
+        distro_codename="n/a",
+        distro_major_version="2",
+        distro_version="2.7",
+    ),
+    platform(
+        arch="aarch64",
+        os="Linux",
+        # no distro in termux on android
+    ),
+    # platform(
+    #    arch="x86_64",
+    #    os="Linux",
+    #    distro="",
+    #    distro_name="",
+    #    distro_codename="",
+    #    distro_like="",
+    #    distro_major_version="",
+    #    distro_version="",
+    # ),
+    # Mac OS X ####################
+    platform(
+        arch="x86_64",
+        os="Darwin",
+        distro="darwin",
+        distro_name="Darwin",
+        distro_major_version="19",
+        distro_version="19.6.0",
+        mac_release="10.15.7",
+    ),
+    platform(
+        arch="x86_64",
+        os="Darwin",
+        distro="darwin",
+        distro_name="Darwin",
+        distro_major_version="21",
+        distro_version="21.6.0",
+        mac_release="12.6.2",
+    ),
+)

pants-plugins/uses_services/exceptions.py

@@ -0,0 +1,167 @@
+# Copyright 2023 The StackStorm Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+from __future__ import annotations
+
+from dataclasses import dataclass
+from textwrap import dedent
+
+from uses_services.platform_rules import Platform
+
+
+@dataclass(frozen=True)
+class ServiceSpecificMessages:
+    service: str
+
+    service_start_cmd_el_7: str
+    service_start_cmd_el: str
+    not_installed_clause_el: str
+    install_instructions_el: str
+
+    service_start_cmd_deb: str
+    not_installed_clause_deb: str
+    install_instructions_deb: str
+
+    service_start_cmd_generic: str
+
+
+class ServiceMissingError(Exception):
+    """Error raised when a test uses a service but that service is missing."""
+
+    def __init__(
+        self, service: str, platform: Platform, instructions: str = "", msg=None
+    ):
+        if msg is None:
+            msg = f"The {service} service does not seem to be running or is not accessible!"
+            if instructions:
+                msg += f"\n{instructions}"
+        super().__init__(msg)
+        self.service = service
+        self.platform = platform
+        self.instructions = instructions
+
+    @classmethod
+    def generate(
+        cls, platform: Platform, messages: ServiceSpecificMessages
+    ) -> ServiceMissingError:
+        service = messages.service
+
+        supported = False
+        if platform.distro in ["centos", "rhel"] or "rhel" in platform.distro_like:
+            supported = True
+            if platform.distro_major_version == "7":
+                service_start_cmd = messages.service_start_cmd_el_7
+            else:
+                service_start_cmd = messages.service_start_cmd_el
+            not_installed_clause = messages.not_installed_clause_el
+            install_instructions = messages.install_instructions_el
+
+        elif (
+            platform.distro in ["ubuntu", "debian"] or "debian" in platform.distro_like
+        ):
+            supported = True
+            service_start_cmd = messages.service_start_cmd_deb
+            not_installed_clause = messages.not_installed_clause_deb
+            install_instructions = messages.install_instructions_deb
+
+        if supported:
+            instructions = dedent(
+                f"""\
+                If {service} is installed, but not running try:
+
+                {service_start_cmd}
+
+                If {service} is not installed, {not_installed_clause}:
+
+                """
+            ).format(
+                service=service,
+                service_start_cmd=service_start_cmd,
+                not_installed_clause=not_installed_clause,
+            )
+            instructions += install_instructions
+        elif platform.os == "Linux":
+            instructions = dedent(
+                f"""\
+                You are on Linux using {platform.distro_name}, which is not
+                one of our generally supported distributions. We recommend
+                you use vagrant for local development with something like:
+
+                vagrant init stackstorm/st2
+                vagrant up
+                vagrant ssh
+
+                Please see: https://docs.stackstorm.com/install/vagrant.html
+
+                For anyone who wants to attempt local development without vagrant,
+                you are pretty much on your own. At a minimum you need to install
+                and start {service} with something like:
+
+                {messages.service_start_cmd_generic}
+
+                We would be interested to hear about alternative distros people
+                are using for development. If you are able, please let us know
+                on slack which distro you are using:
+
+                Arch: {platform.arch}
+                Distro: {platform.distro}
+                Distro Name: {platform.distro_name}
+                Distro Codename: {platform.distro_codename}
+                Distro Family: {platform.distro_like}
+                Distro Major Version: {platform.distro_major_version}
+                Distro Version: {platform.distro_version}
+
+                Thanks and Good Luck!
+                """
+            )
+        elif platform.os == "Darwin":  # MacOS
+            instructions = dedent(
+                f"""\
+                You are on Mac OS. Generally we recommend using vagrant for local
+                development on Mac OS with something like:
+
+                vagrant init stackstorm/st2
+                vagrant up
+                vagrant ssh
+
+                Please see: https://docs.stackstorm.com/install/vagrant.html
+
+                For anyone who wants to attempt local development without vagrant,
+                you may run into some speed bumps. Other StackStorm developers have
+                been known to use Mac OS for development, so feel free to ask for
+                help in slack. At a minimum you need to install and start {service}.
+                """
+            )
+        else:
+            instructions = dedent(
+                f"""\
+                You are not on Linux. In this case we recommend using vagrant
+                for local development with something like:
+
+                vagrant init stackstorm/st2
+                vagrant up
+                vagrant ssh
+
+                Please see: https://docs.stackstorm.com/install/vagrant.html
+
+                For anyone who wants to attempt local development without vagrant,
+                you are pretty much on your own. At a minimum you need to install
+                and start {service}. Good luck!
+                """
+            )
+
+        return cls(
+            service=service,
+            platform=platform,
+            instructions=instructions,
+        )