diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index cc236afdb9..8391949d30 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -39,6 +39,12 @@ jobs: python-version-short: '3.8' python-version: '3.8.10' + services: + mongo: + image: mongo:4.4 + ports: + - 27017:27017 + env: COLUMNS: '120' diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 3d37b236c2..7274fdbbdd 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -14,7 +14,7 @@ Added working on StackStorm, improve our security posture, and improve CI reliability thanks in part to pants' use of PEX lockfiles. This is not a user-facing addition. #5778 #5789 #5817 #5795 #5830 #5833 #5834 #5841 #5840 #5838 #5842 #5837 #5849 #5850 - #5846 #5853 #5848 #5847 #5858 #5857 #5860 #5868 #5871 + #5846 #5853 #5848 #5847 #5858 #5857 #5860 #5868 #5871 #5864 Contributed by @cognifloyd * Added a joint index to solve the problem of slow mongo queries for scheduled executions. #5805 diff --git a/pants-plugins/README.md b/pants-plugins/README.md index 8f99aa3d5e..b995e2b6d5 100644 --- a/pants-plugins/README.md +++ b/pants-plugins/README.md @@ -8,6 +8,9 @@ The plugins here add custom goals or other logic into pants. To see available goals, do "./pants help goals" and "./pants help $goal". +These plugins might be useful outside of the StackStorm project: +- `uses_services` + These StackStorm-specific plugins might be useful in other StackStorm-related repos. - `pack_metadata` @@ -66,3 +69,13 @@ the `fmt` goal (eg `./pants fmt contrib/schemas::`), the schemas will be regenerated if any of the files used to generate them have changed. Also, running the `lint` goal will fail if the schemas need to be regenerated. + +### `uses_seevices` plugin + +This plugin validates that services are running if required. For example, some tests +need mongo, so this plugin can ensure mongo is running. If it is not running, then +an error with instructions on how to run it are given to the user. + +`uses_services` has some StackStorm-specific assumptions in it, but it might be +generalizable. There are several other StackStorm-specific plugins, but some of +them are only useful in the st2 repo. diff --git a/pants-plugins/uses_services/BUILD b/pants-plugins/uses_services/BUILD new file mode 100644 index 0000000000..19808c0b61 --- /dev/null +++ b/pants-plugins/uses_services/BUILD @@ -0,0 +1,19 @@ +python_sources( + sources=["*.py", "!*_test.py", "!data_fixtures.py"], +) + +python_test_utils( + name="test_utils", + sources=["data_fixtures.py"], +) + +python_tests( + name="tests", + # Do not enable `uses` for these tests. + # These tests still need the running services, but if the plugin + # is somehow broken, then any failures would prevent these tests + # from running to tell us what is wrong. + # overrides={ + # "mongo_rules_test.py": {"uses": ["mongo"]}, + # }, +) diff --git a/pants-plugins/uses_services/__init__.py b/pants-plugins/uses_services/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/pants-plugins/uses_services/data_fixtures.py b/pants-plugins/uses_services/data_fixtures.py new file mode 100644 index 0000000000..39304cb2fb --- /dev/null +++ b/pants-plugins/uses_services/data_fixtures.py @@ -0,0 +1,144 @@ +# Copyright 2023 The StackStorm Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +from __future__ import annotations + +from .platform_rules import Platform + + +def platform( + arch="", + os="", + distro="", + distro_name="", + distro_codename="", + distro_like="", + distro_major_version="", + distro_version="", + mac_release="", + win_release="", +) -> Platform: + """Create a Platform with all values defaulted to the empty string.""" + return Platform( + arch=arch, + os=os, + distro=distro, + distro_name=distro_name, + distro_codename=distro_codename, + distro_like=distro_like, + distro_major_version=distro_major_version, + distro_version=distro_version, + mac_release=mac_release, + win_release=win_release, + ) + + +platform_samples = ( + platform(), # empty + # EL distros ################## + platform( + arch="x86_64", + os="Linux", + distro="centos", + distro_name="Centos Linux", + distro_codename="Core", + distro_like="rhel fedora", + distro_major_version="7", + distro_version="7", + ), + platform( + arch="x86_64", + os="Linux", + distro="rocky", + distro_name="Rocky Linux", + distro_codename="Green Obsidian", + distro_like="rhel centos fedora", + distro_major_version="8", + distro_version="8.7", + ), + # debian distros ############## + platform( + arch="x86_64", + os="Linux", + distro="ubuntu", + distro_name="Ubuntu", + distro_codename="xenial", + distro_like="debian", + distro_major_version="16", + distro_version="16.04", + ), + platform( + arch="x86_64", + os="Linux", + distro="ubuntu", + distro_name="Ubuntu", + distro_codename="bionic", + distro_like="debian", + distro_major_version="18", + distro_version="18.04", + ), + platform( + arch="x86_64", + os="Linux", + distro="ubuntu", + distro_name="Ubuntu", + distro_codename="focal", + distro_like="debian", + distro_major_version="20", + distro_version="20.04", + ), + # other Linux distros ######### + platform( + arch="x86_64", + os="Linux", + distro="gentoo", + distro_name="Gentoo", + distro_codename="n/a", + distro_major_version="2", + distro_version="2.7", + ), + platform( + arch="aarch64", + os="Linux", + # no distro in termux on android + ), + # platform( + # arch="x86_64", + # os="Linux", + # distro="", + # distro_name="", + # distro_codename="", + # distro_like="", + # distro_major_version="", + # distro_version="", + # ), + # Mac OS X #################### + platform( + arch="x86_64", + os="Darwin", + distro="darwin", + distro_name="Darwin", + distro_major_version="19", + distro_version="19.6.0", + mac_release="10.15.7", + ), + platform( + arch="x86_64", + os="Darwin", + distro="darwin", + distro_name="Darwin", + distro_major_version="21", + distro_version="21.6.0", + mac_release="12.6.2", + ), +) diff --git a/pants-plugins/uses_services/exceptions.py b/pants-plugins/uses_services/exceptions.py new file mode 100644 index 0000000000..6da31cf46b --- /dev/null +++ b/pants-plugins/uses_services/exceptions.py @@ -0,0 +1,167 @@ +# Copyright 2023 The StackStorm Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +from __future__ import annotations + +from dataclasses import dataclass +from textwrap import dedent + +from uses_services.platform_rules import Platform + + +@dataclass(frozen=True) +class ServiceSpecificMessages: + service: str + + service_start_cmd_el_7: str + service_start_cmd_el: str + not_installed_clause_el: str + install_instructions_el: str + + service_start_cmd_deb: str + not_installed_clause_deb: str + install_instructions_deb: str + + service_start_cmd_generic: str + + +class ServiceMissingError(Exception): + """Error raised when a test uses a service but that service is missing.""" + + def __init__( + self, service: str, platform: Platform, instructions: str = "", msg=None + ): + if msg is None: + msg = f"The {service} service does not seem to be running or is not accessible!" + if instructions: + msg += f"\n{instructions}" + super().__init__(msg) + self.service = service + self.platform = platform + self.instructions = instructions + + @classmethod + def generate( + cls, platform: Platform, messages: ServiceSpecificMessages + ) -> ServiceMissingError: + service = messages.service + + supported = False + if platform.distro in ["centos", "rhel"] or "rhel" in platform.distro_like: + supported = True + if platform.distro_major_version == "7": + service_start_cmd = messages.service_start_cmd_el_7 + else: + service_start_cmd = messages.service_start_cmd_el + not_installed_clause = messages.not_installed_clause_el + install_instructions = messages.install_instructions_el + + elif ( + platform.distro in ["ubuntu", "debian"] or "debian" in platform.distro_like + ): + supported = True + service_start_cmd = messages.service_start_cmd_deb + not_installed_clause = messages.not_installed_clause_deb + install_instructions = messages.install_instructions_deb + + if supported: + instructions = dedent( + f"""\ + If {service} is installed, but not running try: + + {service_start_cmd} + + If {service} is not installed, {not_installed_clause}: + + """ + ).format( + service=service, + service_start_cmd=service_start_cmd, + not_installed_clause=not_installed_clause, + ) + instructions += install_instructions + elif platform.os == "Linux": + instructions = dedent( + f"""\ + You are on Linux using {platform.distro_name}, which is not + one of our generally supported distributions. We recommend + you use vagrant for local development with something like: + + vagrant init stackstorm/st2 + vagrant up + vagrant ssh + + Please see: https://docs.stackstorm.com/install/vagrant.html + + For anyone who wants to attempt local development without vagrant, + you are pretty much on your own. At a minimum you need to install + and start {service} with something like: + + {messages.service_start_cmd_generic} + + We would be interested to hear about alternative distros people + are using for development. If you are able, please let us know + on slack which distro you are using: + + Arch: {platform.arch} + Distro: {platform.distro} + Distro Name: {platform.distro_name} + Distro Codename: {platform.distro_codename} + Distro Family: {platform.distro_like} + Distro Major Version: {platform.distro_major_version} + Distro Version: {platform.distro_version} + + Thanks and Good Luck! + """ + ) + elif platform.os == "Darwin": # MacOS + instructions = dedent( + f"""\ + You are on Mac OS. Generally we recommend using vagrant for local + development on Mac OS with something like: + + vagrant init stackstorm/st2 + vagrant up + vagrant ssh + + Please see: https://docs.stackstorm.com/install/vagrant.html + + For anyone who wants to attempt local development without vagrant, + you may run into some speed bumps. Other StackStorm developers have + been known to use Mac OS for development, so feel free to ask for + help in slack. At a minimum you need to install and start {service}. + """ + ) + else: + instructions = dedent( + f"""\ + You are not on Linux. In this case we recommend using vagrant + for local development with something like: + + vagrant init stackstorm/st2 + vagrant up + vagrant ssh + + Please see: https://docs.stackstorm.com/install/vagrant.html + + For anyone who wants to attempt local development without vagrant, + you are pretty much on your own. At a minimum you need to install + and start {service}. Good luck! + """ + ) + + return cls( + service=service, + platform=platform, + instructions=instructions, + ) diff --git a/pants-plugins/uses_services/mongo_rules.py b/pants-plugins/uses_services/mongo_rules.py new file mode 100644 index 0000000000..ad8333132f --- /dev/null +++ b/pants-plugins/uses_services/mongo_rules.py @@ -0,0 +1,204 @@ +# Copyright 2023 The StackStorm Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import os + +from dataclasses import dataclass +from textwrap import dedent + +from pants.backend.python.goals.pytest_runner import ( + PytestPluginSetupRequest, + PytestPluginSetup, +) +from pants.backend.python.util_rules.pex import ( + PexRequest, + PexRequirements, + VenvPex, + VenvPexProcess, + rules as pex_rules, +) +from pants.engine.fs import CreateDigest, Digest, FileContent +from pants.engine.rules import collect_rules, Get, MultiGet, rule +from pants.engine.process import FallibleProcessResult, ProcessCacheScope +from pants.engine.target import Target +from pants.engine.unions import UnionRule +from pants.util.logging import LogLevel + +from uses_services.exceptions import ServiceMissingError, ServiceSpecificMessages +from uses_services.platform_rules import Platform +from uses_services.scripts.is_mongo_running import ( + __file__ as is_mongo_running_full_path, +) +from uses_services.target_types import UsesServicesField + + +@dataclass(frozen=True) +class UsesMongoRequest: + """One or more targets need a running mongo service using these settings. + + The db_* attributes represent the db connection settings from st2.conf. + In st2 code, they come from: + oslo_config.cfg.CONF.database.{host,port,db_name,connection_timeout} + """ + + # These config opts currently hard-coded in: + # for unit tests: st2tests/st2tests/config.py + # for integration tests: conf/st2.tests*.conf st2tests/st2tests/fixtures/conf/st2.tests*.conf + # (changed by setting ST2_CONFIG_PATH env var inside the tests) + # TODO: for unit tests: modify code to pull db connect settings from env vars + # TODO: for int tests: modify st2.tests*.conf on the fly to set the per-pantsd-slot db_name + # and either add env vars for db connect settings or modify conf files as well + + # with our version of oslo.config (newer are slower) we can't directly override opts w/ environment variables. + + db_host: str = "127.0.0.1" # localhost in test_db.DbConnectionTestCase + db_port: int = 27017 + # db_name is "st2" in test_db.DbConnectionTestCase + db_name: str = f"st2-test{os.environ.get('ST2TESTS_PARALLEL_SLOT', '')}" + db_connection_timeout: int = 3000 + + +@dataclass(frozen=True) +class MongoIsRunning: + pass + + +class PytestUsesMongoRequest(PytestPluginSetupRequest): + @classmethod + def is_applicable(cls, target: Target) -> bool: + if not target.has_field(UsesServicesField): + return False + uses = target.get(UsesServicesField).value + return uses is not None and "mongo" in uses + + +@rule( + desc="Ensure mongodb is running and accessible before running tests.", + level=LogLevel.DEBUG, +) +async def mongo_is_running_for_pytest( + request: PytestUsesMongoRequest, +) -> PytestPluginSetup: + # TODO: delete these comments once the Makefile becomes irrelevant. + # the comments explore how the Makefile prepares to run and runs tests + + # The st2-test database gets dropped between (in Makefile based testing): + # - each component (st2*/ && various config/ dirs) in Makefile + # - DbTestCase/CleanDbTestCase setUpClass + + # Makefile + # .run-unit-tests-coverage (<- .combine-unit-tests-coverage <- .coverage.unit <- .unit-tests-coverage-html <- ci-unit <- ci) + # echo "----- Dropping st2-test db -----" + # mongo st2-test --eval "db.dropDatabase();" + # for component in $(COMPONENTS_TEST) + # nosetests $(NOSE_OPTS) -s -v $(NOSE_COVERAGE_FLAGS) $(NOSE_COVERAGE_PACKAGES) $$component/tests/unit + + # this will raise an error if mongo is not running + _ = await Get(MongoIsRunning, UsesMongoRequest()) + + return PytestPluginSetup() + + +@rule( + desc="Test to see if mongodb is running and accessible.", + level=LogLevel.DEBUG, +) +async def mongo_is_running( + request: UsesMongoRequest, platform: Platform +) -> MongoIsRunning: + script_path = "./is_mongo_running.py" + + # pants is already watching this directory as it is under a source root. + # So, we don't need to double watch with PathGlobs, just open it. + with open(is_mongo_running_full_path, "rb") as script_file: + script_contents = script_file.read() + + script_digest, mongoengine_pex = await MultiGet( + Get(Digest, CreateDigest([FileContent(script_path, script_contents)])), + Get( + VenvPex, + PexRequest( + output_filename="mongoengine.pex", + internal_only=True, + requirements=PexRequirements({"mongoengine", "pymongo"}), + ), + ), + ) + + result = await Get( + FallibleProcessResult, + VenvPexProcess( + mongoengine_pex, + argv=( + script_path, + request.db_host, + str(request.db_port), + request.db_name, + str(request.db_connection_timeout), + ), + input_digest=script_digest, + description="Checking to see if Mongo is up and accessible.", + # this can change from run to run, so don't cache results. + cache_scope=ProcessCacheScope.PER_SESSION, + level=LogLevel.DEBUG, + ), + ) + is_running = result.exit_code == 0 + + if is_running: + return MongoIsRunning() + + # mongo is not running, so raise an error with instructions. + raise ServiceMissingError.generate( + platform=platform, + messages=ServiceSpecificMessages( + service="mongo", + service_start_cmd_el_7="service mongo start", + service_start_cmd_el="systemctl start mongod", + not_installed_clause_el="this is one way to install it:", + install_instructions_el=dedent( + """\ + # Add key and repo for the latest stable MongoDB (4.0) + sudo rpm --import https://www.mongodb.org/static/pgp/server-4.0.asc + sudo sh -c "cat < /etc/yum.repos.d/mongodb-org-4.repo + [mongodb-org-4] + name=MongoDB Repository + baseurl=https://repo.mongodb.org/yum/redhat/${OSRELEASE_VERSION}/mongodb-org/4.0/x86_64/ + gpgcheck=1 + enabled=1 + gpgkey=https://www.mongodb.org/static/pgp/server-4.0.asc + EOT" + # Install mongo + sudo yum -y install mongodb-org + # Don't forget to start mongo. + """ + ), + service_start_cmd_deb="systemctl start mongod", + not_installed_clause_deb="this is one way to install it:", + install_instructions_deb=dedent( + """\ + sudo apt-get install -y mongodb-org + # Don't forget to start mongo. + """ + ), + service_start_cmd_generic="systemctl start mongod", + ), + ) + + +def rules(): + return [ + *collect_rules(), + UnionRule(PytestPluginSetupRequest, PytestUsesMongoRequest), + *pex_rules(), + ] diff --git a/pants-plugins/uses_services/mongo_rules_test.py b/pants-plugins/uses_services/mongo_rules_test.py new file mode 100644 index 0000000000..ca4b15a2f0 --- /dev/null +++ b/pants-plugins/uses_services/mongo_rules_test.py @@ -0,0 +1,91 @@ +# Copyright 2023 The StackStorm Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +from __future__ import annotations + +import pytest + +from pants.engine.internals.scheduler import ExecutionError +from pants.testutil.rule_runner import QueryRule, RuleRunner + +from .data_fixtures import platform, platform_samples +from .exceptions import ServiceMissingError +from .mongo_rules import ( + MongoIsRunning, + UsesMongoRequest, + rules as mongo_rules, +) +from .platform_rules import Platform + + +@pytest.fixture +def rule_runner() -> RuleRunner: + return RuleRunner( + rules=[ + *mongo_rules(), + QueryRule(MongoIsRunning, (UsesMongoRequest, Platform)), + ], + target_types=[], + ) + + +def run_mongo_is_running( + rule_runner: RuleRunner, + uses_mongo_request: UsesMongoRequest, + mock_platform: Platform, + *, + extra_args: list[str] | None = None, +) -> MongoIsRunning: + rule_runner.set_options( + [ + "--backend-packages=uses_services", + *(extra_args or ()), + ], + env_inherit={"PATH", "PYENV_ROOT", "HOME"}, + ) + result = rule_runner.request( + MongoIsRunning, + [uses_mongo_request, mock_platform], + ) + return result + + +# Warning this requires that mongo be running +def test_mongo_is_running(rule_runner: RuleRunner) -> None: + request = UsesMongoRequest() + mock_platform = platform() + + # we are asserting that this does not raise an exception + is_running = run_mongo_is_running(rule_runner, request, mock_platform) + assert is_running + + +@pytest.mark.parametrize("mock_platform", platform_samples) +def test_mongo_not_running(rule_runner: RuleRunner, mock_platform: Platform) -> None: + request = UsesMongoRequest( + db_host="127.100.20.7", + db_port=10, # unassigned port, unlikely to be used + ) + + with pytest.raises(ExecutionError) as exception_info: + run_mongo_is_running(rule_runner, request, mock_platform) + + execution_error = exception_info.value + assert len(execution_error.wrapped_exceptions) == 1 + + exc = execution_error.wrapped_exceptions[0] + assert isinstance(exc, ServiceMissingError) + + assert exc.service == "mongo" + assert "The mongo service does not seem to be running" in str(exc) + assert exc.instructions != "" diff --git a/pants-plugins/uses_services/platform_rules.py b/pants-plugins/uses_services/platform_rules.py new file mode 100644 index 0000000000..92408d8ca8 --- /dev/null +++ b/pants-plugins/uses_services/platform_rules.py @@ -0,0 +1,84 @@ +# Copyright 2023 The StackStorm Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import json + +from pants.backend.python.util_rules.pex import ( + PexRequest, + PexRequirements, + VenvPex, + VenvPexProcess, + rules as pex_rules, +) +from pants.engine.fs import CreateDigest, Digest, FileContent +from pants.engine.process import ProcessCacheScope, ProcessResult +from pants.engine.rules import collect_rules, Get, MultiGet, rule +from pants.util.logging import LogLevel + +# noinspection PyProtectedMember +from uses_services.scripts.inspect_platform import ( + Platform, + __file__ as inspect_platform_full_path, +) + +__all__ = ["Platform", "get_platform", "rules"] + + +@rule( + desc="Get details (os, distro, etc) about platform running tests.", + level=LogLevel.DEBUG, +) +async def get_platform() -> Platform: + script_path = "./inspect_platform.py" + + # pants is already watching this directory as it is under a source root. + # So, we don't need to double watch with PathGlobs, just open it. + with open(inspect_platform_full_path, "rb") as script_file: + script_contents = script_file.read() + + script_digest, distro_pex = await MultiGet( + Get( + Digest, + CreateDigest([FileContent(script_path, script_contents)]), + ), + Get( + VenvPex, + PexRequest( + output_filename="distro.pex", + internal_only=True, + requirements=PexRequirements({"distro"}), + ), + ), + ) + + result = await Get( + ProcessResult, + VenvPexProcess( + distro_pex, + argv=(script_path,), + input_digest=script_digest, + description="Introspecting platform (arch, os, distro)", + # this can change from run to run, so don't cache results. + cache_scope=ProcessCacheScope.PER_RESTART_SUCCESSFUL, + level=LogLevel.DEBUG, + ), + ) + platform = json.loads(result.stdout) + return Platform(**platform) + + +def rules(): + return [ + *collect_rules(), + *pex_rules(), + ] diff --git a/pants-plugins/uses_services/platform_rules_test.py b/pants-plugins/uses_services/platform_rules_test.py new file mode 100644 index 0000000000..f1c07ac087 --- /dev/null +++ b/pants-plugins/uses_services/platform_rules_test.py @@ -0,0 +1,49 @@ +# Copyright 2023 The StackStorm Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +from __future__ import annotations + +import dataclasses + +import pytest + +from pants.testutil.rule_runner import QueryRule, RuleRunner + +from .platform_rules import Platform, rules as platform_rules + + +@pytest.fixture +def rule_runner() -> RuleRunner: + return RuleRunner( + rules=[ + *platform_rules(), + QueryRule(Platform, ()), + ], + target_types=[], + ) + + +def test_get_platform(rule_runner: RuleRunner) -> None: + rule_runner.set_options( + ["--backend-packages=uses_services"], + env_inherit={"PATH", "PYENV_ROOT", "HOME"}, + ) + + platform = rule_runner.request(Platform, ()) + + assert isinstance(platform, Platform) + assert dataclasses.is_dataclass(platform) + # there isn't a good way to inject mocks into the script that + # the rule_runner runs in a venv. So, there isn't a nice way + # to test the values of the Platform fields as people could + # run tests on any platform. diff --git a/pants-plugins/uses_services/register.py b/pants-plugins/uses_services/register.py new file mode 100644 index 0000000000..5ca4363ab3 --- /dev/null +++ b/pants-plugins/uses_services/register.py @@ -0,0 +1,29 @@ +# Copyright 2023 The StackStorm Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +from pants.backend.python.target_types import ( + PythonTestTarget, + PythonTestsGeneratorTarget, +) + +from uses_services import mongo_rules, platform_rules +from uses_services.target_types import UsesServicesField + + +def rules(): + return [ + PythonTestsGeneratorTarget.register_plugin_field(UsesServicesField), + PythonTestTarget.register_plugin_field(UsesServicesField), + *platform_rules.rules(), + *mongo_rules.rules(), + ] diff --git a/pants-plugins/uses_services/scripts/BUILD b/pants-plugins/uses_services/scripts/BUILD new file mode 100644 index 0000000000..db46e8d6c9 --- /dev/null +++ b/pants-plugins/uses_services/scripts/BUILD @@ -0,0 +1 @@ +python_sources() diff --git a/pants-plugins/uses_services/scripts/__init__.py b/pants-plugins/uses_services/scripts/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/pants-plugins/uses_services/scripts/inspect_platform.py b/pants-plugins/uses_services/scripts/inspect_platform.py new file mode 100644 index 0000000000..383535c9b1 --- /dev/null +++ b/pants-plugins/uses_services/scripts/inspect_platform.py @@ -0,0 +1,57 @@ +# Copyright 2023 The StackStorm Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import json + +from dataclasses import asdict, dataclass + +__all__ = ["Platform"] + + +@dataclass(frozen=True) +class Platform: + arch: str + os: str + distro: str + distro_name: str + distro_codename: str + distro_like: str + distro_major_version: str + distro_version: str + mac_release: str + win_release: str + + +def _get_platform() -> Platform: + # late import so that Platform can be imported in the pants plugin as well + import distro + import platform + + return Platform( + arch=platform.machine(), # x86_64 + os=platform.system(), # Linux, Darwin + distro=distro.id(), # rhel, ubuntu, centos, gentoo, darwin + distro_name=distro.name(), # Ubuntu, Centos Linux, Gentoo, Darwin + distro_codename=distro.codename(), # xenial, Core, n/a, '' + distro_like=distro.like(), # debian, rhel fedora, '', '' + distro_major_version=distro.major_version(), # 16, 7, 2, 19 + distro_version=distro.version(), # 16.04, 7, 2.7, 19.6.0 + mac_release=platform.mac_ver()[0], # '', 10.15.7 + win_release=platform.win32_ver()[0], # '' + ) + + +if __name__ == "__main__": + platform = _get_platform() + platform_dict = asdict(platform) + print(json.dumps(platform_dict)) diff --git a/pants-plugins/uses_services/scripts/is_mongo_running.py b/pants-plugins/uses_services/scripts/is_mongo_running.py new file mode 100644 index 0000000000..8d5ecfce8a --- /dev/null +++ b/pants-plugins/uses_services/scripts/is_mongo_running.py @@ -0,0 +1,58 @@ +# Copyright 2023 The StackStorm Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import sys + + +def _is_mongo_running( + db_host: str, db_port: int, db_name: str, connection_timeout_ms: int +) -> bool: + """Connect to mongo with connection logic that mirrors the st2 code. + + In particular, this is based on st2common.models.db.db_setup(). + This should not import the st2 code as it should be self-contained. + """ + # late import so that __file__ can be imported in the pants plugin without these imports + import mongoengine + from pymongo.errors import ConnectionFailure + from pymongo.errors import ServerSelectionTimeoutError + + connection = mongoengine.connection.connect( + db_name, + host=db_host, + port=db_port, + connectTimeoutMS=connection_timeout_ms, + serverSelectionTimeoutMS=connection_timeout_ms, + ) + + # connection.connect() is lazy. Make a command to test the connection. + try: + # The ismaster command is cheap and does not require auth + connection.admin.command("ismaster") + except (ConnectionFailure, ServerSelectionTimeoutError): + return False + return True + + +if __name__ == "__main__": + args = dict((k, v) for k, v in enumerate(sys.argv)) + db_host = args.get(1, "127.0.0.1") + db_port = args.get(2, 27017) + db_name = args.get(3, "st2-test") + connection_timeout_ms = args.get(4, 3000) + + is_running = _is_mongo_running( + db_host, int(db_port), db_name, int(connection_timeout_ms) + ) + exit_code = 0 if is_running else 1 + sys.exit(exit_code) diff --git a/pants-plugins/uses_services/target_types.py b/pants-plugins/uses_services/target_types.py new file mode 100644 index 0000000000..5723ceb9ae --- /dev/null +++ b/pants-plugins/uses_services/target_types.py @@ -0,0 +1,23 @@ +# Copyright 2023 The StackStorm Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +from pants.engine.target import StringSequenceField + + +supported_services = ("mongo", "rabbitmq", "redis") + + +class UsesServicesField(StringSequenceField): + alias = "uses" + help = "Define the services that a test target depends on (mongo, rabbitmq, redis)." + valid_choices = supported_services diff --git a/pants.toml b/pants.toml index 375f9a1843..86b9499a34 100644 --- a/pants.toml +++ b/pants.toml @@ -28,6 +28,7 @@ backend_packages = [ "pack_metadata", "sample_conf", "schemas", + "uses_services", ] # pants ignores files in .gitignore, .*/ directories, /dist/ directory, and __pycache__. pants_ignore.add = [ diff --git a/st2common/tests/unit/BUILD b/st2common/tests/unit/BUILD index fe53c9f265..bc4f9fb0b4 100644 --- a/st2common/tests/unit/BUILD +++ b/st2common/tests/unit/BUILD @@ -9,6 +9,7 @@ python_tests( # several files import tests.unit.base which is ambiguous. Tell pants which one to use. "st2common/tests/unit/base.py", ], + uses=["mongo"], ) python_sources() diff --git a/st2common/tests/unit/services/BUILD b/st2common/tests/unit/services/BUILD index 57341b1358..44c0254066 100644 --- a/st2common/tests/unit/services/BUILD +++ b/st2common/tests/unit/services/BUILD @@ -1,3 +1,4 @@ python_tests( name="tests", + uses=["mongo"], )