From 36cb91b1a402281c9dd463bee09264977f00d7ce Mon Sep 17 00:00:00 2001 From: Jacob Floyd Date: Wed, 15 Jun 2022 23:12:57 -0500 Subject: [PATCH 01/13] add pants plugin to regenerate contrib/schemas on fmt --- contrib/schemas/BUILD | 5 ++ pants-plugins/schemas/BUILD | 1 + pants-plugins/schemas/__init__.py | 0 pants-plugins/schemas/register.py | 24 ++++++ pants-plugins/schemas/rules.py | 102 ++++++++++++++++++++++++++ pants-plugins/schemas/target_types.py | 37 ++++++++++ pants.toml | 1 + 7 files changed, 170 insertions(+) create mode 100644 contrib/schemas/BUILD create mode 100644 pants-plugins/schemas/BUILD create mode 100644 pants-plugins/schemas/__init__.py create mode 100644 pants-plugins/schemas/register.py create mode 100644 pants-plugins/schemas/rules.py create mode 100644 pants-plugins/schemas/target_types.py diff --git a/contrib/schemas/BUILD b/contrib/schemas/BUILD new file mode 100644 index 0000000000..4e825e22d4 --- /dev/null +++ b/contrib/schemas/BUILD @@ -0,0 +1,5 @@ +schemas( + dependencies=[ + "st2common/bin/st2-generate-schemas", + ], +) diff --git a/pants-plugins/schemas/BUILD b/pants-plugins/schemas/BUILD new file mode 100644 index 0000000000..db46e8d6c9 --- /dev/null +++ b/pants-plugins/schemas/BUILD @@ -0,0 +1 @@ +python_sources() diff --git a/pants-plugins/schemas/__init__.py b/pants-plugins/schemas/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/pants-plugins/schemas/register.py b/pants-plugins/schemas/register.py new file mode 100644 index 0000000000..bdd47f146a --- /dev/null +++ b/pants-plugins/schemas/register.py @@ -0,0 +1,24 @@ +# Copyright 2022 The StackStorm Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from schemas.rules import rules as schemas_rules +from schemas.target_types import Schemas + + +def rules(): + return [*schemas_rules()] + + +def target_types(): + return [Schemas] diff --git a/pants-plugins/schemas/rules.py b/pants-plugins/schemas/rules.py new file mode 100644 index 0000000000..157caa1473 --- /dev/null +++ b/pants-plugins/schemas/rules.py @@ -0,0 +1,102 @@ +# Copyright 2021 The StackStorm Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +from dataclasses import dataclass + +from pants.backend.python.target_types import EntryPoint +from pants.backend.python.util_rules.pex import ( + VenvPex, + VenvPexProcess, +) +from pants.backend.python.util_rules.pex_from_targets import PexFromTargetsRequest +from pants.core.goals.fmt import FmtResult, FmtRequest +from pants.engine.addresses import Address +from pants.engine.fs import ( + Digest, + Snapshot, +) +from pants.engine.process import FallibleProcessResult +from pants.engine.rules import Get, collect_rules, rule +from pants.engine.target import FieldSet +from pants.engine.unions import UnionRule +from pants.util.logging import LogLevel + +from schemas.target_types import SchemasSourcesField + + +CMD = "generate_schemas" + + +@dataclass(frozen=True) +class GenerateSchemasFieldSet(FieldSet): + required_fields = (SchemasSourcesField,) + + sources: SchemasSourcesField + + +class GenerateSchemasViaFmtRequest(FmtRequest): + field_set_type = GenerateSchemasFieldSet + name = CMD + + +@rule( + desc="Update contrib/schemas/*.json with st2-generate-schemas", + level=LogLevel.DEBUG, +) +async def generate_schemas_via_fmt( + request: GenerateSchemasViaFmtRequest, +) -> FmtResult: + # There will only be one target+field_set, but we iterate + # to satisfy how fmt expects that there could be more than one. + + # actually generate it with an external script. + # Generation cannot be inlined here because it needs to import the st2 code. + pex = await Get( + VenvPex, + PexFromTargetsRequest( + [ + Address( + "st2common/st2common/cmd", + target_name="cmd", + relative_file_path=f"{CMD}.py", + ) + ], + output_filename=f"{CMD}.pex", + internal_only=True, + main=EntryPoint.parse("st2common.cmd.{CMD}:main"), + ), + ) + + output_directory = "contrib/schemas" + + result = await Get( + FallibleProcessResult, + VenvPexProcess( + pex, + argv=(output_directory,), + input_digest=request.snapshot.digest, + output_directories=[output_directory], + description="Regenerating st2 metadata schemas in contrib/schemas", + level=LogLevel.DEBUG, + ), + ) + + output_snapshot = await Get(Snapshot, Digest, result.output_digest) + return FmtResult.create(request, result, output_snapshot, strip_chroot_path=True) + + +def rules(): + return [ + *collect_rules(), + UnionRule(FmtRequest, GenerateSchemasViaFmtRequest), + ] diff --git a/pants-plugins/schemas/target_types.py b/pants-plugins/schemas/target_types.py new file mode 100644 index 0000000000..e5c07495d3 --- /dev/null +++ b/pants-plugins/schemas/target_types.py @@ -0,0 +1,37 @@ +# Copyright 2022 The StackStorm Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +from pants.engine.fs import GlobMatchErrorBehavior +from pants.engine.target import ( + COMMON_TARGET_FIELDS, + Dependencies, + MultipleSourcesField, + Target, +) + + +class SchemasSourcesField(MultipleSourcesField): + expected_file_extensions = (".json",) + default = ("*.json",) + uses_source_roots = False + + # make sure at least one schema is present or fmt will be skipped. + default_glob_match_error_behavior = GlobMatchErrorBehavior.error + + +class Schemas(Target): + alias = "schemas" + core_fields = (*COMMON_TARGET_FIELDS, Dependencies, SchemasSourcesField) + help = ( + "Generate st2 metadata (pack, action, rule, ...) schemas from python sources." + ) diff --git a/pants.toml b/pants.toml index c1a48911bf..3de5e4904f 100644 --- a/pants.toml +++ b/pants.toml @@ -24,6 +24,7 @@ backend_packages = [ # internal plugins in pants-plugins/ "pants.backend.plugin_development", + "schemas", ] # pants ignores files in .gitignore, .*/ directories, /dist/ directory, and __pycache__. pants_ignore.add = [ From 42a7d2ff21cd230d8acdfe5fa3a4286f20cadbaa Mon Sep 17 00:00:00 2001 From: Jacob Floyd Date: Fri, 1 Jul 2022 18:01:47 -0500 Subject: [PATCH 02/13] pants-plugins: update pants api usage for latest pants versions --- pants-plugins/schemas/rules.py | 8 ++++---- pants-plugins/schemas/target_types.py | 5 +++++ 2 files changed, 9 insertions(+), 4 deletions(-) diff --git a/pants-plugins/schemas/rules.py b/pants-plugins/schemas/rules.py index 157caa1473..f875e85c5a 100644 --- a/pants-plugins/schemas/rules.py +++ b/pants-plugins/schemas/rules.py @@ -19,7 +19,7 @@ VenvPexProcess, ) from pants.backend.python.util_rules.pex_from_targets import PexFromTargetsRequest -from pants.core.goals.fmt import FmtResult, FmtRequest +from pants.core.goals.fmt import FmtResult, FmtTargetsRequest from pants.engine.addresses import Address from pants.engine.fs import ( Digest, @@ -44,7 +44,7 @@ class GenerateSchemasFieldSet(FieldSet): sources: SchemasSourcesField -class GenerateSchemasViaFmtRequest(FmtRequest): +class GenerateSchemasViaFmtTargetsRequest(FmtTargetsRequest): field_set_type = GenerateSchemasFieldSet name = CMD @@ -54,7 +54,7 @@ class GenerateSchemasViaFmtRequest(FmtRequest): level=LogLevel.DEBUG, ) async def generate_schemas_via_fmt( - request: GenerateSchemasViaFmtRequest, + request: GenerateSchemasViaFmtTargetsRequest, ) -> FmtResult: # There will only be one target+field_set, but we iterate # to satisfy how fmt expects that there could be more than one. @@ -98,5 +98,5 @@ async def generate_schemas_via_fmt( def rules(): return [ *collect_rules(), - UnionRule(FmtRequest, GenerateSchemasViaFmtRequest), + UnionRule(FmtTargetsRequest, GenerateSchemasViaFmtTargetsRequest), ] diff --git a/pants-plugins/schemas/target_types.py b/pants-plugins/schemas/target_types.py index e5c07495d3..5e442a17cf 100644 --- a/pants-plugins/schemas/target_types.py +++ b/pants-plugins/schemas/target_types.py @@ -17,6 +17,7 @@ Dependencies, MultipleSourcesField, Target, + generate_multiple_sources_field_help_message, ) @@ -28,6 +29,10 @@ class SchemasSourcesField(MultipleSourcesField): # make sure at least one schema is present or fmt will be skipped. default_glob_match_error_behavior = GlobMatchErrorBehavior.error + help = generate_multiple_sources_field_help_message( + "Example: `sources=['*.json', '!ignore.json']`" + ) + class Schemas(Target): alias = "schemas" From c5b8ac8e08d09495405cb53afd8e2b26b5947240 Mon Sep 17 00:00:00 2001 From: Jacob Floyd Date: Sun, 11 Dec 2022 21:49:49 -0600 Subject: [PATCH 03/13] add description of schemas plugin to pants-plugins/README.md --- pants-plugins/README.md | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/pants-plugins/README.md b/pants-plugins/README.md index 139d23486d..a20d0b0fa4 100644 --- a/pants-plugins/README.md +++ b/pants-plugins/README.md @@ -7,3 +7,15 @@ This replaces the Makefile and related scripts such that they are more discovera The plugins here add custom goals or other logic into pants. To see available goals, do "./pants help goals" and "./pants help $goal". + +These StackStorm-specific plugins are probably only useful for the st2 repo. +- `schemas` + +### `schemas` plugin + +This plugin wires up pants to make sure `contrib/schemas/*.json` gets +regenerated whenever the source files change. Now, whenever someone runs +the `fmt` goal (eg `./pants fmt contrib/schemas::`), the schemas will +be regenerated if any of the files used to generate them have changed. +Also, running the `lint` goal will fail if the schemas need to be +regenerated. From 67dec998f384fbf9e779ebdab16b6bd27d7ebc1d Mon Sep 17 00:00:00 2001 From: Jacob Floyd Date: Fri, 16 Dec 2022 18:44:15 -0600 Subject: [PATCH 04/13] add pants-plugins test boilerplate for schemas plugins --- pants-plugins/schemas/BUILD | 4 ++ pants-plugins/schemas/rules_test.py | 79 +++++++++++++++++++++++++++++ 2 files changed, 83 insertions(+) create mode 100644 pants-plugins/schemas/rules_test.py diff --git a/pants-plugins/schemas/BUILD b/pants-plugins/schemas/BUILD index db46e8d6c9..0eea8b1cf1 100644 --- a/pants-plugins/schemas/BUILD +++ b/pants-plugins/schemas/BUILD @@ -1 +1,5 @@ python_sources() + +python_tests( + name="tests", +) diff --git a/pants-plugins/schemas/rules_test.py b/pants-plugins/schemas/rules_test.py new file mode 100644 index 0000000000..ce1ef48938 --- /dev/null +++ b/pants-plugins/schemas/rules_test.py @@ -0,0 +1,79 @@ +# Copyright 2022 The StackStorm Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pytest + +from pants.backend.python import target_types_rules +from pants.backend.python.target_types import PythonSourcesGeneratorTarget +from pants.core.util_rules import config_files, source_files +from pants.engine.target import Target +from pants.core.goals.fmt import FmtResult +from pants.testutil.rule_runner import QueryRule, RuleRunner + +from .rules import GenerateSchemasFieldSet, GenerateSchemasViaFmtTargetsRequest, rules as schemas_rules + + +@pytest.fixture +def rule_runner() -> RuleRunner: + return RuleRunner( + rules=[ + *schemas_rules(), + *bandit_subsystem_rules(), + *source_files.rules(), + *config_files.rules(), + *target_types_rules.rules(), + QueryRule(FmtResult, (GenerateSchemasViaFmtTargetsRequest,)), + ], + target_types=[PythonSourcesGeneratorTarget], + ) + + +def run_st2_generate_schemas( + rule_runner: RuleRunner, targets: list[Target], *, extra_args: list[str] | None = None +) -> FmtResult: + rule_runner.set_options( + [ + "--backend-packages=schemas", + *(extra_args or ()), + ], + env_inherit={"PATH", "PYENV_ROOT", "HOME"}, + ) + field_sets = [GenerateSchemasFieldSet.create(tgt) for tgt in targets] + input_sources = rule_runner.request( + SourceFiles, + [ + SourceFilesRequest(field_set.source for field_set in field_sets), + ], + ) + fmt_result = rule_runner.request( + FmtResult, + [ + GenerateSchemasViaFmtTargetsRequest(field_sets, snapshot=input_sources.snapshot), + ], + ) + return results.results + + +# copied from pantsbuild/pants.git/src/python/pants/backend/python/lint/black/rules_integration_test.py +def get_snapshot(rule_runner: RuleRunner, source_files: dict[str, str]) -> Snapshot: + files = [FileContent(path, content.encode()) for path, content in source_files.items()] + digest = rule_runner.request(Digest, [CreateDigest(files)]) + return rule_runner.request(Snapshot, [digest]) + + +def test_something(rule_runner: RuleRunner) -> None: + rule_runner.write_files({"action.json": "{}", "BUILD": "schemas(name='t')"}) + tgt = rule_runner.get_target(Address("", target_name="t", relative_file_path="action.json")) + fmt_result = run_st2_generate_schemas(rule_runner, [tgt]) + # TODO: add asserts From b4819f0437433f6bde3b637af78d6d683f7121f8 Mon Sep 17 00:00:00 2001 From: Jacob Floyd Date: Fri, 16 Dec 2022 20:53:58 -0600 Subject: [PATCH 05/13] improve pants-plugins/schemas/rules_test --- pants-plugins/schemas/rules.py | 2 ++ pants-plugins/schemas/rules_test.py | 13 ++++++------- 2 files changed, 8 insertions(+), 7 deletions(-) diff --git a/pants-plugins/schemas/rules.py b/pants-plugins/schemas/rules.py index f875e85c5a..08009579e1 100644 --- a/pants-plugins/schemas/rules.py +++ b/pants-plugins/schemas/rules.py @@ -14,6 +14,7 @@ from dataclasses import dataclass from pants.backend.python.target_types import EntryPoint +from pants.backend.python.util_rules import pex from pants.backend.python.util_rules.pex import ( VenvPex, VenvPexProcess, @@ -99,4 +100,5 @@ def rules(): return [ *collect_rules(), UnionRule(FmtTargetsRequest, GenerateSchemasViaFmtTargetsRequest), + *pex.rules(), ] diff --git a/pants-plugins/schemas/rules_test.py b/pants-plugins/schemas/rules_test.py index ce1ef48938..f34a3065b0 100644 --- a/pants-plugins/schemas/rules_test.py +++ b/pants-plugins/schemas/rules_test.py @@ -11,17 +11,18 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import annotations import pytest -from pants.backend.python import target_types_rules -from pants.backend.python.target_types import PythonSourcesGeneratorTarget -from pants.core.util_rules import config_files, source_files +from pants.core.util_rules import source_files +from pants.core.util_rules.source_files import SourceFiles, SourceFilesRequest from pants.engine.target import Target from pants.core.goals.fmt import FmtResult from pants.testutil.rule_runner import QueryRule, RuleRunner from .rules import GenerateSchemasFieldSet, GenerateSchemasViaFmtTargetsRequest, rules as schemas_rules +from .target_types import Schemas @pytest.fixture @@ -29,13 +30,11 @@ def rule_runner() -> RuleRunner: return RuleRunner( rules=[ *schemas_rules(), - *bandit_subsystem_rules(), *source_files.rules(), - *config_files.rules(), - *target_types_rules.rules(), QueryRule(FmtResult, (GenerateSchemasViaFmtTargetsRequest,)), + QueryRule(SourceFiles, (SourceFilesRequest,)), ], - target_types=[PythonSourcesGeneratorTarget], + target_types=[Schemas], ) From c313fa3b1861a6cb11053cc1e809957ef67bc9f9 Mon Sep 17 00:00:00 2001 From: Jacob Floyd Date: Fri, 16 Dec 2022 22:37:12 -0600 Subject: [PATCH 06/13] more rules --- pants-plugins/schemas/rules_test.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/pants-plugins/schemas/rules_test.py b/pants-plugins/schemas/rules_test.py index f34a3065b0..bf88b8d07c 100644 --- a/pants-plugins/schemas/rules_test.py +++ b/pants-plugins/schemas/rules_test.py @@ -15,8 +15,12 @@ import pytest +from pants.backend.python import target_types_rules from pants.core.util_rules import source_files +from pants.core.util_rules.archive import rules as archive_rules from pants.core.util_rules.source_files import SourceFiles, SourceFilesRequest +from pants.core.util_rules import system_binaries +from pants.engine.fs import rules as fs_rules from pants.engine.target import Target from pants.core.goals.fmt import FmtResult from pants.testutil.rule_runner import QueryRule, RuleRunner @@ -31,6 +35,10 @@ def rule_runner() -> RuleRunner: rules=[ *schemas_rules(), *source_files.rules(), + *archive_rules(), + *fs_rules(), + *system_binaries.rules(), + *target_types_rules.rules(), QueryRule(FmtResult, (GenerateSchemasViaFmtTargetsRequest,)), QueryRule(SourceFiles, (SourceFilesRequest,)), ], From f61960bb002edb1713e4458b04313167f7e7da27 Mon Sep 17 00:00:00 2001 From: Jacob Floyd Date: Fri, 16 Dec 2022 23:41:15 -0600 Subject: [PATCH 07/13] register required rules for schemas plugin --- pants-plugins/schemas/rules.py | 3 ++- pants-plugins/schemas/rules_test.py | 13 ++----------- 2 files changed, 4 insertions(+), 12 deletions(-) diff --git a/pants-plugins/schemas/rules.py b/pants-plugins/schemas/rules.py index 08009579e1..34eb31981e 100644 --- a/pants-plugins/schemas/rules.py +++ b/pants-plugins/schemas/rules.py @@ -14,7 +14,7 @@ from dataclasses import dataclass from pants.backend.python.target_types import EntryPoint -from pants.backend.python.util_rules import pex +from pants.backend.python.util_rules import pex, pex_from_targets from pants.backend.python.util_rules.pex import ( VenvPex, VenvPexProcess, @@ -101,4 +101,5 @@ def rules(): *collect_rules(), UnionRule(FmtTargetsRequest, GenerateSchemasViaFmtTargetsRequest), *pex.rules(), + *pex_from_targets.rules(), ] diff --git a/pants-plugins/schemas/rules_test.py b/pants-plugins/schemas/rules_test.py index bf88b8d07c..724ffa4cf6 100644 --- a/pants-plugins/schemas/rules_test.py +++ b/pants-plugins/schemas/rules_test.py @@ -15,12 +15,8 @@ import pytest -from pants.backend.python import target_types_rules -from pants.core.util_rules import source_files -from pants.core.util_rules.archive import rules as archive_rules from pants.core.util_rules.source_files import SourceFiles, SourceFilesRequest -from pants.core.util_rules import system_binaries -from pants.engine.fs import rules as fs_rules +from pants.engine.addresses import Address from pants.engine.target import Target from pants.core.goals.fmt import FmtResult from pants.testutil.rule_runner import QueryRule, RuleRunner @@ -34,11 +30,6 @@ def rule_runner() -> RuleRunner: return RuleRunner( rules=[ *schemas_rules(), - *source_files.rules(), - *archive_rules(), - *fs_rules(), - *system_binaries.rules(), - *target_types_rules.rules(), QueryRule(FmtResult, (GenerateSchemasViaFmtTargetsRequest,)), QueryRule(SourceFiles, (SourceFilesRequest,)), ], @@ -60,7 +51,7 @@ def run_st2_generate_schemas( input_sources = rule_runner.request( SourceFiles, [ - SourceFilesRequest(field_set.source for field_set in field_sets), + SourceFilesRequest(field_set.sources for field_set in field_sets), ], ) fmt_result = rule_runner.request( From 51653247bb31e53248aa9e992a718e9b365664d7 Mon Sep 17 00:00:00 2001 From: Jacob Floyd Date: Sat, 17 Dec 2022 00:31:39 -0600 Subject: [PATCH 08/13] get pants-plugins test harness working --- pants-plugins/schemas/rules.py | 3 +- pants-plugins/schemas/rules_test.py | 77 ++++++++++++++++++++++++----- 2 files changed, 68 insertions(+), 12 deletions(-) diff --git a/pants-plugins/schemas/rules.py b/pants-plugins/schemas/rules.py index 34eb31981e..108c826315 100644 --- a/pants-plugins/schemas/rules.py +++ b/pants-plugins/schemas/rules.py @@ -35,6 +35,7 @@ from schemas.target_types import SchemasSourcesField +CMD_DIR = "st2common/st2common/cmd" CMD = "generate_schemas" @@ -67,7 +68,7 @@ async def generate_schemas_via_fmt( PexFromTargetsRequest( [ Address( - "st2common/st2common/cmd", + CMD_DIR, target_name="cmd", relative_file_path=f"{CMD}.py", ) diff --git a/pants-plugins/schemas/rules_test.py b/pants-plugins/schemas/rules_test.py index 724ffa4cf6..d79b58fc57 100644 --- a/pants-plugins/schemas/rules_test.py +++ b/pants-plugins/schemas/rules_test.py @@ -15,13 +15,23 @@ import pytest +from pants.backend.python import target_types_rules +from pants.backend.python.target_types import PythonSourcesGeneratorTarget + +# from pants.core.util_rules import config_files, source_files from pants.core.util_rules.source_files import SourceFiles, SourceFilesRequest from pants.engine.addresses import Address from pants.engine.target import Target from pants.core.goals.fmt import FmtResult from pants.testutil.rule_runner import QueryRule, RuleRunner -from .rules import GenerateSchemasFieldSet, GenerateSchemasViaFmtTargetsRequest, rules as schemas_rules +from .rules import ( + CMD, + CMD_DIR, + GenerateSchemasFieldSet, + GenerateSchemasViaFmtTargetsRequest, + rules as schemas_rules, +) from .target_types import Schemas @@ -30,15 +40,21 @@ def rule_runner() -> RuleRunner: return RuleRunner( rules=[ *schemas_rules(), + # *source_files.rules(), + # *config_files.rules(), + *target_types_rules.rules(), QueryRule(FmtResult, (GenerateSchemasViaFmtTargetsRequest,)), QueryRule(SourceFiles, (SourceFilesRequest,)), ], - target_types=[Schemas], + target_types=[Schemas, PythonSourcesGeneratorTarget], ) def run_st2_generate_schemas( - rule_runner: RuleRunner, targets: list[Target], *, extra_args: list[str] | None = None + rule_runner: RuleRunner, + targets: list[Target], + *, + extra_args: list[str] | None = None, ) -> FmtResult: rule_runner.set_options( [ @@ -57,21 +73,60 @@ def run_st2_generate_schemas( fmt_result = rule_runner.request( FmtResult, [ - GenerateSchemasViaFmtTargetsRequest(field_sets, snapshot=input_sources.snapshot), + GenerateSchemasViaFmtTargetsRequest( + field_sets, snapshot=input_sources.snapshot + ), ], ) - return results.results + return fmt_result # copied from pantsbuild/pants.git/src/python/pants/backend/python/lint/black/rules_integration_test.py -def get_snapshot(rule_runner: RuleRunner, source_files: dict[str, str]) -> Snapshot: - files = [FileContent(path, content.encode()) for path, content in source_files.items()] - digest = rule_runner.request(Digest, [CreateDigest(files)]) - return rule_runner.request(Snapshot, [digest]) +# def get_snapshot(rule_runner: RuleRunner, source_files: dict[str, str]) -> Snapshot: +# files = [ +# FileContent(path, content.encode()) for path, content in source_files.items() +# ] +# digest = rule_runner.request(Digest, [CreateDigest(files)]) +# return rule_runner.request(Snapshot, [digest]) + + +# add dummy script at st2common/st2common/cmd/generate_schemas.py that the test can load. +GENERATE_SCHEMAS_PY = """ +import os + + +def main(): + print('Generated schema for the "dummy" model.') + schema_text = "{schema_text}" + schema_file = os.path.join("{schemas_dir}", "dummy.json") + print('Schema will be written to "%s".' % schema_file) + with open(schema_file, "w") as f: + f.write(schema_text) + f.write("\n") +""" + + +def write_files( + schemas_dir: str, schema_file: str, before: str, after: str, rule_runner: RuleRunner +) -> None: + rule_runner.write_files( + { + f"{schemas_dir}/{schema_file}": before, + f"{schemas_dir}/BUILD": "schemas(name='t')", + # add in the target that's hard-coded in the generate_schemas_via_fmt rue + f"{CMD_DIR}/{CMD}.py": GENERATE_SCHEMAS_PY.format( + schemas_dir=schemas_dir, schema_text=after + ), + f"{CMD_DIR}/BUILD": "python_sources()", + } + ) def test_something(rule_runner: RuleRunner) -> None: - rule_runner.write_files({"action.json": "{}", "BUILD": "schemas(name='t')"}) - tgt = rule_runner.get_target(Address("", target_name="t", relative_file_path="action.json")) + write_files("my_dir", "dummy.json", rule_runner) + + tgt = rule_runner.get_target( + Address("my_dir", target_name="t", relative_file_path="dummy.json") + ) fmt_result = run_st2_generate_schemas(rule_runner, [tgt]) # TODO: add asserts From 31bd708a53b4e199dcb2c7481a5c82f363396783 Mon Sep 17 00:00:00 2001 From: Jacob Floyd Date: Sat, 17 Dec 2022 22:15:52 -0600 Subject: [PATCH 09/13] refactor and add tests for pants-plugins/schemas --- pants-plugins/schemas/rules.py | 45 +++++++++++++++-------- pants-plugins/schemas/rules_test.py | 57 ++++++++++++++++++++++------- 2 files changed, 73 insertions(+), 29 deletions(-) diff --git a/pants-plugins/schemas/rules.py b/pants-plugins/schemas/rules.py index 108c826315..02f9897903 100644 --- a/pants-plugins/schemas/rules.py +++ b/pants-plugins/schemas/rules.py @@ -24,10 +24,11 @@ from pants.engine.addresses import Address from pants.engine.fs import ( Digest, + MergeDigests, Snapshot, ) from pants.engine.process import FallibleProcessResult -from pants.engine.rules import Get, collect_rules, rule +from pants.engine.rules import Get, MultiGet, collect_rules, rule from pants.engine.target import FieldSet from pants.engine.unions import UnionRule from pants.util.logging import LogLevel @@ -35,6 +36,7 @@ from schemas.target_types import SchemasSourcesField +CMD_SOURCE_ROOT = "st2common" CMD_DIR = "st2common/st2common/cmd" CMD = "generate_schemas" @@ -75,26 +77,37 @@ async def generate_schemas_via_fmt( ], output_filename=f"{CMD}.pex", internal_only=True, - main=EntryPoint.parse("st2common.cmd.{CMD}:main"), + main=EntryPoint.parse(f"st2common.cmd.{CMD}:main"), ), ) - output_directory = "contrib/schemas" - - result = await Get( - FallibleProcessResult, - VenvPexProcess( - pex, - argv=(output_directory,), - input_digest=request.snapshot.digest, - output_directories=[output_directory], - description="Regenerating st2 metadata schemas in contrib/schemas", - level=LogLevel.DEBUG, - ), + output_directories = [fs.address.spec_path for fs in request.field_sets] + + results = await MultiGet( + Get( + FallibleProcessResult, + VenvPexProcess( + pex, + argv=(output_directory,), + # This script actually ignores the input files. + input_digest=request.snapshot.digest, + output_directories=[output_directory], + description=f"Regenerating st2 metadata schemas in {output_directory}", + level=LogLevel.DEBUG, + ), + ) + for output_directory in output_directories + ) + + output_snapshot = await Get( + Snapshot, MergeDigests(result.output_digest for result in results) ) - output_snapshot = await Get(Snapshot, Digest, result.output_digest) - return FmtResult.create(request, result, output_snapshot, strip_chroot_path=True) + # TODO: Use more than just the first process result. + # Only file changes for extra runs get preserved. + return FmtResult.create( + request, results[0], output_snapshot, strip_chroot_path=True + ) def rules(): diff --git a/pants-plugins/schemas/rules_test.py b/pants-plugins/schemas/rules_test.py index d79b58fc57..2245ffa6e5 100644 --- a/pants-plugins/schemas/rules_test.py +++ b/pants-plugins/schemas/rules_test.py @@ -18,9 +18,9 @@ from pants.backend.python import target_types_rules from pants.backend.python.target_types import PythonSourcesGeneratorTarget -# from pants.core.util_rules import config_files, source_files from pants.core.util_rules.source_files import SourceFiles, SourceFilesRequest from pants.engine.addresses import Address +from pants.engine.fs import CreateDigest, Digest, FileContent, Snapshot from pants.engine.target import Target from pants.core.goals.fmt import FmtResult from pants.testutil.rule_runner import QueryRule, RuleRunner @@ -28,6 +28,7 @@ from .rules import ( CMD, CMD_DIR, + CMD_SOURCE_ROOT, GenerateSchemasFieldSet, GenerateSchemasViaFmtTargetsRequest, rules as schemas_rules, @@ -40,8 +41,6 @@ def rule_runner() -> RuleRunner: return RuleRunner( rules=[ *schemas_rules(), - # *source_files.rules(), - # *config_files.rules(), *target_types_rules.rules(), QueryRule(FmtResult, (GenerateSchemasViaFmtTargetsRequest,)), QueryRule(SourceFiles, (SourceFilesRequest,)), @@ -59,6 +58,7 @@ def run_st2_generate_schemas( rule_runner.set_options( [ "--backend-packages=schemas", + "--source-root-patterns=/st2common", *(extra_args or ()), ], env_inherit={"PATH", "PYENV_ROOT", "HOME"}, @@ -82,12 +82,12 @@ def run_st2_generate_schemas( # copied from pantsbuild/pants.git/src/python/pants/backend/python/lint/black/rules_integration_test.py -# def get_snapshot(rule_runner: RuleRunner, source_files: dict[str, str]) -> Snapshot: -# files = [ -# FileContent(path, content.encode()) for path, content in source_files.items() -# ] -# digest = rule_runner.request(Digest, [CreateDigest(files)]) -# return rule_runner.request(Snapshot, [digest]) +def get_snapshot(rule_runner: RuleRunner, source_files: dict[str, str]) -> Snapshot: + files = [ + FileContent(path, content.encode()) for path, content in source_files.items() + ] + digest = rule_runner.request(Digest, [CreateDigest(files)]) + return rule_runner.request(Snapshot, [digest]) # add dummy script at st2common/st2common/cmd/generate_schemas.py that the test can load. @@ -102,7 +102,6 @@ def main(): print('Schema will be written to "%s".' % schema_file) with open(schema_file, "w") as f: f.write(schema_text) - f.write("\n") """ @@ -117,16 +116,48 @@ def write_files( f"{CMD_DIR}/{CMD}.py": GENERATE_SCHEMAS_PY.format( schemas_dir=schemas_dir, schema_text=after ), + f"{CMD_DIR}/__init__.py": "", # st2common/st2common/cmd/ + f"{CMD_DIR}/../__init__.py": "", # st2common/st2common/ f"{CMD_DIR}/BUILD": "python_sources()", } ) -def test_something(rule_runner: RuleRunner) -> None: - write_files("my_dir", "dummy.json", rule_runner) +def test_changed(rule_runner: RuleRunner) -> None: + write_files( + schemas_dir="my_dir", + schema_file="dummy.json", + before="BEFORE", + after="AFTER", + rule_runner=rule_runner, + ) tgt = rule_runner.get_target( Address("my_dir", target_name="t", relative_file_path="dummy.json") ) fmt_result = run_st2_generate_schemas(rule_runner, [tgt]) - # TODO: add asserts + assert f'Schema will be written to "my_dir/dummy.json".' in fmt_result.stdout + assert fmt_result.output == get_snapshot( + rule_runner, {"my_dir/dummy.json": "AFTER"} + ) + assert fmt_result.did_change is True + + +def test_unchanged(rule_runner: RuleRunner) -> None: + write_files( + schemas_dir="my_dir", + schema_file="dummy.json", + before="AFTER", + after="AFTER", + rule_runner=rule_runner, + ) + + tgt = rule_runner.get_target( + Address("my_dir", target_name="t", relative_file_path="dummy.json") + ) + fmt_result = run_st2_generate_schemas(rule_runner, [tgt]) + assert f'Schema will be written to "my_dir/dummy.json".' in fmt_result.stdout + assert fmt_result.output == get_snapshot( + rule_runner, {"my_dir/dummy.json": "AFTER"} + ) + assert fmt_result.did_change is False From 25ebdee550fb9b2d92bdce8ab88d1b6938f507a2 Mon Sep 17 00:00:00 2001 From: Jacob Floyd Date: Sat, 17 Dec 2022 22:26:30 -0600 Subject: [PATCH 10/13] do not hard code path to __init__.py files in test --- pants-plugins/schemas/rules.py | 10 ++++---- pants-plugins/schemas/rules_test.py | 37 ++++++++++++++++------------- 2 files changed, 25 insertions(+), 22 deletions(-) diff --git a/pants-plugins/schemas/rules.py b/pants-plugins/schemas/rules.py index 02f9897903..eec528dd5e 100644 --- a/pants-plugins/schemas/rules.py +++ b/pants-plugins/schemas/rules.py @@ -22,11 +22,7 @@ from pants.backend.python.util_rules.pex_from_targets import PexFromTargetsRequest from pants.core.goals.fmt import FmtResult, FmtTargetsRequest from pants.engine.addresses import Address -from pants.engine.fs import ( - Digest, - MergeDigests, - Snapshot, -) +from pants.engine.fs import MergeDigests, Snapshot from pants.engine.process import FallibleProcessResult from pants.engine.rules import Get, MultiGet, collect_rules, rule from pants.engine.target import FieldSet @@ -36,8 +32,10 @@ from schemas.target_types import SchemasSourcesField +# these constants are also used in the tests. CMD_SOURCE_ROOT = "st2common" CMD_DIR = "st2common/st2common/cmd" +CMD_MODULE = "st2common.cmd" CMD = "generate_schemas" @@ -77,7 +75,7 @@ async def generate_schemas_via_fmt( ], output_filename=f"{CMD}.pex", internal_only=True, - main=EntryPoint.parse(f"st2common.cmd.{CMD}:main"), + main=EntryPoint.parse(f"{CMD_MODULE}.{CMD}:main"), ), ) diff --git a/pants-plugins/schemas/rules_test.py b/pants-plugins/schemas/rules_test.py index 2245ffa6e5..60e709c674 100644 --- a/pants-plugins/schemas/rules_test.py +++ b/pants-plugins/schemas/rules_test.py @@ -13,6 +13,8 @@ # limitations under the License. from __future__ import annotations +import os + import pytest from pants.backend.python import target_types_rules @@ -58,7 +60,7 @@ def run_st2_generate_schemas( rule_runner.set_options( [ "--backend-packages=schemas", - "--source-root-patterns=/st2common", + f"--source-root-patterns=/{CMD_SOURCE_ROOT}", *(extra_args or ()), ], env_inherit={"PATH", "PYENV_ROOT", "HOME"}, @@ -108,19 +110,22 @@ def main(): def write_files( schemas_dir: str, schema_file: str, before: str, after: str, rule_runner: RuleRunner ) -> None: - rule_runner.write_files( - { - f"{schemas_dir}/{schema_file}": before, - f"{schemas_dir}/BUILD": "schemas(name='t')", - # add in the target that's hard-coded in the generate_schemas_via_fmt rue - f"{CMD_DIR}/{CMD}.py": GENERATE_SCHEMAS_PY.format( - schemas_dir=schemas_dir, schema_text=after - ), - f"{CMD_DIR}/__init__.py": "", # st2common/st2common/cmd/ - f"{CMD_DIR}/../__init__.py": "", # st2common/st2common/ - f"{CMD_DIR}/BUILD": "python_sources()", - } - ) + files = { + f"{schemas_dir}/{schema_file}": before, + f"{schemas_dir}/BUILD": "schemas(name='t')", + # add in the target that's hard-coded in the generate_schemas_via_fmt rue + f"{CMD_DIR}/{CMD}.py": GENERATE_SCHEMAS_PY.format( + schemas_dir=schemas_dir, schema_text=after + ), + f"{CMD_DIR}/BUILD": "python_sources()", + } + + module = CMD_DIR + while module != CMD_SOURCE_ROOT: + files[f"{module}/__init__.py"] = "" + module = os.path.dirname(module) + + rule_runner.write_files(files) def test_changed(rule_runner: RuleRunner) -> None: @@ -136,7 +141,7 @@ def test_changed(rule_runner: RuleRunner) -> None: Address("my_dir", target_name="t", relative_file_path="dummy.json") ) fmt_result = run_st2_generate_schemas(rule_runner, [tgt]) - assert f'Schema will be written to "my_dir/dummy.json".' in fmt_result.stdout + assert 'Schema will be written to "my_dir/dummy.json".' in fmt_result.stdout assert fmt_result.output == get_snapshot( rule_runner, {"my_dir/dummy.json": "AFTER"} ) @@ -156,7 +161,7 @@ def test_unchanged(rule_runner: RuleRunner) -> None: Address("my_dir", target_name="t", relative_file_path="dummy.json") ) fmt_result = run_st2_generate_schemas(rule_runner, [tgt]) - assert f'Schema will be written to "my_dir/dummy.json".' in fmt_result.stdout + assert 'Schema will be written to "my_dir/dummy.json".' in fmt_result.stdout assert fmt_result.output == get_snapshot( rule_runner, {"my_dir/dummy.json": "AFTER"} ) From 2ef7ae1ffef88d642aa302dea29c4aa71dd8cd46 Mon Sep 17 00:00:00 2001 From: Jacob Floyd Date: Sun, 18 Dec 2022 15:13:26 -0600 Subject: [PATCH 11/13] clean up FmtResult creation --- pants-plugins/schemas/rules.py | 24 ++++++++++++++++-------- 1 file changed, 16 insertions(+), 8 deletions(-) diff --git a/pants-plugins/schemas/rules.py b/pants-plugins/schemas/rules.py index eec528dd5e..0f0b646856 100644 --- a/pants-plugins/schemas/rules.py +++ b/pants-plugins/schemas/rules.py @@ -28,6 +28,7 @@ from pants.engine.target import FieldSet from pants.engine.unions import UnionRule from pants.util.logging import LogLevel +from pants.util.strutil import strip_v2_chroot_path from schemas.target_types import SchemasSourcesField @@ -58,10 +59,7 @@ class GenerateSchemasViaFmtTargetsRequest(FmtTargetsRequest): async def generate_schemas_via_fmt( request: GenerateSchemasViaFmtTargetsRequest, ) -> FmtResult: - # There will only be one target+field_set, but we iterate - # to satisfy how fmt expects that there could be more than one. - - # actually generate it with an external script. + # We use a pex to actually generate the schemas with an external script. # Generation cannot be inlined here because it needs to import the st2 code. pex = await Get( VenvPex, @@ -79,6 +77,8 @@ async def generate_schemas_via_fmt( ), ) + # There will probably only be one target+field_set, but we iterate + # to satisfy how fmt expects that there could be more than one. output_directories = [fs.address.spec_path for fs in request.field_sets] results = await MultiGet( @@ -101,10 +101,18 @@ async def generate_schemas_via_fmt( Snapshot, MergeDigests(result.output_digest for result in results) ) - # TODO: Use more than just the first process result. - # Only file changes for extra runs get preserved. - return FmtResult.create( - request, results[0], output_snapshot, strip_chroot_path=True + stdout = "\n".join( + [strip_v2_chroot_path(process_result.stdout) for process_result in results] + ) + stderr = "\n".join( + [strip_v2_chroot_path(process_result.stderr) for process_result in results] + ) + return FmtResult( + input=request.snapshot, + output=output_snapshot, + stdout=stdout, + stderr=stderr, + formatter_name=request.name, ) From e0ee91f6d5bc726e55b17d9f6e73e00689832d34 Mon Sep 17 00:00:00 2001 From: Jacob Floyd Date: Wed, 14 Dec 2022 15:14:10 -0600 Subject: [PATCH 12/13] update changelog entry --- CHANGELOG.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index e14c784128..b9fc5ae46e 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -14,7 +14,7 @@ Added working on StackStorm, improve our security posture, and improve CI reliability thanks in part to pants' use of PEX lockfiles. This is not a user-facing addition. #5778 #5789 #5817 #5795 #5830 #5833 #5834 #5841 #5840 #5838 #5842 #5837 #5849 #5850 - #5846 #5853 #5848 + #5846 #5853 #5848 #5847 Contributed by @cognifloyd * Added a joint index to solve the problem of slow mongo queries for scheduled executions. #5805 From 60efbe4f6ae3950f3d1294997991ae34c247cf9a Mon Sep 17 00:00:00 2001 From: Jacob Floyd Date: Thu, 5 Jan 2023 11:11:49 -0600 Subject: [PATCH 13/13] update pants-plugins/schemas copyright to 2023 --- pants-plugins/schemas/register.py | 2 +- pants-plugins/schemas/rules.py | 2 +- pants-plugins/schemas/rules_test.py | 2 +- pants-plugins/schemas/target_types.py | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/pants-plugins/schemas/register.py b/pants-plugins/schemas/register.py index bdd47f146a..e6f91c9708 100644 --- a/pants-plugins/schemas/register.py +++ b/pants-plugins/schemas/register.py @@ -1,4 +1,4 @@ -# Copyright 2022 The StackStorm Authors. +# Copyright 2023 The StackStorm Authors. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/pants-plugins/schemas/rules.py b/pants-plugins/schemas/rules.py index 0f0b646856..4b49e8c3b6 100644 --- a/pants-plugins/schemas/rules.py +++ b/pants-plugins/schemas/rules.py @@ -1,4 +1,4 @@ -# Copyright 2021 The StackStorm Authors. +# Copyright 2023 The StackStorm Authors. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/pants-plugins/schemas/rules_test.py b/pants-plugins/schemas/rules_test.py index 60e709c674..308db057a6 100644 --- a/pants-plugins/schemas/rules_test.py +++ b/pants-plugins/schemas/rules_test.py @@ -1,4 +1,4 @@ -# Copyright 2022 The StackStorm Authors. +# Copyright 2023 The StackStorm Authors. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/pants-plugins/schemas/target_types.py b/pants-plugins/schemas/target_types.py index 5e442a17cf..f255a8780f 100644 --- a/pants-plugins/schemas/target_types.py +++ b/pants-plugins/schemas/target_types.py @@ -1,4 +1,4 @@ -# Copyright 2022 The StackStorm Authors. +# Copyright 2023 The StackStorm Authors. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License.