Add pants-plugins/schemas to streamline regenerating contrib/schemas

CHANGELOG.rst

@@ -14,7 +14,7 @@ Added
   working on StackStorm, improve our security posture, and improve CI reliability thanks in part
   to pants' use of PEX lockfiles. This is not a user-facing addition.
   #5778 #5789 #5817 #5795 #5830 #5833 #5834 #5841 #5840 #5838 #5842 #5837 #5849 #5850
-  #5846 #5853 #5848
+  #5846 #5853 #5848 #5847
   Contributed by @cognifloyd
 
 * Added a joint index to solve the problem of slow mongo queries for scheduled executions. #5805

contrib/schemas/BUILD

@@ -0,0 +1,5 @@
+schemas(
+    dependencies=[
+        "st2common/bin/st2-generate-schemas",
+    ],
+)

pants-plugins/README.md

@@ -7,3 +7,15 @@ This replaces the Makefile and related scripts such that they are more discovera
 The plugins here add custom goals or other logic into pants.
 
 To see available goals, do "./pants help goals" and "./pants help $goal".
+
+These StackStorm-specific plugins are probably only useful for the st2 repo.
+- `schemas`
+
+### `schemas` plugin
+
+This plugin wires up pants to make sure `contrib/schemas/*.json` gets
+regenerated whenever the source files change. Now, whenever someone runs
+the `fmt` goal (eg `./pants fmt contrib/schemas::`), the schemas will
+be regenerated if any of the files used to generate them have changed.
+Also, running the `lint` goal will fail if the schemas need to be
+regenerated.

pants-plugins/schemas/BUILD

@@ -0,0 +1,5 @@
+python_sources()
+
+python_tests(
+    name="tests",
+)

pants-plugins/schemas/register.py

@@ -0,0 +1,24 @@
+# Copyright 2023 The StackStorm Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from schemas.rules import rules as schemas_rules
+from schemas.target_types import Schemas
+
+
+def rules():
+    return [*schemas_rules()]
+
+
+def target_types():
+    return [Schemas]

pants-plugins/schemas/rules.py

@@ -0,0 +1,125 @@
+# Copyright 2023 The StackStorm Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+from dataclasses import dataclass
+
+from pants.backend.python.target_types import EntryPoint
+from pants.backend.python.util_rules import pex, pex_from_targets
+from pants.backend.python.util_rules.pex import (
+    VenvPex,
+    VenvPexProcess,
+)
+from pants.backend.python.util_rules.pex_from_targets import PexFromTargetsRequest
+from pants.core.goals.fmt import FmtResult, FmtTargetsRequest
+from pants.engine.addresses import Address
+from pants.engine.fs import MergeDigests, Snapshot
+from pants.engine.process import FallibleProcessResult
+from pants.engine.rules import Get, MultiGet, collect_rules, rule
+from pants.engine.target import FieldSet
+from pants.engine.unions import UnionRule
+from pants.util.logging import LogLevel
+from pants.util.strutil import strip_v2_chroot_path
+
+from schemas.target_types import SchemasSourcesField
+
+
+# these constants are also used in the tests.
+CMD_SOURCE_ROOT = "st2common"
+CMD_DIR = "st2common/st2common/cmd"
+CMD_MODULE = "st2common.cmd"
+CMD = "generate_schemas"
+
+
+@dataclass(frozen=True)
+class GenerateSchemasFieldSet(FieldSet):
+    required_fields = (SchemasSourcesField,)
+
+    sources: SchemasSourcesField
+
+
+class GenerateSchemasViaFmtTargetsRequest(FmtTargetsRequest):
+    field_set_type = GenerateSchemasFieldSet
+    name = CMD
+
+
+@rule(
+    desc="Update contrib/schemas/*.json with st2-generate-schemas",
+    level=LogLevel.DEBUG,
+)
+async def generate_schemas_via_fmt(
+    request: GenerateSchemasViaFmtTargetsRequest,
+) -> FmtResult:
+    # We use a pex to actually generate the schemas with an external script.
+    # Generation cannot be inlined here because it needs to import the st2 code.
+    pex = await Get(
+        VenvPex,
+        PexFromTargetsRequest(
+            [
+                Address(
+                    CMD_DIR,
+                    target_name="cmd",
+                    relative_file_path=f"{CMD}.py",
+                )
+            ],
+            output_filename=f"{CMD}.pex",
+            internal_only=True,
+            main=EntryPoint.parse(f"{CMD_MODULE}.{CMD}:main"),
+        ),
+    )
+
+    # There will probably only be one target+field_set, but we iterate
+    # to satisfy how fmt expects that there could be more than one.
+    output_directories = [fs.address.spec_path for fs in request.field_sets]
+
+    results = await MultiGet(
+        Get(
+            FallibleProcessResult,
+            VenvPexProcess(
+                pex,
+                argv=(output_directory,),
+                # This script actually ignores the input files.
+                input_digest=request.snapshot.digest,
+                output_directories=[output_directory],
+                description=f"Regenerating st2 metadata schemas in {output_directory}",
+                level=LogLevel.DEBUG,
+            ),
+        )
+        for output_directory in output_directories
+    )
+
+    output_snapshot = await Get(
+        Snapshot, MergeDigests(result.output_digest for result in results)
+    )
+
+    stdout = "\n".join(
+        [strip_v2_chroot_path(process_result.stdout) for process_result in results]
+    )
+    stderr = "\n".join(
+        [strip_v2_chroot_path(process_result.stderr) for process_result in results]
+    )
+    return FmtResult(
+        input=request.snapshot,
+        output=output_snapshot,
+        stdout=stdout,
+        stderr=stderr,
+        formatter_name=request.name,
+    )
+
+
+def rules():
+    return [
+        *collect_rules(),
+        UnionRule(FmtTargetsRequest, GenerateSchemasViaFmtTargetsRequest),
+        *pex.rules(),
+        *pex_from_targets.rules(),
+    ]

pants-plugins/schemas/rules_test.py

@@ -0,0 +1,168 @@
+# Copyright 2023 The StackStorm Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+from __future__ import annotations
+
+import os
+
+import pytest
+
+from pants.backend.python import target_types_rules
+from pants.backend.python.target_types import PythonSourcesGeneratorTarget
+
+from pants.core.util_rules.source_files import SourceFiles, SourceFilesRequest
+from pants.engine.addresses import Address
+from pants.engine.fs import CreateDigest, Digest, FileContent, Snapshot
+from pants.engine.target import Target
+from pants.core.goals.fmt import FmtResult
+from pants.testutil.rule_runner import QueryRule, RuleRunner
+
+from .rules import (
+    CMD,
+    CMD_DIR,
+    CMD_SOURCE_ROOT,
+    GenerateSchemasFieldSet,
+    GenerateSchemasViaFmtTargetsRequest,
+    rules as schemas_rules,
+)
+from .target_types import Schemas
+
+
+@pytest.fixture
+def rule_runner() -> RuleRunner:
+    return RuleRunner(
+        rules=[
+            *schemas_rules(),
+            *target_types_rules.rules(),
+            QueryRule(FmtResult, (GenerateSchemasViaFmtTargetsRequest,)),
+            QueryRule(SourceFiles, (SourceFilesRequest,)),
+        ],
+        target_types=[Schemas, PythonSourcesGeneratorTarget],
+    )
+
+
+def run_st2_generate_schemas(
+    rule_runner: RuleRunner,
+    targets: list[Target],
+    *,
+    extra_args: list[str] | None = None,
+) -> FmtResult:
+    rule_runner.set_options(
+        [
+            "--backend-packages=schemas",
+            f"--source-root-patterns=/{CMD_SOURCE_ROOT}",
+            *(extra_args or ()),
+        ],
+        env_inherit={"PATH", "PYENV_ROOT", "HOME"},
+    )
+    field_sets = [GenerateSchemasFieldSet.create(tgt) for tgt in targets]
+    input_sources = rule_runner.request(
+        SourceFiles,
+        [
+            SourceFilesRequest(field_set.sources for field_set in field_sets),
+        ],
+    )
+    fmt_result = rule_runner.request(
+        FmtResult,
+        [
+            GenerateSchemasViaFmtTargetsRequest(
+                field_sets, snapshot=input_sources.snapshot
+            ),
+        ],
+    )
+    return fmt_result
+
+
+# copied from pantsbuild/pants.git/src/python/pants/backend/python/lint/black/rules_integration_test.py
+def get_snapshot(rule_runner: RuleRunner, source_files: dict[str, str]) -> Snapshot:
+    files = [
+        FileContent(path, content.encode()) for path, content in source_files.items()
+    ]
+    digest = rule_runner.request(Digest, [CreateDigest(files)])
+    return rule_runner.request(Snapshot, [digest])
+
+
+# add dummy script at st2common/st2common/cmd/generate_schemas.py that the test can load.
+GENERATE_SCHEMAS_PY = """
+import os
+
+
+def main():
+    print('Generated schema for the "dummy" model.')
+    schema_text = "{schema_text}"
+    schema_file = os.path.join("{schemas_dir}", "dummy.json")
+    print('Schema will be written to "%s".' % schema_file)
+    with open(schema_file, "w") as f:
+        f.write(schema_text)
+"""
+
+
+def write_files(
+    schemas_dir: str, schema_file: str, before: str, after: str, rule_runner: RuleRunner
+) -> None:
+    files = {
+        f"{schemas_dir}/{schema_file}": before,
+        f"{schemas_dir}/BUILD": "schemas(name='t')",
+        # add in the target that's hard-coded in the generate_schemas_via_fmt rue
+        f"{CMD_DIR}/{CMD}.py": GENERATE_SCHEMAS_PY.format(
+            schemas_dir=schemas_dir, schema_text=after
+        ),
+        f"{CMD_DIR}/BUILD": "python_sources()",
+    }
+
+    module = CMD_DIR
+    while module != CMD_SOURCE_ROOT:
+        files[f"{module}/__init__.py"] = ""
+        module = os.path.dirname(module)
+
+    rule_runner.write_files(files)
+
+
+def test_changed(rule_runner: RuleRunner) -> None:
+    write_files(
+        schemas_dir="my_dir",
+        schema_file="dummy.json",
+        before="BEFORE",
+        after="AFTER",
+        rule_runner=rule_runner,
+    )
+
+    tgt = rule_runner.get_target(
+        Address("my_dir", target_name="t", relative_file_path="dummy.json")
+    )
+    fmt_result = run_st2_generate_schemas(rule_runner, [tgt])
+    assert 'Schema will be written to "my_dir/dummy.json".' in fmt_result.stdout
+    assert fmt_result.output == get_snapshot(
+        rule_runner, {"my_dir/dummy.json": "AFTER"}
+    )
+    assert fmt_result.did_change is True
+
+
+def test_unchanged(rule_runner: RuleRunner) -> None:
+    write_files(
+        schemas_dir="my_dir",
+        schema_file="dummy.json",
+        before="AFTER",
+        after="AFTER",
+        rule_runner=rule_runner,
+    )
+
+    tgt = rule_runner.get_target(
+        Address("my_dir", target_name="t", relative_file_path="dummy.json")
+    )
+    fmt_result = run_st2_generate_schemas(rule_runner, [tgt])
+    assert 'Schema will be written to "my_dir/dummy.json".' in fmt_result.stdout
+    assert fmt_result.output == get_snapshot(
+        rule_runner, {"my_dir/dummy.json": "AFTER"}
+    )
+    assert fmt_result.did_change is False

pants-plugins/schemas/target_types.py

@@ -0,0 +1,42 @@
+# Copyright 2023 The StackStorm Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+from pants.engine.fs import GlobMatchErrorBehavior
+from pants.engine.target import (
+    COMMON_TARGET_FIELDS,
+    Dependencies,
+    MultipleSourcesField,
+    Target,
+    generate_multiple_sources_field_help_message,
+)
+
+
+class SchemasSourcesField(MultipleSourcesField):
+    expected_file_extensions = (".json",)
+    default = ("*.json",)
+    uses_source_roots = False
+
+    # make sure at least one schema is present or fmt will be skipped.
+    default_glob_match_error_behavior = GlobMatchErrorBehavior.error
+
+    help = generate_multiple_sources_field_help_message(
+        "Example: `sources=['*.json', '!ignore.json']`"
+    )
+
+
+class Schemas(Target):
+    alias = "schemas"
+    core_fields = (*COMMON_TARGET_FIELDS, Dependencies, SchemasSourcesField)
+    help = (
+        "Generate st2 metadata (pack, action, rule, ...) schemas from python sources."
+    )