Add pants-plugins/pack_metadata to inform pants about our packs

CHANGELOG.rst

@@ -14,7 +14,7 @@ Added
   working on StackStorm, improve our security posture, and improve CI reliability thanks in part
   to pants' use of PEX lockfiles. This is not a user-facing addition.
   #5778 #5789 #5817 #5795 #5830 #5833 #5834 #5841 #5840 #5838 #5842 #5837 #5849 #5850
-  #5846 #5853 #5848 #5847 #5858 #5857 #5860
+  #5846 #5853 #5848 #5847 #5858 #5857 #5860 #5868
   Contributed by @cognifloyd
 
 * Added a joint index to solve the problem of slow mongo queries for scheduled executions. #5805

pants-plugins/README.md

@@ -8,6 +8,9 @@ The plugins here add custom goals or other logic into pants.
 
 To see available goals, do "./pants help goals" and "./pants help $goal".
 
+These StackStorm-specific plugins might be useful in other StackStorm-related repos.
+- `pack_metadata`
+
 These StackStorm-specific plugins are probably only useful for the st2 repo.
 - `api_spec`
 - `sample_conf`
@@ -26,6 +29,26 @@ This plugin also wires up pants so that the `lint` goal runs additional
 api spec validation on `st2common/st2common/openapi.yaml` with something
 like `./pants lint st2common/st2common/openapi.yaml`.
 
+### `pack_metadata` plugin
+
+This plugin adds two new targets to pants:
+- `pack_metadata`
+- `pack_metadata_in_git_submodule`
+
+These targets include all StackStorm pack metadata files in a pack.
+Pack metadata includes top-level files (`pack.yaml`, `<pack>.yaml.example`,
+`config.schema.yaml`, and `icon.png`) and metadata (`*.yaml`, `*.yml`)
+for actions, action-aliases, policies, rules, and sensors.
+
+This plugin also wires up the `tailor` goal, so that it will add a
+`pack_metadata(name="metadata")` target wherever it finds a `pack.yaml` file.
+
+One of the packs in this repo is in a git submodule to test our handling
+of git submodules (`st2tests/st2tests/fixtures/packs/test_content_version`).
+If it is not checked out, then some of the tests will fail.
+If it is not checked out, `pack_metadata_in_git_submodule` handles providing
+a helpful, instructive error message as early as possible.
+
 ### `sample_conf` plugin
 
 This plugin wires up pants to make sure `conf/st2.conf.sample` gets

pants-plugins/pack_metadata/BUILD

@@ -0,0 +1,5 @@
+python_sources()
+
+python_tests(
+    name="tests",
+)

pants-plugins/pack_metadata/register.py

@@ -0,0 +1,23 @@
+# Copyright 2023 The StackStorm Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+from pack_metadata import tailor
+from pack_metadata.target_types import PackMetadata, PackMetadataInGitSubmodule
+
+
+def rules():
+    return tailor.rules()
+
+
+def target_types():
+    return [PackMetadata, PackMetadataInGitSubmodule]

pants-plugins/pack_metadata/tailor.py

@@ -0,0 +1,62 @@
+# Copyright 2023 The StackStorm Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+import os
+from dataclasses import dataclass
+
+from pants.core.goals.tailor import (
+    AllOwnedSources,
+    PutativeTarget,
+    PutativeTargets,
+    PutativeTargetsRequest,
+)
+from pants.engine.fs import PathGlobs, Paths
+from pants.engine.rules import collect_rules, Get, rule, UnionRule
+from pants.util.logging import LogLevel
+
+from pack_metadata.target_types import PackMetadata
+
+
+@dataclass(frozen=True)
+class PutativePackMetadataTargetsRequest(PutativeTargetsRequest):
+    pass
+
+
+@rule(
+    desc="Find pack (config, action, alias, sensor, icon, etc) metadata files.",
+    level=LogLevel.DEBUG,
+)
+async def find_putative_targets(
+    _: PutativePackMetadataTargetsRequest, all_owned_sources: AllOwnedSources
+) -> PutativeTargets:
+    all_pack_yaml_files = await Get(Paths, PathGlobs(["**/pack.yaml"]))
+
+    unowned_pack_yaml_files = set(all_pack_yaml_files.files) - set(all_owned_sources)
+    unowned_pack_dirs = [os.path.dirname(p) for p in unowned_pack_yaml_files]
+
+    name = "metadata"
+    return PutativeTargets(
+        [
+            PutativeTarget.for_target_type(
+                PackMetadata, dirname, name, ("pack.yaml",), kwargs={"name": name}
+            )
+            for dirname in unowned_pack_dirs
+        ]
+    )
+
+
+def rules():
+    return [
+        *collect_rules(),
+        UnionRule(PutativeTargetsRequest, PutativePackMetadataTargetsRequest),
+    ]

pants-plugins/pack_metadata/tailor_test.py

@@ -0,0 +1,107 @@
+# Copyright 2023 The StackStorm Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+from __future__ import annotations
+
+import pytest
+
+from pants.core.goals.tailor import (
+    AllOwnedSources,
+    PutativeTarget,
+    PutativeTargets,
+)
+from pants.testutil.rule_runner import QueryRule, RuleRunner
+
+from .tailor import (
+    PutativePackMetadataTargetsRequest,
+    rules as pack_metadata_rules,
+)
+from .target_types import PackMetadata, PackMetadataInGitSubmodule
+
+
+@pytest.fixture
+def rule_runner() -> RuleRunner:
+    return RuleRunner(
+        rules=[
+            *pack_metadata_rules(),
+            QueryRule(
+                PutativeTargets, (PutativePackMetadataTargetsRequest, AllOwnedSources)
+            ),
+        ],
+        target_types=[PackMetadata, PackMetadataInGitSubmodule],
+    )
+
+
+def test_find_putative_targets(rule_runner: RuleRunner) -> None:
+    rule_runner.write_files(
+        {
+            "packs/already_owned/pack.yaml": "---\nname: already_owned\n",
+            "packs/already_owned/actions/action.yaml": "---\nname: action\n",
+            "packs/foo/pack.yaml": "---\nname: foo\n",
+            "packs/foo/actions/action.yaml": "---\nname: action\n",
+            "packs/bar/pack.yaml": "---\nname: bar\n",
+            "packs/bar/sensors/sensor.yaml": "---\nname: sensor\n",
+            "other/deep/baz/pack.yaml": "---\nname: baz\n",
+        }
+    )
+    pts = rule_runner.request(
+        PutativeTargets,
+        [
+            PutativePackMetadataTargetsRequest(
+                (
+                    "packs",
+                    "packs/already_owned",
+                    "packs/already_owned/actions",
+                    "packs/foo",
+                    "packs/foo/actions",
+                    "packs/bar",
+                    "packs/bar/sensors",
+                    "other/deep/baz",
+                )
+            ),
+            AllOwnedSources(
+                [
+                    "packs/already_owned/pack.yaml",
+                    "packs/already_owned/actions/action.yaml",
+                ]
+            ),
+        ],
+    )
+    assert (
+        PutativeTargets(
+            [
+                PutativeTarget.for_target_type(
+                    PackMetadata,
+                    path="packs/foo",
+                    name="metadata",
+                    triggering_sources=["pack.yaml"],
+                    kwargs={"name": "metadata"},
+                ),
+                PutativeTarget.for_target_type(
+                    PackMetadata,
+                    path="packs/bar",
+                    name="metadata",
+                    triggering_sources=["pack.yaml"],
+                    kwargs={"name": "metadata"},
+                ),
+                PutativeTarget.for_target_type(
+                    PackMetadata,
+                    path="other/deep/baz",
+                    name="metadata",
+                    triggering_sources=["pack.yaml"],
+                    kwargs={"name": "metadata"},
+                ),
+            ]
+        )
+        == pts
+    )

pants-plugins/pack_metadata/target_types.py

@@ -0,0 +1,79 @@
+# Copyright 2023 The StackStorm Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+from typing import Sequence
+
+from pants.engine.target import COMMON_TARGET_FIELDS, Dependencies
+from pants.core.target_types import (
+    ResourcesGeneratingSourcesField,
+    ResourcesGeneratorTarget,
+)
+
+
+class UnmatchedGlobsError(Exception):
+    """Error thrown when a required set of globs didn't match."""
+
+
+class PackMetadataSourcesField(ResourcesGeneratingSourcesField):
+    required = False
+    default = (
+        # metadata does not include any python, shell, or other sources.
+        "pack.yaml",
+        "config.schema.yaml",
+        "*.yaml.example",
+        "**/*.yaml",
+        "**/*.yml",
+        "icon.png",  # used in st2web ui
+        # "requirements*.txt",  # including this causes target conflicts
+        # "README.md",
+        # "HISTORY.md",
+    )
+
+
+class PackMetadataInGitSubmoduleSources(PackMetadataSourcesField):
+    required = True
+
+    def validate_resolved_files(self, files: Sequence[str]) -> None:
+        if not files:
+            raise UnmatchedGlobsError(
+                # see: st2tests.fixturesloader.GIT_SUBMODULES_NOT_CHECKED_OUT_ERROR
+                "One or more git submodules is not checked out. Make sure to run "
+                '"git submodule update --init --recursive"'
+                "in the repository root directory to check out all the submodules."
+            )
+        super().validate_resolved_files(files)
+
+
+class PackMetadata(ResourcesGeneratorTarget):
+    alias = "pack_metadata"
+    core_fields = (*COMMON_TARGET_FIELDS, Dependencies, PackMetadataSourcesField)
+    help = (
+        "Loose pack metadata files.\n\n"
+        "Pack metadata includes top-level files (pack.yaml, <pack>.yaml.example, "
+        "config.schema.yaml, and icon.png) and metadata for actions, "
+        "action-aliases, policies, rules, and sensors."
+    )
+
+
+class PackMetadataInGitSubmodule(PackMetadata):
+    alias = "pack_metadata_in_git_submodule"
+    core_fields = (
+        *COMMON_TARGET_FIELDS,
+        Dependencies,
+        PackMetadataInGitSubmoduleSources,
+    )
+    help = PackMetadata.help + (
+        "\npack_metadata_in_git_submodule variant errors if the sources field "
+        "has unmatched globs. It prints instructions on how to checkout git "
+        "submodules."
+    )

pants-plugins/pack_metadata/target_types_test.py

@@ -0,0 +1,68 @@
+# Copyright 2023 The StackStorm Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+from __future__ import annotations
+
+import pytest
+
+from pants.engine.addresses import Address
+from pants.engine.internals.scheduler import ExecutionError
+from pants.testutil.rule_runner import RuleRunner
+
+from .target_types import (
+    PackMetadata,
+    # PackMetadataSourcesField,
+    PackMetadataInGitSubmodule,
+    # PackMetadataInGitSubmoduleSources,
+    UnmatchedGlobsError,
+)
+
+
+@pytest.fixture
+def rule_runner() -> RuleRunner:
+    return RuleRunner(
+        rules=[],
+        target_types=[PackMetadata, PackMetadataInGitSubmodule],
+    )
+
+
+GIT_SUBMODULE_BUILD_FILE = """
+pack_metadata_in_git_submodule(
+    name="metadata",
+    sources=["./submodule_dir/pack.yaml"],
+)
+"""
+
+
+def test_git_submodule_sources_missing(rule_runner: RuleRunner) -> None:
+    rule_runner.write_files(
+        {
+            "packs/BUILD": GIT_SUBMODULE_BUILD_FILE,
+        }
+    )
+    with pytest.raises(ExecutionError) as e:
+        _ = rule_runner.get_target(Address("packs", target_name="metadata"))
+    exc = e.value.wrapped_exceptions[0]
+    assert isinstance(exc, UnmatchedGlobsError)
+    assert "One or more git submodules is not checked out" in str(exc)
+
+
+def test_git_submodule_sources_present(rule_runner: RuleRunner) -> None:
+    rule_runner.write_files(
+        {
+            "packs/BUILD": GIT_SUBMODULE_BUILD_FILE,
+            "packs/submodule_dir/pack.yaml": "---\nname: foobar\n",
+        }
+    )
+    # basically: this asserts that it does not raise UnmatchedGlobsError
+    _ = rule_runner.get_target(Address("packs", target_name="metadata"))