Merge pull request #1 from kavabanga-ai/develop

Move rego files into their respective apps (cvat-ai#7806)

.github/workflows/full.yml

@@ -207,11 +207,10 @@ jobs:
 
       - name: Running OPA tests
         run: |
-          python cvat/apps/iam/rules/tests/generate_tests.py \
-            --output-dir cvat/apps/iam/rules/
+          python cvat/apps/iam/rules/tests/generate_tests.py
 
-          docker compose run --rm -v "$PWD/cvat/apps/iam/rules/:/mnt/rules" \
-            cvat_opa test /mnt/rules
+          docker compose run --rm -v "$PWD:/mnt/src:ro" -w /mnt/src \
+            cvat_opa test cvat/apps/*/rules
 
       - name: Running unit tests
         env:

.github/workflows/main.yml

@@ -226,11 +226,10 @@ jobs:
 
       - name: Running OPA tests
         run: |
-          python cvat/apps/iam/rules/tests/generate_tests.py \
-            --output-dir cvat/apps/iam/rules/
+          python cvat/apps/iam/rules/tests/generate_tests.py
 
-          docker compose run --rm -v "$PWD/cvat/apps/iam/rules/:/mnt/rules" \
-            cvat_opa test /mnt/rules
+          docker compose run --rm -v "$PWD:/mnt/src:ro" -w /mnt/src \
+            cvat_opa test cvat/apps/*/rules
 
       - name: Running unit tests
         env:

.github/workflows/regallint.yml

@@ -9,4 +9,4 @@ jobs:
         uses: StyraInc/[email protected]
         with:
           version: v0.11.0
-      - run: regal lint --format=github cvat/apps/iam/rules
+      - run: regal lint --format=github cvat/apps/*/rules

.github/workflows/schedule.yml

@@ -153,11 +153,10 @@ jobs:
 
       - name: OPA tests
         run: |
-          python cvat/apps/iam/rules/tests/generate_tests.py \
-            --output-dir cvat/apps/iam/rules/
+          python cvat/apps/iam/rules/tests/generate_tests.py
 
-          docker compose run --rm -v "$PWD/cvat/apps/iam/rules/:/mnt/rules" \
-            cvat_opa test /mnt/rules
+          docker compose run --rm -v "$PWD:/mnt/src:ro" -w /mnt/src \
+            cvat_opa test cvat/apps/*/rules
 
       - name: REST API and SDK tests
         run: |

.gitignore

@@ -64,3 +64,5 @@ cvat-core/reports
 # produced by prepare in the root package.json script
 .husky
 
+# produced by cvat/apps/iam/rules/tests/generate_tests.py
+/cvat/apps/*/rules/*_test.gen.rego

cvat/apps/iam/permissions.py

@@ -9,6 +9,7 @@
 import operator
 from abc import ABCMeta, abstractmethod
 from enum import Enum
+from pathlib import Path
 from typing import Any, Dict, List, Optional, Sequence, TypeVar
 
 from attrs import define, field
@@ -20,6 +21,7 @@
 from cvat.apps.organizations.models import Membership, Organization
 from cvat.utils.http import make_requests_session
 
+from .utils import add_opa_rules_path
 
 class StrEnum(str, Enum):
     def __str__(self) -> str:
@@ -243,7 +245,7 @@ def has_object_permission(self, request, view, obj) -> bool:
 
 def load_app_permissions(config: AppConfig) -> None:
     """
-    Ensures that permissions from the given app are loaded.
+    Ensures that permissions and OPA rules from the given app are loaded.
 
     This function should be called from the AppConfig.ready() method of every
     app that defines a permissions module.
@@ -254,3 +256,5 @@ def load_app_permissions(config: AppConfig) -> None:
         isinstance(attr, type) and issubclass(attr, OpenPolicyAgentPermission)
         for attr in vars(permissions_module).values()
     )
+
+    add_opa_rules_path(Path(config.path, "rules"))

cvat/apps/iam/rules/tests/generate_tests.py

@@ -4,37 +4,24 @@
 #
 # SPDX-License-Identifier: MIT
 
-import os
-import os.path as osp
 import subprocess
 import sys
 from argparse import ArgumentParser, Namespace
 from concurrent.futures import ThreadPoolExecutor
 from functools import partial
-from glob import glob
 from typing import Optional, Sequence
+from pathlib import Path
 
+REPO_ROOT = Path(__file__).resolve().parents[5]
 
 def create_arg_parser() -> ArgumentParser:
     parser = ArgumentParser(add_help=True)
     parser.add_argument(
-        "-c",
-        "--config-dir",
-        default=None,
-        help="The directory with test configs in CSV format (default: the default location)",
-    )
-    parser.add_argument(
-        "-g",
-        "--gen-dir",
-        default=None,
-        help="The directory with test generators (default: the default location)",
-    )
-    parser.add_argument(
-        "-o",
-        "--output-dir",
-        default=".",
-        type=osp.abspath,
-        help="The output directory for rego files (default: current dir)",
+        "-a",
+        "--apps-dir",
+        type=Path,
+        default=REPO_ROOT / "cvat/apps",
+        help="The directory with Django apps (default: cvat/apps)",
     )
     return parser
 
@@ -45,28 +32,27 @@ def parse_args(args: Optional[Sequence[str]] = None) -> Namespace:
     return parsed_args
 
 
-def call_generator(module_path: str, gen_params: Namespace):
+def call_generator(generator_path: Path, gen_params: Namespace) -> None:
+    rules_dir = generator_path.parents[2]
     subprocess.check_call(
-        ["python3", module_path, gen_params.config_dir], cwd=gen_params.output_dir
+        [sys.executable, generator_path, rules_dir / 'tests/configs'], cwd=rules_dir
     )
 
 
 def main(args: Optional[Sequence[str]] = None) -> int:
     args = parse_args(args)
 
-    args.config_dir = osp.abspath(args.config_dir or osp.join(osp.dirname(__file__), "configs"))
-    args.gen_dir = osp.abspath(args.gen_dir or osp.join(osp.dirname(__file__), "generators"))
-
-    assert osp.isdir(args.config_dir)
-    assert osp.isdir(args.gen_dir)
+    generator_paths = list(args.apps_dir.glob("*/rules/tests/generators/*_test.gen.rego.py"))
 
-    os.makedirs(args.output_dir, exist_ok=True)
+    if not generator_paths:
+        sys.exit("error: no generators found")
 
     with ThreadPoolExecutor() as pool:
-        pool.map(
+        for _ in pool.map(
             partial(call_generator, gen_params=args),
-            glob(osp.join(args.gen_dir, "*_test.gen.rego.py")),
-        )
+            generator_paths,
+        ):
+            pass # consume all results in order to propagate exceptions
 
 
 if __name__ == "__main__":

cvat/apps/iam/utils.py

@@ -5,23 +5,27 @@
 import io
 import tarfile
 
-from django.conf import settings
+_OPA_RULES_PATHS = {
+    Path(__file__).parent / 'rules',
+}
 
-@functools.lru_cache()
+@functools.lru_cache(maxsize=None)
 def get_opa_bundle() -> Tuple[bytes, str]:
-    rules_paths = [Path(settings.BASE_DIR) / rel_path for rel_path in settings.IAM_OPA_RULES_PATH.strip(':').split(':')]
-
     bundle_file = io.BytesIO()
 
     with tarfile.open(fileobj=bundle_file, mode='w:gz') as tar:
-        for p in rules_paths:
+        for p in _OPA_RULES_PATHS:
             for f in p.glob('*[!.gen].rego'):
                 tar.add(name=f, arcname=f.relative_to(p.parent))
 
     bundle = bundle_file.getvalue()
     etag = hashlib.blake2b(bundle).hexdigest()
     return bundle, etag
 
+def add_opa_rules_path(path: Path) -> None:
+    _OPA_RULES_PATHS.add(path)
+    get_opa_bundle.cache_clear()
+
 def get_dummy_user(email):
     from allauth.account.models import EmailAddress
     from allauth.account import app_settings

cvat/settings/base.py

@@ -237,7 +237,6 @@ def generate_secret_key():
 IAM_ROLES = [IAM_ADMIN_ROLE, 'business', 'user', 'worker']
 IAM_OPA_HOST = 'http://opa:8181'
 IAM_OPA_DATA_URL = f'{IAM_OPA_HOST}/v1/data'
-IAM_OPA_RULES_PATH = 'cvat/apps/iam/rules:'
 LOGIN_URL = 'rest_login'
 LOGIN_REDIRECT_URL = '/'
 

site/content/en/docs/contributing/running-tests.md

@@ -204,24 +204,22 @@ of the corresponding task in `./vscode/launch.json`, for example:
 ### Generate tests
 
 ```bash
-python cvat/apps/iam/rules/tests/generate_tests.py \
-   --output-dir cvat/apps/iam/rules/
+python cvat/apps/iam/rules/tests/generate_tests.py
 ```
 
 ### Run testing
 
 - In a Docker container
 ```bash
-docker run --rm -v ${PWD}/cvat/apps/iam/rules:/rules \
-   openpolicyagent/opa:0.63.0 \
-   test /rules -v
+docker compose run --rm -v "$PWD:/mnt/src:ro" -w /mnt/src \
+    cvat_opa test -v cvat/apps/*/rules
 ```
 
 - or execute OPA directly
 ```bash
 curl -L -o opa https://openpolicyagent.org/downloads/v0.63.0/opa_linux_amd64_static
 chmod +x ./opa
-./opa test cvat/apps/iam/rules
+./opa test cvat/apps/*/rules
 ```
 
 ### Linting Rego
@@ -230,14 +228,14 @@ The Rego policies in this project are linted using [Regal](https://github.com/st
 
 - In a Docker container
 ```bash
-docker run --rm -v ${PWD}/cvat/apps/iam/rules:/rules \
+docker run --rm -v ${PWD}:/mnt/src:ro -w /mnt/src \
     ghcr.io/styrainc/regal:0.11.0 \
-    lint /rules
+    lint cvat/apps/*/rules
 ```
 
 - or execute Regal directly
 ```bash
 curl -L -o regal https://github.com/StyraInc/regal/releases/download/v0.11.0/regal_Linux_x86_64
 chmod +x ./regal
-./regal lint cvat/apps/iam/rules
+./regal lint cvat/apps/*/rules
 ```

site/content/en/docs/manual/advanced/iam_user_roles.md

@@ -21,7 +21,7 @@ using the [**Free plan**](https://www.cvat.ai/pricing/cloud) and can be lifted u
 
 All roles are predefined and cannot be modified through the user interface.
 However, within the _self-hosted solution_, roles can be adjusted using `.rego`
-files stored in `cvat/apps/iam/rules/`.
+files stored in `cvat/apps/*/rules/`.
 Rego is a declarative language employed for defining
 OPA (Open Policy Agent) policies, and its syntax is detailed
 in the [**OPA documentation**](https://www.openpolicyagent.org/docs/latest/policy-language/).