convert-from-legacy to "dirac internal legacy cs-sync"

src/diracx/cli/internal.py → src/diracx/cli/internal/__init__.py

@@ -1,6 +1,3 @@
-from __future__ import absolute_import
-
-import json
 from pathlib import Path
 
 import git
@@ -19,9 +16,11 @@
     UserConfig,
 )
 
-from .utils import AsyncTyper
+from ..utils import AsyncTyper
+from . import legacy
 
 app = AsyncTyper()
+app.add_typer(legacy.app, name="legacy")
 
 
 @app.command()
@@ -49,7 +48,7 @@ def generate_cs(
         Users={},
         Groups={
             user_group: GroupConfig(
-                JobShare=None, Properties=["NormalUser"], Quota=None, Users=[]
+                JobShare=None, Properties={"NormalUser"}, Quota=None, Users=set()
             )
         },
     )
@@ -62,8 +61,7 @@ def generate_cs(
     repo = git.Repo.init(repo_path, initial_branch="master")
     yaml_path = repo_path / "default.yml"
     typer.echo(f"Writing configuration to {yaml_path}", err=True)
-    config_data = json.loads(config.json(exclude_unset=True))
-    yaml_path.write_text(yaml.safe_dump(config_data))
+    yaml_path.write_text(yaml.safe_dump(config.dict(exclude_unset=True)))
     repo.index.add([yaml_path.relative_to(repo_path)])
     repo.index.commit("Initial commit")
     typer.echo(f"Successfully created repo in {config_repo}", err=True)
@@ -100,8 +98,7 @@ def add_user(
     repo = git.Repo.init(repo_path)
     yaml_path = repo_path / "default.yml"
     typer.echo(f"Writing back configuration to {yaml_path}", err=True)
-    config_data = json.loads(config.json(exclude_unset=True))
-    yaml_path.write_text(yaml.safe_dump(config_data))
+    yaml_path.write_text(yaml.safe_dump(config.dict(exclude_unset=True)))
     repo.index.add([yaml_path.relative_to(repo_path)])
     repo.index.commit(
         f"Added user {sub} ({preferred_username}) to vo {vo} and user_group {user_group}"

csSync/convert-from-legacy.py → src/diracx/cli/internal/legacy.py

@@ -1,27 +1,15 @@
-#!/usr/bin/env python
-from __future__ import annotations
-
-import argparse
-import json
 import os
 from pathlib import Path
 
 import diraccfg
 import yaml
 from pydantic import BaseModel
 
-from diracx.core.config import DEFAULT_CONFIG_FILE, Config
-
-
-def _git_path(value: str) -> Path:
-    repo = Path(value)
-    if not (repo / ".git").is_dir():
-        raise ValueError(f"{repo} does not appear to be a git repository")
-    return repo
+from diracx.core.config import Config
 
+from ..utils import AsyncTyper
 
-def _list_to_str(original: str) -> list[str]:
-    return [x.strip() for x in original.split(",") if x.strip()]
+app = AsyncTyper()
 
 
 class IdPConfig(BaseModel):
@@ -39,35 +27,36 @@ class ConversionConfig(BaseModel):
     VOs: dict[str, VOConfig]
 
 
-def parse_args():
-    parser = argparse.ArgumentParser("Convert the legacy DIRAC CS to the new format")
-    parser.add_argument("old_file", type=Path)
-    parser.add_argument("conversion_config", type=Path)
-    parser.add_argument("repo", type=_git_path)
-    args = parser.parse_args()
+# def parse_args():
+#     parser = argparse.ArgumentParser("Convert the legacy DIRAC CS to the new format")
+#     parser.add_argument("old_file", type=Path)
+#     parser.add_argument("conversion_config", type=Path)
+#     parser.add_argument("repo", type=Path)
+#     args = parser.parse_args()
+
+
+#     main(args.old_file, args.conversion_config, args.repo / DEFAULT_CONFIG_FILE)
+
 
+@app.command()
+def cs_sync(old_file: Path, conversion_config: Path, new_file: Path):
+    """Load the old CS and convert it to the new YAML format"""
     if not os.environ.get("DIRAC_COMPAT_ENABLE_CS_CONVERSION"):
         raise RuntimeError(
             "DIRAC_COMPAT_ENABLE_CS_CONVERSION must be set for the conversion to be possible"
         )
 
-    main(args.old_file, args.conversion_config, args.repo / DEFAULT_CONFIG_FILE)
-
-
-def main(old_file: Path, conversion_config: Path, new_file: Path):
-    """Load the old CS and convert it to the new YAML format"""
     old_data = old_file.read_text()
     cfg = diraccfg.CFG().loadFromBuffer(old_data)
     raw = cfg.getAsDict()
 
-    apply_fixes(raw, conversion_config)
+    _apply_fixes(raw, conversion_config)
 
     config = Config.parse_obj(raw)
-    new_data = json.loads(config.json(exclude_unset=True))
-    new_file.write_text(yaml.safe_dump(new_data))
+    new_file.write_text(yaml.safe_dump(config.dict(exclude_unset=True)))
 
 
-def apply_fixes(raw, conversion_config: Path):
+def _apply_fixes(raw, conversion_config: Path):
     """Modify raw in place to make any layout changes between the old and new structure"""
 
     conv_config = ConversionConfig.parse_obj(
@@ -147,7 +136,3 @@ def apply_fixes(raw, conversion_config: Path):
                     # We ignore the DN and CA
                     raw["Registry"][vo]["Users"][subject].pop("DN", None)
                     raw["Registry"][vo]["Users"][subject].pop("CA", None)
-
-
-if __name__ == "__main__":
-    parse_args()

src/diracx/core/config/schema.py

@@ -22,9 +22,11 @@ def legacy_adaptor(cls, v):
         # though ideally we should parse the type hints properly.
         for field, hint in cls.__annotations__.items():
             # Convert comma separated lists to actual lists
-            if hint in {"list[str]", "list[SecurityProperty]"} and isinstance(
-                v.get(field), str
-            ):
+            if hint in {
+                "list[str]",
+                "set[str]",
+                "set[SecurityProperty]",
+            } and isinstance(v.get(field), str):
                 v[field] = [x.strip() for x in v[field].split(",") if x.strip()]
             # If the field is optional and the value is "None" convert it to None
             if "| None" in hint and field in v:
@@ -48,9 +50,9 @@ class GroupConfig(BaseModel):
     AutoUploadPilotProxy: bool = False
     AutoUploadProxy: bool = False
     JobShare: Optional[int]
-    Properties: list[SecurityProperty]
+    Properties: set[SecurityProperty]
     Quota: Optional[int]
-    Users: list[str]
+    Users: set[str]
     AllowBackgroundTQs: bool = False
     VOMSRole: Optional[str]
     AutoSyncVOMS: bool = False

csSync/tests/integration_test.yaml → tests/cli/legacy/cs_sync/integration_test.yaml

@@ -52,20 +52,20 @@ Registry:
     DefaultGroup: jenkins_user
     Groups:
       jenkins_fcadmin:
-        Properties:
-        - FileCatalogManagement
-        - NormalUser
-        Users:
-        - e2cb28ec-1a1e-40ee-a56d-d899b79879ce
-        - 26dbe36e-cf5c-4c52-a834-29a1c904ef74
-        - a95ab678-3fa4-41b9-b863-fe62ce8064ce
+        Properties: !!set
+          FileCatalogManagement: null
+          NormalUser: null
+        Users: !!set
+          26dbe36e-cf5c-4c52-a834-29a1c904ef74: null
+          a95ab678-3fa4-41b9-b863-fe62ce8064ce: null
+          e2cb28ec-1a1e-40ee-a56d-d899b79879ce: null
       jenkins_user:
-        Properties:
-        - NormalUser
-        Users:
-        - e2cb28ec-1a1e-40ee-a56d-d899b79879ce
-        - 26dbe36e-cf5c-4c52-a834-29a1c904ef74
-        - a95ab678-3fa4-41b9-b863-fe62ce8064ce
+        Properties: !!set
+          NormalUser: null
+        Users: !!set
+          26dbe36e-cf5c-4c52-a834-29a1c904ef74: null
+          a95ab678-3fa4-41b9-b863-fe62ce8064ce: null
+          e2cb28ec-1a1e-40ee-a56d-d899b79879ce: null
     IdP:
       ClientID: 995ed3b9-d5bd-49d3-a7f4-7fc7dbd5a0cd
       URL: https://jenkins.invalid/
@@ -83,23 +83,23 @@ Registry:
     DefaultGroup: dirac_user
     Groups:
       dirac_admin:
-        Properties:
-        - AlarmsManagement
-        - ServiceAdministrator
-        - CSAdministrator
-        - JobAdministrator
-        - FullDelegation
-        - ProxyManagement
-        - Operator
-        Users:
-        - 26b14fc9-6d40-4ca5-b014-6234eaf0fb6e
+        Properties: !!set
+          AlarmsManagement: null
+          CSAdministrator: null
+          FullDelegation: null
+          JobAdministrator: null
+          Operator: null
+          ProxyManagement: null
+          ServiceAdministrator: null
+        Users: !!set
+          26b14fc9-6d40-4ca5-b014-6234eaf0fb6e: null
       dirac_user:
-        Properties:
-        - NormalUser
-        Users:
-        - 26b14fc9-6d40-4ca5-b014-6234eaf0fb6e
-        - d3adc733-6588-4d6f-8581-5986b02d0c87
-        - ff2152ff-34f4-4739-b106-3def37e291e3
+        Properties: !!set
+          NormalUser: null
+        Users: !!set
+          26b14fc9-6d40-4ca5-b014-6234eaf0fb6e: null
+          d3adc733-6588-4d6f-8581-5986b02d0c87: null
+          ff2152ff-34f4-4739-b106-3def37e291e3: null
     IdP:
       ClientID: 072afab5-ed92-46e0-a61d-4ecbc96e0770
       URL: https://vo.invalid/

tests/cli/legacy/cs_sync/test_cssync.py

@@ -0,0 +1,34 @@
+from pathlib import Path
+
+import yaml
+from typer.testing import CliRunner
+
+from diracx.cli import app
+
+runner = CliRunner()
+
+file_path = Path(__file__).parent
+
+
+def test_cs_sync(tmp_path, monkeypatch):
+    monkeypatch.setenv("DIRAC_COMPAT_ENABLE_CS_CONVERSION", "Yes")
+
+    output_file = tmp_path / "default.yaml"
+
+    result = runner.invoke(
+        app,
+        [
+            "internal",
+            "legacy",
+            "cs-sync",
+            f"{file_path / 'integration_test.cfg'}",
+            f"{file_path / 'convert_integration_test.yaml'}",
+            str(output_file),
+        ],
+    )
+    assert result.exit_code == 0
+    assert output_file.is_file()
+
+    actual_output = yaml.safe_load(output_file.read_text())
+    expected_output = yaml.safe_load((file_path / "integration_test.yaml").read_text())
+    assert actual_output == expected_output