Merge pull request #115 from chaen/csSync

Add syncronisation script for converying from the old CS

src/diracx/cli/internal.py → src/diracx/cli/internal/__init__.py

@@ -1,6 +1,3 @@
-from __future__ import absolute_import
-
-import json
 from pathlib import Path
 
 import git
@@ -19,9 +16,11 @@
     UserConfig,
 )
 
-from .utils import AsyncTyper
+from ..utils import AsyncTyper
+from . import legacy
 
 app = AsyncTyper()
+app.add_typer(legacy.app, name="legacy")
 
 
 @app.command()
@@ -49,7 +48,7 @@ def generate_cs(
         Users={},
         Groups={
             user_group: GroupConfig(
-                JobShare=None, Properties=["NormalUser"], Quota=None, Users=[]
+                JobShare=None, Properties={"NormalUser"}, Quota=None, Users=set()
             )
         },
     )
@@ -62,8 +61,7 @@ def generate_cs(
     repo = git.Repo.init(repo_path, initial_branch="master")
     yaml_path = repo_path / "default.yml"
     typer.echo(f"Writing configuration to {yaml_path}", err=True)
-    config_data = json.loads(config.json(exclude_unset=True))
-    yaml_path.write_text(yaml.safe_dump(config_data))
+    yaml_path.write_text(yaml.safe_dump(config.dict(exclude_unset=True)))
     repo.index.add([yaml_path.relative_to(repo_path)])
     repo.index.commit("Initial commit")
     typer.echo(f"Successfully created repo in {config_repo}", err=True)
@@ -76,8 +74,6 @@ def add_user(
     vo: str = "testvo",
     user_group: str = "user",
     sub: str = "usersub",
-    dn: str = "DN",
-    ca: str = "CA",
     preferred_username: str = "preferred_username",
 ):
     """Add a user to an existing vo and group"""
@@ -87,7 +83,7 @@ def add_user(
 
     repo_path = Path(config_repo.path)
 
-    new_user = UserConfig(CA=ca, DN=dn, PreferedUsername=preferred_username)
+    new_user = UserConfig(PreferedUsername=preferred_username)
 
     config = ConfigSource.create_from_url(backend_url=repo_path).read_config()
 
@@ -97,13 +93,12 @@ def add_user(
 
     config.Registry[vo].Users[sub] = new_user
 
-    config.Registry[vo].Groups[user_group].Users.append(sub)
+    config.Registry[vo].Groups[user_group].Users.add(sub)
 
     repo = git.Repo.init(repo_path)
     yaml_path = repo_path / "default.yml"
     typer.echo(f"Writing back configuration to {yaml_path}", err=True)
-    config_data = json.loads(config.json(exclude_unset=True))
-    yaml_path.write_text(yaml.safe_dump(config_data))
+    yaml_path.write_text(yaml.safe_dump(config.dict(exclude_unset=True)))
     repo.index.add([yaml_path.relative_to(repo_path)])
     repo.index.commit(
         f"Added user {sub} ({preferred_username}) to vo {vo} and user_group {user_group}"

src/diracx/cli/internal/legacy.py

@@ -0,0 +1,138 @@
+import os
+from pathlib import Path
+
+import diraccfg
+import yaml
+from pydantic import BaseModel
+
+from diracx.core.config import Config
+
+from ..utils import AsyncTyper
+
+app = AsyncTyper()
+
+
+class IdPConfig(BaseModel):
+    URL: str
+    ClientID: str
+
+
+class VOConfig(BaseModel):
+    DefaultGroup: str
+    IdP: IdPConfig
+    UserSubjects: dict[str, str]
+
+
+class ConversionConfig(BaseModel):
+    VOs: dict[str, VOConfig]
+
+
+# def parse_args():
+#     parser = argparse.ArgumentParser("Convert the legacy DIRAC CS to the new format")
+#     parser.add_argument("old_file", type=Path)
+#     parser.add_argument("conversion_config", type=Path)
+#     parser.add_argument("repo", type=Path)
+#     args = parser.parse_args()
+
+
+#     main(args.old_file, args.conversion_config, args.repo / DEFAULT_CONFIG_FILE)
+
+
+@app.command()
+def cs_sync(old_file: Path, conversion_config: Path, new_file: Path):
+    """Load the old CS and convert it to the new YAML format"""
+    if not os.environ.get("DIRAC_COMPAT_ENABLE_CS_CONVERSION"):
+        raise RuntimeError(
+            "DIRAC_COMPAT_ENABLE_CS_CONVERSION must be set for the conversion to be possible"
+        )
+
+    old_data = old_file.read_text()
+    cfg = diraccfg.CFG().loadFromBuffer(old_data)
+    raw = cfg.getAsDict()
+
+    _apply_fixes(raw, conversion_config)
+
+    config = Config.parse_obj(raw)
+    new_file.write_text(yaml.safe_dump(config.dict(exclude_unset=True)))
+
+
+def _apply_fixes(raw, conversion_config: Path):
+    """Modify raw in place to make any layout changes between the old and new structure"""
+
+    conv_config = ConversionConfig.parse_obj(
+        yaml.safe_load(conversion_config.read_text())
+    )
+
+    raw.pop("DiracX", None)
+    # Remove dips specific parts from the CS
+    raw["DIRAC"].pop("Extensions", None)
+    raw["DIRAC"].pop("Framework", None)
+    raw["DIRAC"].pop("Security", None)
+
+    # This is VOMS specific and no longer reqired
+    raw["DIRAC"].pop("ConnConf", None)
+
+    # Setups are no longer supported
+    raw["DIRAC"].pop("DefaultSetup", None)
+    raw["DIRAC"].pop("Setups", None)
+    raw["DIRAC"].pop("Configuration", None)
+
+    # All installations are no multi-VO
+    raw["DIRAC"].pop("VirtualOrganization", None)
+
+    # The default group now lives in /Registry
+    raw["DIRAC"].pop("DefaultGroup", None)
+
+    # Check that we have the config for all the VOs
+    vos = set(raw["Registry"]["VO"])
+    if non_configured_vos := vos - set(conv_config.VOs):
+        print(f"{non_configured_vos} don't have a migration config, ignoring")
+
+    # Modify the registry to be fully multi-VO
+    original_registry = raw.pop("Registry")
+    raw["Registry"] = {}
+
+    for vo, vo_meta in conv_config.VOs.items():
+        raw["Registry"][vo] = {
+            "IdP": vo_meta.IdP,
+            "DefaultGroup": vo_meta.DefaultGroup,
+            "Users": {},
+            "Groups": {},
+        }
+        if "DefaultStorageQuota" in original_registry:
+            raw["Registry"][vo]["DefaultStorageQuota"] = original_registry[
+                "DefaultStorageQuota"
+            ]
+        if "DefaultProxyLifeTime" in original_registry:
+            raw["Registry"][vo]["DefaultProxyLifeTime"] = original_registry[
+                "DefaultProxyLifeTime"
+            ]
+        # Find the groups that belong to this VO
+        vo_users = set()
+        for name, info in original_registry["Groups"].items():
+            if "VO" not in info:
+                print(
+                    f"Can't convert group {name} because it is not associated to any VO"
+                )
+                continue
+            if info.get("VO", None) == vo:
+                raw["Registry"][vo]["Groups"][name] = {
+                    k: v for k, v in info.items() if k not in {"IdPRole", "VO"}
+                }
+                nicknames = {u.strip() for u in info["Users"].split(",") if u.strip()}
+                vo_users |= nicknames
+                raw["Registry"][vo]["Groups"][name]["Users"] = [
+                    vo_meta.UserSubjects[n]
+                    for n in nicknames
+                    if n in vo_meta.UserSubjects
+                ]
+        # Find the users that belong to this VO
+        for name, info in original_registry["Users"].items():
+            if name in vo_users:
+                if subject := vo_meta.UserSubjects.get(name):
+                    raw["Registry"][vo]["Users"][subject] = info | {
+                        "PreferedUsername": name
+                    }
+                    # We ignore the DN and CA
+                    raw["Registry"][vo]["Users"][subject].pop("DN", None)
+                    raw["Registry"][vo]["Users"][subject].pop("CA", None)

src/diracx/core/config/__init__.py

@@ -70,7 +70,9 @@ def create(cls):
         return cls.create_from_url(backend_url=os.environ["DIRACX_CONFIG_BACKEND_URL"])
 
     @classmethod
-    def create_from_url(cls, *, backend_url: ConfigSourceUrl | Path | str):
+    def create_from_url(
+        cls, *, backend_url: ConfigSourceUrl | Path | str
+    ) -> ConfigSource:
         url = parse_obj_as(ConfigSourceUrl, str(backend_url))
         return cls.__registry[url.scheme](backend_url=url)
 

src/diracx/core/config/schema.py

@@ -22,9 +22,11 @@ def legacy_adaptor(cls, v):
         # though ideally we should parse the type hints properly.
         for field, hint in cls.__annotations__.items():
             # Convert comma separated lists to actual lists
-            if hint in {"list[str]", "list[SecurityProperty]"} and isinstance(
-                v.get(field), str
-            ):
+            if hint in {
+                "list[str]",
+                "set[str]",
+                "set[SecurityProperty]",
+            } and isinstance(v.get(field), str):
                 v[field] = [x.strip() for x in v[field].split(",") if x.strip()]
             # If the field is optional and the value is "None" convert it to None
             if "| None" in hint and field in v:
@@ -34,8 +36,6 @@ def legacy_adaptor(cls, v):
 
 
 class UserConfig(BaseModel):
-    CA: str
-    DN: str
     PreferedUsername: str
     Email: EmailStr | None
     Suspended: list[str] = []
@@ -50,9 +50,9 @@ class GroupConfig(BaseModel):
     AutoUploadPilotProxy: bool = False
     AutoUploadProxy: bool = False
     JobShare: Optional[int]
-    Properties: list[SecurityProperty]
+    Properties: set[SecurityProperty]
     Quota: Optional[int]
-    Users: list[str]
+    Users: set[str]
     AllowBackgroundTQs: bool = False
     VOMSRole: Optional[str]
     AutoSyncVOMS: bool = False

src/diracx/routers/auth.py

@@ -267,7 +267,8 @@ async def exchange_token(
 
     # Extract attributes from the settings and configuration
     issuer = settings.token_issuer
-    dirac_properties = config.Registry[vo].Groups[dirac_group].Properties
+    # dirac_properties needs to be a list in the token as to be json serializable
+    dirac_properties = sorted(config.Registry[vo].Groups[dirac_group].Properties)
 
     # Check that the subject is part of the dirac users
     if sub not in config.Registry[vo].Groups[dirac_group].Users:

tests/cli/legacy/cs_sync/convert_integration_test.yaml

@@ -0,0 +1,19 @@
+VOs:
+  Jenkins:
+    DefaultGroup: jenkins_user
+    IdP:
+      ClientID: 995ed3b9-d5bd-49d3-a7f4-7fc7dbd5a0cd
+      URL: https://jenkins.invalid/
+    UserSubjects:
+      adminusername: e2cb28ec-1a1e-40ee-a56d-d899b79879ce
+      ciuser: 26dbe36e-cf5c-4c52-a834-29a1c904ef74
+      trialUser: a95ab678-3fa4-41b9-b863-fe62ce8064ce
+  vo:
+    DefaultGroup: dirac_user
+    IdP:
+      ClientID: 072afab5-ed92-46e0-a61d-4ecbc96e0770
+      URL: https://vo.invalid/
+    UserSubjects:
+      adminusername: 26b14fc9-6d40-4ca5-b014-6234eaf0fb6e
+      ciuser: d3adc733-6588-4d6f-8581-5986b02d0c87
+      trialUser: ff2152ff-34f4-4739-b106-3def37e291e3