Merge pull request #140 from chaen/generate_values_yaml

Starting point for "dirac internal legacy generate-helm-values"

src/diracx/__init__.py

@@ -2,7 +2,7 @@
 from importlib.metadata import PackageNotFoundError, version
 
 logging.basicConfig(
-    level=logging.DEBUG, format="%(asctime)s | %(name)s | %(levelname)s | %(message)s"
+    level=logging.WARNING, format="%(asctime)s | %(name)s | %(levelname)s | %(message)s"
 )
 
 try:

src/diracx/cli/internal/legacy.py

@@ -1,9 +1,16 @@
+import base64
+import hashlib
+import json
 import os
 from pathlib import Path
+from typing import cast
+from urllib.parse import urljoin, urlparse
 
 import diraccfg
+import typer
 import yaml
 from pydantic import BaseModel
+from typer import Option
 
 from diracx.core.config import Config
 
@@ -136,3 +143,134 @@ def _apply_fixes(raw, conversion_config: Path):
                     # We ignore the DN and CA
                     raw["Registry"][vo]["Users"][subject].pop("DN", None)
                     raw["Registry"][vo]["Users"][subject].pop("CA", None)
+
+
+@app.command()
+def generate_helm_values(
+    public_cfg: Path = Option(help="Path to the cfg file served by the CS"),
+    secret_cfg: Path = Option(
+        default=None, help="Path to the cfg containing the secret"
+    ),
+    output_file: Path = Option(help="Where to dump the yam file"),
+):
+    """Generate an initial values.yaml to run a DiracX installation
+    compatible with a DIRAC instance. The file is not complete, and needs
+    manual editing"""
+
+    helm_values = {
+        "developer": {"enabled": False},
+        "init-cs": {"enabled": True},
+        "init-secrets": {"enabled": True},
+        "init-sql": {"enabled": False, "env": {}},
+        "cert-manager": {"enabled": False},
+        "cert-manager-issuer": {"enabled": False},
+        "minio": {"enabled": False},
+        "dex": {"enabled": False},
+        "opensearch": {"enabled": False},
+        "ingress": {
+            "enabled": True,
+            "className": None,
+            "tlsSecretName": None,
+            "annotations": {
+                "route.openshift.io/termination": "edge",
+                "haproxy.router.openshift.io/ip_whitelist": "",
+            },
+        },
+        "rabbitmq": {"enabled": False},
+        "mysql": {"enabled": False},
+        "diracx": {
+            "manageOSIndices": False,
+            "mysqlDatabases": [],
+            "osDatabases": [],
+            "settings": {},
+        },
+    }
+
+    cfg = diraccfg.CFG().loadFromBuffer(public_cfg.read_text())
+
+    if secret_cfg:
+        cfg = cfg.mergeWith(diraccfg.CFG().loadFromBuffer(secret_cfg.read_text()))
+
+    cfg = cast(dict, cfg.getAsDict())
+
+    diracx_url = cfg["DiracX"]["URL"]
+    diracx_hostname = urlparse(diracx_url).netloc.split(":", 1)[0]
+    # Remove the port
+    diracx_config = {
+        "manageOSIndices": False,
+        "mysqlDatabases": [],
+        "osDatabases": [],
+        "settings": {},
+    }
+
+    diracx_settings: dict[str, str] = {}
+    diracx_config["settings"] = diracx_settings
+    helm_values["diracx"] = diracx_config
+    diracx_config["hostname"] = diracx_hostname
+
+    diracx_settings["DIRACX_SERVICE_AUTH_ALLOWED_REDIRECTS"] = json.dumps(
+        [
+            urljoin(diracx_url, "api/docs/oauth2-redirect"),
+            urljoin(diracx_url, "/#authentication-callback"),
+        ]
+    )
+
+    default_db_user = cfg["Systems"].get("Databases", {}).get("User")
+    default_db_password = cfg["Systems"].get("Databases", {}).get("Password")
+
+    default_setup = cfg["DIRAC"]["Setup"]
+
+    all_db_configs = {}
+    for system, system_config in cfg["Systems"].items():
+        system_setup = cfg["DIRAC"]["Setups"][default_setup].get(system, None)
+        if system_setup:
+            all_db_configs.update(system_config[system_setup].get("Databases", {}))
+
+    from diracx.core.extensions import select_from_extension
+
+    for entry_point in select_from_extension(group="diracx.db.sql"):
+        db_name = entry_point.name
+        # There is a DIRAC AuthDB, but it is not the same
+        # as the DiracX one
+        if db_name == "AuthDB":
+            url_name = "DIRACX_DB_URL_AUTHDB"
+            connection_string = "FILL ME: I am a new DB, create me"
+        else:
+            db_config = all_db_configs[db_name]
+            url_name = f"DIRACX_DB_URL_{entry_point.name.upper()}"
+            db_user = db_config.get("User", default_db_user)
+            db_password = db_config.get("Password", default_db_password)
+            db_host = db_config["Host"]
+            db_port = db_config["Port"]
+            indb_name = db_config["DBName"]
+
+            connection_string = f"mysql+aiomysql://{db_user}:{db_password}@{db_host}:{db_port}/{indb_name}"
+        diracx_settings[url_name] = connection_string
+
+    # Settings for the legacy
+    try:
+        diracx_settings["DIRACX_LEGACY_EXCHANGE_HASHED_API_KEY"] = hashlib.sha256(
+            base64.urlsafe_b64decode(cfg["DiracX"]["LegacyExchangeApiKey"])
+        ).hexdigest()
+    except KeyError:
+        typer.echo(
+            "ERROR: you must have '/DiracX/LegacyExchangeApiKey' already set", err=True
+        )
+        raise typer.Exit(1) from None
+    # Sandboxstore settings
+    # TODO: Integrate minio for production use (ingress, etc)
+    # By default, take the server hostname and prepend "sandboxes"
+    diracx_settings[
+        "DIRACX_SANDBOX_STORE_BUCKET_NAME"
+    ] = f"{diracx_hostname.split('.')[0]}-sandboxes"
+    diracx_settings["DIRACX_SANDBOX_STORE_S3_CLIENT_KWARGS"] = json.dumps(
+        {
+            "endpoint_url": "FILL ME",
+            "aws_access_key_id": "FILL ME",
+            "aws_secret_access_key": "FILL ME",
+        }
+    )
+
+    diracx_settings["DIRACX_SERVICE_JOBS_ENABLED"] = "true"
+    diracx_settings["DIRACX_SANDBOX_STORE_AUTO_CREATE_BUCKET"] = "true"
+    output_file.write_text(yaml.safe_dump(helm_values))

tests/cli/legacy/cs_sync/integration_test.cfg

@@ -26,6 +26,11 @@ DIRAC
     }
   }
 }
+DiracX
+{
+  URL = https://dirac-integration-tests.invalid:1234
+  LegacyExchangeApiKey = diracx:legacy:ChangeME
+}
 Systems
 {
   Configuration

tests/cli/legacy/cs_sync/integration_test_secret.cfg

@@ -0,0 +1,136 @@
+#This section determines which DIRAC components will be installed and where
+LocalInstallation
+{
+  Release = rel-v8r0
+  InstallType = server
+  TargetPath = /home/dirac/ServerInstallDIR
+  SiteName = DIRAC.Jenkins.ch
+  Setup = dirac-JenkinsSetup
+  InstanceName = Production
+  VirtualOrganization = vo
+  SkipCADownload = True
+  UseServerCertificate = True
+  #ConfigurationServer = https://myprimaryserver.name:8443/Configuration/Server
+  ConfigurationName = Production
+  #LogLevel of the installed components
+  LogLevel = DEBUG
+  AdminUserName = adminusername
+  #DN of the Admin user certificate (default: None )
+  #In order the find out the DN that needs to be included in the Configuration for a given
+  #host or user certificate the following command can be used::
+  #openssl x509 -noout -subject -enddate -in <certfile.pem>
+  AdminUserDN = /C=ch/O=DIRAC/OU=DIRAC CI/CN=ciuser
+  AdminUserEmail = [email protected]
+  AdminGroupName = dirac_admin
+  #DN of the host certificate (*) (default: None )
+  HostDN = /C=ch/O=DIRAC/OU=DIRAC CI/CN=server
+  ConfigurationMaster = yes
+  Host = server
+  #List of Systems to be installed - by default all services are added
+  Systems = Accounting
+  Systems += Configuration
+  Systems += DataManagement
+  Systems += Framework
+  Systems += Monitoring
+  Systems += RequestManagement
+  Systems += ResourceStatus
+  Systems += StorageManagement
+  Systems += Production
+  Systems += Transformation
+  Systems += WorkloadManagement
+  Systems += Tornado
+  #List of DataBases to be installed - minimal list for a running base server
+  Databases = InstalledComponentsDB
+  Databases += ResourceStatusDB
+  #List of Services to be installed - minimal list for a running base server
+  Services = Configuration/Server
+  Services += Framework/ComponentMonitoring
+  Services += Framework/SystemAdministrator
+  Services += ResourceStatus/ResourceStatus
+  Database
+  {
+    User = Dirac
+    Password = Dirac
+    RootUser = root
+    RootPwd = password
+    Host = mysql
+    Port = 3306
+  }
+  NoSQLDatabase
+  {
+    User = elastic
+    Password = changeme
+    Host = elasticsearch
+    Port = 9200
+    SSL = No
+  }
+}
+DIRAC
+{
+  Setup = dirac-JenkinsSetup
+  VirtualOrganization = vo
+  Hostname = server
+  Security
+  {
+  }
+  Setups
+  {
+    dirac-JenkinsSetup
+    {
+      Configuration = Production
+      Accounting = Production
+      DataManagement = Production
+      Framework = Production
+      Monitoring = Production
+      RequestManagement = Production
+      ResourceStatus = Production
+      StorageManagement = Production
+      Production = Production
+      Transformation = Production
+      WorkloadManagement = Production
+      Tornado = Production
+    }
+  }
+  Configuration
+  {
+    Master = yes
+    Name = Production
+    Servers = dips://server:9135/Configuration/Server
+  }
+}
+LocalSite
+{
+  Site = DIRAC.Jenkins.ch
+}
+Systems
+{
+  Databases
+  {
+    User = Dirac
+    Password = Dirac
+    Host = mysql
+    Port = 3306
+  }
+  NoSQLDatabases
+  {
+    Host = elasticsearch
+    Port = 9200
+    User = elastic
+    Password = changeme
+    SSL = No
+  }
+}
+Resources
+{
+  StorageElements
+  {
+    S3-INDIRECT
+    {
+      S3
+      {
+        Aws_access_key_id = fakeId
+        Aws_secret_access_key = fakeKey
+      }
+    }
+  }
+}

tests/cli/legacy/test_legacy.py

@@ -0,0 +1,33 @@
+from pathlib import Path
+
+import yaml
+from typer.testing import CliRunner
+
+from diracx.cli import app
+
+runner = CliRunner()
+
+file_path = Path(__file__).parent
+
+
+def test_generate_helm_values(tmp_path, monkeypatch):
+    output_file = tmp_path / "values.yaml"
+
+    result = runner.invoke(
+        app,
+        [
+            "internal",
+            "legacy",
+            "generate-helm-values",
+            "--public-cfg",
+            str(file_path / "cs_sync" / "integration_test.cfg"),
+            "--secret-cfg",
+            str(file_path / "cs_sync" / "integration_test_secret.cfg"),
+            "--output-file",
+            str(output_file),
+        ],
+    )
+    assert result.exit_code == 0
+    assert output_file.is_file()
+
+    assert isinstance(yaml.safe_load(output_file.read_text()), dict)