From f577c63002028571692a60834b2e7506494e251c Mon Sep 17 00:00:00 2001 From: Christophe Haen Date: Tue, 17 Oct 2023 14:04:34 +0200 Subject: [PATCH 1/2] Lower log level to WARNING --- src/diracx/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/diracx/__init__.py b/src/diracx/__init__.py index a3755310..f80e8beb 100644 --- a/src/diracx/__init__.py +++ b/src/diracx/__init__.py @@ -2,7 +2,7 @@ from importlib.metadata import PackageNotFoundError, version logging.basicConfig( - level=logging.DEBUG, format="%(asctime)s | %(name)s | %(levelname)s | %(message)s" + level=logging.WARNING, format="%(asctime)s | %(name)s | %(levelname)s | %(message)s" ) try: From a4fe61f1c002cf6a72b6acf9d00e3865fbf5845d Mon Sep 17 00:00:00 2001 From: Christophe Haen Date: Tue, 17 Oct 2023 14:05:30 +0200 Subject: [PATCH 2/2] Add "dirac internal legacy generate-helm-values" --- src/diracx/cli/internal/legacy.py | 138 ++++++++++++++++++ tests/cli/legacy/cs_sync/integration_test.cfg | 5 + .../cs_sync/integration_test_secret.cfg | 136 +++++++++++++++++ tests/cli/legacy/test_legacy.py | 33 +++++ 4 files changed, 312 insertions(+) create mode 100644 tests/cli/legacy/cs_sync/integration_test_secret.cfg create mode 100644 tests/cli/legacy/test_legacy.py diff --git a/src/diracx/cli/internal/legacy.py b/src/diracx/cli/internal/legacy.py index c289e0aa..f7cefea9 100644 --- a/src/diracx/cli/internal/legacy.py +++ b/src/diracx/cli/internal/legacy.py @@ -1,9 +1,16 @@ +import base64 +import hashlib +import json import os from pathlib import Path +from typing import cast +from urllib.parse import urljoin, urlparse import diraccfg +import typer import yaml from pydantic import BaseModel +from typer import Option from diracx.core.config import Config @@ -136,3 +143,134 @@ def _apply_fixes(raw, conversion_config: Path): # We ignore the DN and CA raw["Registry"][vo]["Users"][subject].pop("DN", None) raw["Registry"][vo]["Users"][subject].pop("CA", None) + + +@app.command() +def generate_helm_values( + public_cfg: Path = Option(help="Path to the cfg file served by the CS"), + secret_cfg: Path = Option( + default=None, help="Path to the cfg containing the secret" + ), + output_file: Path = Option(help="Where to dump the yam file"), +): + """Generate an initial values.yaml to run a DiracX installation + compatible with a DIRAC instance. The file is not complete, and needs + manual editing""" + + helm_values = { + "developer": {"enabled": False}, + "init-cs": {"enabled": True}, + "init-secrets": {"enabled": True}, + "init-sql": {"enabled": False, "env": {}}, + "cert-manager": {"enabled": False}, + "cert-manager-issuer": {"enabled": False}, + "minio": {"enabled": False}, + "dex": {"enabled": False}, + "opensearch": {"enabled": False}, + "ingress": { + "enabled": True, + "className": None, + "tlsSecretName": None, + "annotations": { + "route.openshift.io/termination": "edge", + "haproxy.router.openshift.io/ip_whitelist": "", + }, + }, + "rabbitmq": {"enabled": False}, + "mysql": {"enabled": False}, + "diracx": { + "manageOSIndices": False, + "mysqlDatabases": [], + "osDatabases": [], + "settings": {}, + }, + } + + cfg = diraccfg.CFG().loadFromBuffer(public_cfg.read_text()) + + if secret_cfg: + cfg = cfg.mergeWith(diraccfg.CFG().loadFromBuffer(secret_cfg.read_text())) + + cfg = cast(dict, cfg.getAsDict()) + + diracx_url = cfg["DiracX"]["URL"] + diracx_hostname = urlparse(diracx_url).netloc.split(":", 1)[0] + # Remove the port + diracx_config = { + "manageOSIndices": False, + "mysqlDatabases": [], + "osDatabases": [], + "settings": {}, + } + + diracx_settings: dict[str, str] = {} + diracx_config["settings"] = diracx_settings + helm_values["diracx"] = diracx_config + diracx_config["hostname"] = diracx_hostname + + diracx_settings["DIRACX_SERVICE_AUTH_ALLOWED_REDIRECTS"] = json.dumps( + [ + urljoin(diracx_url, "api/docs/oauth2-redirect"), + urljoin(diracx_url, "/#authentication-callback"), + ] + ) + + default_db_user = cfg["Systems"].get("Databases", {}).get("User") + default_db_password = cfg["Systems"].get("Databases", {}).get("Password") + + default_setup = cfg["DIRAC"]["Setup"] + + all_db_configs = {} + for system, system_config in cfg["Systems"].items(): + system_setup = cfg["DIRAC"]["Setups"][default_setup].get(system, None) + if system_setup: + all_db_configs.update(system_config[system_setup].get("Databases", {})) + + from diracx.core.extensions import select_from_extension + + for entry_point in select_from_extension(group="diracx.db.sql"): + db_name = entry_point.name + # There is a DIRAC AuthDB, but it is not the same + # as the DiracX one + if db_name == "AuthDB": + url_name = "DIRACX_DB_URL_AUTHDB" + connection_string = "FILL ME: I am a new DB, create me" + else: + db_config = all_db_configs[db_name] + url_name = f"DIRACX_DB_URL_{entry_point.name.upper()}" + db_user = db_config.get("User", default_db_user) + db_password = db_config.get("Password", default_db_password) + db_host = db_config["Host"] + db_port = db_config["Port"] + indb_name = db_config["DBName"] + + connection_string = f"mysql+aiomysql://{db_user}:{db_password}@{db_host}:{db_port}/{indb_name}" + diracx_settings[url_name] = connection_string + + # Settings for the legacy + try: + diracx_settings["DIRACX_LEGACY_EXCHANGE_HASHED_API_KEY"] = hashlib.sha256( + base64.urlsafe_b64decode(cfg["DiracX"]["LegacyExchangeApiKey"]) + ).hexdigest() + except KeyError: + typer.echo( + "ERROR: you must have '/DiracX/LegacyExchangeApiKey' already set", err=True + ) + raise typer.Exit(1) from None + # Sandboxstore settings + # TODO: Integrate minio for production use (ingress, etc) + # By default, take the server hostname and prepend "sandboxes" + diracx_settings[ + "DIRACX_SANDBOX_STORE_BUCKET_NAME" + ] = f"{diracx_hostname.split('.')[0]}-sandboxes" + diracx_settings["DIRACX_SANDBOX_STORE_S3_CLIENT_KWARGS"] = json.dumps( + { + "endpoint_url": "FILL ME", + "aws_access_key_id": "FILL ME", + "aws_secret_access_key": "FILL ME", + } + ) + + diracx_settings["DIRACX_SERVICE_JOBS_ENABLED"] = "true" + diracx_settings["DIRACX_SANDBOX_STORE_AUTO_CREATE_BUCKET"] = "true" + output_file.write_text(yaml.safe_dump(helm_values)) diff --git a/tests/cli/legacy/cs_sync/integration_test.cfg b/tests/cli/legacy/cs_sync/integration_test.cfg index 1f34dfbe..d1da5b17 100644 --- a/tests/cli/legacy/cs_sync/integration_test.cfg +++ b/tests/cli/legacy/cs_sync/integration_test.cfg @@ -26,6 +26,11 @@ DIRAC } } } +DiracX +{ + URL = https://dirac-integration-tests.invalid:1234 + LegacyExchangeApiKey = diracx:legacy:ChangeME +} Systems { Configuration diff --git a/tests/cli/legacy/cs_sync/integration_test_secret.cfg b/tests/cli/legacy/cs_sync/integration_test_secret.cfg new file mode 100644 index 00000000..9e49d194 --- /dev/null +++ b/tests/cli/legacy/cs_sync/integration_test_secret.cfg @@ -0,0 +1,136 @@ +#This section determines which DIRAC components will be installed and where +LocalInstallation +{ + Release = rel-v8r0 + InstallType = server + TargetPath = /home/dirac/ServerInstallDIR + SiteName = DIRAC.Jenkins.ch + Setup = dirac-JenkinsSetup + InstanceName = Production + VirtualOrganization = vo + SkipCADownload = True + UseServerCertificate = True + #ConfigurationServer = https://myprimaryserver.name:8443/Configuration/Server + ConfigurationName = Production + #LogLevel of the installed components + LogLevel = DEBUG + AdminUserName = adminusername + #DN of the Admin user certificate (default: None ) + #In order the find out the DN that needs to be included in the Configuration for a given + #host or user certificate the following command can be used:: + #openssl x509 -noout -subject -enddate -in + AdminUserDN = /C=ch/O=DIRAC/OU=DIRAC CI/CN=ciuser + AdminUserEmail = lhcb-dirac-ci@cern.ch + AdminGroupName = dirac_admin + #DN of the host certificate (*) (default: None ) + HostDN = /C=ch/O=DIRAC/OU=DIRAC CI/CN=server + ConfigurationMaster = yes + Host = server + #List of Systems to be installed - by default all services are added + Systems = Accounting + Systems += Configuration + Systems += DataManagement + Systems += Framework + Systems += Monitoring + Systems += RequestManagement + Systems += ResourceStatus + Systems += StorageManagement + Systems += Production + Systems += Transformation + Systems += WorkloadManagement + Systems += Tornado + #List of DataBases to be installed - minimal list for a running base server + Databases = InstalledComponentsDB + Databases += ResourceStatusDB + #List of Services to be installed - minimal list for a running base server + Services = Configuration/Server + Services += Framework/ComponentMonitoring + Services += Framework/SystemAdministrator + Services += ResourceStatus/ResourceStatus + Database + { + User = Dirac + Password = Dirac + RootUser = root + RootPwd = password + Host = mysql + Port = 3306 + } + NoSQLDatabase + { + User = elastic + Password = changeme + Host = elasticsearch + Port = 9200 + SSL = No + } +} +DIRAC +{ + Setup = dirac-JenkinsSetup + VirtualOrganization = vo + Hostname = server + Security + { + } + Setups + { + dirac-JenkinsSetup + { + Configuration = Production + Accounting = Production + DataManagement = Production + Framework = Production + Monitoring = Production + RequestManagement = Production + ResourceStatus = Production + StorageManagement = Production + Production = Production + Transformation = Production + WorkloadManagement = Production + Tornado = Production + } + } + Configuration + { + Master = yes + Name = Production + Servers = dips://server:9135/Configuration/Server + } +} +LocalSite +{ + Site = DIRAC.Jenkins.ch +} +Systems +{ + Databases + { + User = Dirac + Password = Dirac + Host = mysql + Port = 3306 + } + NoSQLDatabases + { + Host = elasticsearch + Port = 9200 + User = elastic + Password = changeme + SSL = No + } +} +Resources +{ + StorageElements + { + S3-INDIRECT + { + S3 + { + Aws_access_key_id = fakeId + Aws_secret_access_key = fakeKey + } + } + } +} diff --git a/tests/cli/legacy/test_legacy.py b/tests/cli/legacy/test_legacy.py new file mode 100644 index 00000000..f8c8c195 --- /dev/null +++ b/tests/cli/legacy/test_legacy.py @@ -0,0 +1,33 @@ +from pathlib import Path + +import yaml +from typer.testing import CliRunner + +from diracx.cli import app + +runner = CliRunner() + +file_path = Path(__file__).parent + + +def test_generate_helm_values(tmp_path, monkeypatch): + output_file = tmp_path / "values.yaml" + + result = runner.invoke( + app, + [ + "internal", + "legacy", + "generate-helm-values", + "--public-cfg", + str(file_path / "cs_sync" / "integration_test.cfg"), + "--secret-cfg", + str(file_path / "cs_sync" / "integration_test_secret.cfg"), + "--output-file", + str(output_file), + ], + ) + assert result.exit_code == 0 + assert output_file.is_file() + + assert isinstance(yaml.safe_load(output_file.read_text()), dict)