From d9e613fc8c0ac50228690faaf797ac09d7b5d885 Mon Sep 17 00:00:00 2001 From: Evan Carlin Date: Fri, 22 Mar 2024 15:47:37 -0600 Subject: [PATCH] Fix #452: create mlflow component (#462) MLFlow Tracking Server as a service with basic-auth. It supports experiment parameter tracking but does not yet support artifact tracking. --- rsconf/component/bivio_named.py | 3 +- rsconf/component/devbox.py | 4 +- rsconf/component/mlflow.py | 68 +++++++++++++++++++ .../package_data/mlflow/basic-auth.ini.jinja | 5 ++ rsconf/package_data/mlflow/nginx.conf.jinja | 11 +++ tests/pkcli/build_data/1.in/db/000.yml | 5 ++ .../db/secret/tls/mlflow.v9.radia.run.crt | 20 ++++++ .../db/secret/tls/mlflow.v9.radia.run.key | 28 ++++++++ .../secret/v9.radia.run/mlflow_admin_password | 1 + .../secret/v9.radia.run/mlflow_admin_username | 1 + .../1.out/srv/host/v9.radia.run/000.sh | 39 ----------- .../1.out/srv/host/v9.radia.run/mlflow.sh | 23 +++++++ .../v9.radia.run/srv/mlflow/basic-auth.ini | 5 ++ .../srv/host/v9.radia.run/srv/mlflow/env | 4 ++ 14 files changed, 176 insertions(+), 41 deletions(-) create mode 100644 rsconf/component/mlflow.py create mode 100644 rsconf/package_data/mlflow/basic-auth.ini.jinja create mode 100644 rsconf/package_data/mlflow/nginx.conf.jinja create mode 100644 tests/pkcli/build_data/1.in/db/secret/tls/mlflow.v9.radia.run.crt create mode 100644 tests/pkcli/build_data/1.in/db/secret/tls/mlflow.v9.radia.run.key create mode 100644 tests/pkcli/build_data/1.in/db/secret/v9.radia.run/mlflow_admin_password create mode 100644 tests/pkcli/build_data/1.in/db/secret/v9.radia.run/mlflow_admin_username delete mode 100644 tests/pkcli/build_data/1.out/srv/host/v9.radia.run/000.sh create mode 100644 tests/pkcli/build_data/1.out/srv/host/v9.radia.run/mlflow.sh create mode 100644 tests/pkcli/build_data/1.out/srv/host/v9.radia.run/srv/mlflow/basic-auth.ini create mode 100644 tests/pkcli/build_data/1.out/srv/host/v9.radia.run/srv/mlflow/env diff --git a/rsconf/component/bivio_named.py b/rsconf/component/bivio_named.py index 1299d8ea..36f4c2dc 100644 --- a/rsconf/component/bivio_named.py +++ b/rsconf/component/bivio_named.py @@ -11,6 +11,7 @@ class T(component.T): def internal_build(self): + from rsconf import db from rsconf import systemd from rsconf.component import bop @@ -26,7 +27,7 @@ def internal_build(self): etc=run_d.join("etc"), ) z = jc.bivio_named - z.listen_on = "127.0.0.1;" + z.listen_on = f"{db.LOCAL_IP};" nc = self.buildt.get_component("network") nc.add_public_tcp_ports(["domain"]) nc.add_public_udp_ports(["domain"]) diff --git a/rsconf/component/devbox.py b/rsconf/component/devbox.py index a21ac7c5..4f6d811a 100644 --- a/rsconf/component/devbox.py +++ b/rsconf/component/devbox.py @@ -128,6 +128,8 @@ def _gen_secrets(self, jc): self.secrets = PKDict({k: s[k] for k in ("host_key_f", "identity_pub_f")}) def _jupyter_bashrc(self, jc, z, path): + from rsconf import db + self.install_access(mode="600") self.install_ensure_file_exists(path) for n in ("package_path", "sim_types"): @@ -144,7 +146,7 @@ def _jupyter_bashrc(self, jc, z, path): for n in ("DRIVER_LOCAL", "API"): self._env( f"SIREPO_JOB_{n}_SUPERVISOR_URI", - f"http://127.0.0.1:{z.job_supervisor_port}", + f"http://{db.LOCAL_IP}:{z.job_supervisor_port}", path, ) self._rsiviz(jc, z, path) diff --git a/rsconf/component/mlflow.py b/rsconf/component/mlflow.py new file mode 100644 index 00000000..6e3f405c --- /dev/null +++ b/rsconf/component/mlflow.py @@ -0,0 +1,68 @@ +"""MLFlow tracking server + +:copyright: Copyright (c) 2024 RadiaSoft LLC. All Rights Reserved. +:license: http://www.apache.org/licenses/LICENSE-2.0.html +""" + +from pykern import pkconfig +from pykern.pkcollections import PKDict +from pykern.pkdebug import pkdp +from rsconf import component + + +_DB_SUBDIR = "db" +_BASIC_AUTH_CONF_F = "basic-auth.ini" +_BASIC_AUTH_DB_F = "basic-auth.db" +_SECRETS = {"mlflow_admin_password", "mlflow_admin_username"} + + +class T(component.T): + def internal_build_compile(self): + from rsconf import db + from rsconf import systemd + + self.buildt.require_component("docker", "network") + jc, z = self.j2_ctx_init() + z.run_d = systemd.unit_run_d(jc, self.name) + z.db_d = z.run_d.join(_DB_SUBDIR) + z.auth_conf_f = z.run_d.join(_BASIC_AUTH_CONF_F) + z.auth_db_f = z.run_d.join(_BASIC_AUTH_DB_F) + z.mlflow_ip = db.LOCAL_IP + systemd.docker_unit_prepare( + self, + jc, + docker_exec=f"mlflow server --host {z.mlflow_ip} --port {z.service_port} --backend-store-uri 'sqlite:///{z.db_d}/backend-store.db' --no-serve-artifacts --default-artifact-root '{z.db_d}' --app-name basic-auth", + ) + for s in _SECRETS: + z[s] = self.secret_path_value( + s, + gen_secret=lambda: db.random_string(length=16), + visibility="host", + )[0] + + def internal_build_write(self): + from rsconf import systemd + from rsconf.component import docker_registry + from rsconf.component import nginx + + jc = self.j2_ctx + z = jc[self.name] + systemd.docker_unit_enable( + self, + jc, + env=self.python_service_env( + values=PKDict(mlflow_auth_config_path=z.auth_conf_f) + ), + image=docker_registry.absolute_image(self), + volumes=[z.db_d], + ) + self.install_access(mode="700", owner=jc.rsconf_db.run_u) + self.install_directory(z.db_d) + self.install_resource2(_BASIC_AUTH_CONF_F, z.auth_conf_f.dirname, access="400") + nginx.install_vhost( + self, + vhost=z.vhost, + backend_host=z.mlflow_ip, + backend_port=z.service_port, + j2_ctx=jc, + ) diff --git a/rsconf/package_data/mlflow/basic-auth.ini.jinja b/rsconf/package_data/mlflow/basic-auth.ini.jinja new file mode 100644 index 00000000..ecb720f9 --- /dev/null +++ b/rsconf/package_data/mlflow/basic-auth.ini.jinja @@ -0,0 +1,5 @@ +[mlflow] +admin_password = {{ this.mlflow_admin_password }} +admin_username = {{ this.mlflow_admin_username }} +database_uri = sqlite:///{{ this.auth_db_f }} +default_permission = EDIT diff --git a/rsconf/package_data/mlflow/nginx.conf.jinja b/rsconf/package_data/mlflow/nginx.conf.jinja new file mode 100644 index 00000000..b4896721 --- /dev/null +++ b/rsconf/package_data/mlflow/nginx.conf.jinja @@ -0,0 +1,11 @@ +server { + listen {{ nginx.listen_ip }}:443 ssl; + server_name {{ nginx.vhost }}; + root {{ nginx.default_root }}; + ssl_certificate {{ nginx.tls_crt }}; + ssl_certificate_key {{ nginx.tls_key }}; + + location / { + proxy_pass http://{{nginx.backend_host}}:{{ nginx.backend_port }}/; + } +} diff --git a/tests/pkcli/build_data/1.in/db/000.yml b/tests/pkcli/build_data/1.in/db/000.yml index 1bd9c4e9..cae81ba2 100644 --- a/tests/pkcli/build_data/1.in/db/000.yml +++ b/tests/pkcli/build_data/1.in/db/000.yml @@ -502,6 +502,10 @@ host: v9.radia.run: jupyter.v9.radia.run jupyterhub_proxy: listen_any: True + mlflow: + docker_image: radiasoft/mlops + service_port: 8999 + vhost: mlflow.v9.radia.run network: restricted_public_tcp_ports: https: [ 192.168.1.0/24, 127.0.0.1 ] @@ -553,6 +557,7 @@ host: - github_bkp - jupyterhub - jupyterhub_proxy + - mlflow - nfs_client - nfs_server - rsaccounting diff --git a/tests/pkcli/build_data/1.in/db/secret/tls/mlflow.v9.radia.run.crt b/tests/pkcli/build_data/1.in/db/secret/tls/mlflow.v9.radia.run.crt new file mode 100644 index 00000000..63f362e7 --- /dev/null +++ b/tests/pkcli/build_data/1.in/db/secret/tls/mlflow.v9.radia.run.crt @@ -0,0 +1,20 @@ +-----BEGIN CERTIFICATE----- +MIIDRzCCAi+gAwIBAgIJAJYNdpgLaB/AMA0GCSqGSIb3DQEBCwUAMD4xCzAJBgNV +BAYTAlVTMREwDwYDVQQIDAhDb2xvcmFkbzEQMA4GA1UEBwwHQm91bGRlcjEKMAgG +A1UEAwwBbTAgFw0yNDAzMjIxNzUyMjRaGA8yMDUxMDgwNzE3NTIyNFowPjELMAkG +A1UEBhMCVVMxETAPBgNVBAgMCENvbG9yYWRvMRAwDgYDVQQHDAdCb3VsZGVyMQow +CAYDVQQDDAFtMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAzlR/whd4 +f1Z1E7x3XgSAobyT5ckv1/XBtnQkhTL3SmhTuukzp09I1S8Zw8y6W7yOenvumMlH +Alx8NU8PnU+NEge3pk16V0UB0gSXrCknzK0yB8sB2kPl5AWm/AWskV18TPEWaz3Z +9D6ZB7MZUX0ihvcI8uv1GF+UDfYXVYRH/stAzdO0iLwwqsTuRwqB1/6Fy0+4TEP/ +fns4UyuyiOQK1uOFZzlvkchxlz/MftUcwMUfMTvVdkW1AoGQ8XSDDBXJGFKbI8Bo +SBM2DUPz2Ni+3xPcDCZiBZYJCMu3b2x5itHTJOZATW2D9c4GOY7MZQaE5Vwx8DYX +2mgRji1Kk1vwtQIDAQABo0YwRDBCBgNVHREEOzA5ggFtggFsggFmggFsggFvggF3 +ggEuggF2ggE5ggEuggFyggFhggFkggFpggFhggEuggFyggF1ggFuMA0GCSqGSIb3 +DQEBCwUAA4IBAQAs2i/gzrW2IfzSUoPx6N6uL+p8oG4ElZ6AcH0E62kZA1xUvRxj +WxVWZ+u+VTJr8zWe6sDopRL7G//UUgT/E7EUs2bb3rATB2/T/MJrhkEFw1xCAMnV ++4Czh+SJ4RU0CXcmya3k8uj2+eJi3EPuu1k9yfLI11gzh+O7DxJVhqS8TwcGRnBe +iVUdGYvH1ekA4UnH8iYYtfehGafY9DC/AUv79o9Lynadan4/lz3iduY45GADMGjC +KiJptEOfyfvQ6L4vRch7U69i6RbxK2TzbgpgiXqzLHvW3KZB4yiYqOl3G/9k8MFN +yWiMu3F7T+bvJ1zOFEYNMC2YbIjV728adVKP +-----END CERTIFICATE----- diff --git a/tests/pkcli/build_data/1.in/db/secret/tls/mlflow.v9.radia.run.key b/tests/pkcli/build_data/1.in/db/secret/tls/mlflow.v9.radia.run.key new file mode 100644 index 00000000..7917b7b9 --- /dev/null +++ b/tests/pkcli/build_data/1.in/db/secret/tls/mlflow.v9.radia.run.key @@ -0,0 +1,28 @@ +-----BEGIN PRIVATE KEY----- +MIIEvAIBADANBgkqhkiG9w0BAQEFAASCBKYwggSiAgEAAoIBAQDOVH/CF3h/VnUT +vHdeBIChvJPlyS/X9cG2dCSFMvdKaFO66TOnT0jVLxnDzLpbvI56e+6YyUcCXHw1 +Tw+dT40SB7emTXpXRQHSBJesKSfMrTIHywHaQ+XkBab8BayRXXxM8RZrPdn0PpkH +sxlRfSKG9wjy6/UYX5QN9hdVhEf+y0DN07SIvDCqxO5HCoHX/oXLT7hMQ/9+ezhT +K7KI5ArW44VnOW+RyHGXP8x+1RzAxR8xO9V2RbUCgZDxdIMMFckYUpsjwGhIEzYN +Q/PY2L7fE9wMJmIFlgkIy7dvbHmK0dMk5kBNbYP1zgY5jsxlBoTlXDHwNhfaaBGO +LUqTW/C1AgMBAAECggEAUKzXe3Oj6N5MTtg9MCTAivzqM0nUDDQKdyW4deB0ssJS +It42FTA6ASk2gMmXAHGcoCW/KDxjKHgzFMECEPde6HLeCwd2U5Mm4BBtoaJB7pS1 +4DgvVEGCLQNMxHQPgpM0G/2UT4BSrV6ghVMxDhzImE8VT66VGd+dS0wy2XwsjmIF +DWXUFXXLD6QPvOegPS2sxU1x07rKej7f4LBohJvPlNo1DkAiC1TgQPX/BcEvRFcy +eZydxVs3wMt0C3G462Mn83ne4HV+qBDnbzN29f/AXbdt6T5cBEAgAzrUI15d2ahd +Cx32ZcR++NGsAolsYHMslh1sOGLuq1EJB74KzEmjLQKBgQDl59+AxznGiuYVEJVC +DwOK9v1fxg+Y8s7gXpaNb8awe44+83eSRSfMpEMK66Qs9zaMTyNw/GfMDRsql9z3 +VGWM+02ZKGVKHxcxXU4ui+2Y5kXAYxU15B2BD6f7Q7WJydPhkBCpXu1CRYJvweeo +lZC3LLUtSoZ5BTDP4VSAyc9atwKBgQDlv5zQDOWkdpMZQrFCVaTVna7Qb3Xxygx9 +CyXPwuK1VM/EOYrQtPyEzVyQENisMrWoTd5+xUiGWELqgw3G1CZdttdULaSihnUi +kS5dIzZPr70O9MKtKyuA9N2HqPorL4ML82lzNZALNW3GraA3rqgdNfW/d3g5VumC +TeYZjlzT8wKBgBsU/VcGIOAqbH/+vBZT/m7YOXCWvC+gKo5zlfiDOjx/0XnI5ETu +sImCsAMRTv2dAWf1yo78rJ10zcZurTDVWEELOVDZWVUp5GmEPDlvckWYmo7XHSCE +BpW1amxGxZO9mp/vgIbzD6/G0F6p0MgjFjD9qznylXScD+vs9y9UloBxAoGALMcE +pBPvafdmQgUakSgqASxDVwJyRVawymoyy2RbVSYbfE4OvRCZKrTvtnBiGmWjvBK5 +K5H9yZlIKXazZ64PcDJjk1d1/+sex0bud16Whj9lJJYVnzSLvQ18Y6VOZL5U1y5w +vfCRi221YISUdmXHBDJxHAkH2H0U872E/DQf6XsCgYAw5/xBoXcBafwVYBzCMziF +xllrICXrpxGJEU1+MMnTJCtWiSYq6sOvZQTacNa9Pyr/SmNfSmuzJAO3yIXxo8WP +4ja2NxSgMGuomN8zMzRLiZhm5s1+BDIfh2kg9DA8Adv/0wctiCM8gtWfKm4l9Yix +NA7bqCalMR+wcbGgvoOnAQ== +-----END PRIVATE KEY----- diff --git a/tests/pkcli/build_data/1.in/db/secret/v9.radia.run/mlflow_admin_password b/tests/pkcli/build_data/1.in/db/secret/v9.radia.run/mlflow_admin_password new file mode 100644 index 00000000..6fe534b9 --- /dev/null +++ b/tests/pkcli/build_data/1.in/db/secret/v9.radia.run/mlflow_admin_password @@ -0,0 +1 @@ +T3W4dhNjaSlwqjqZ \ No newline at end of file diff --git a/tests/pkcli/build_data/1.in/db/secret/v9.radia.run/mlflow_admin_username b/tests/pkcli/build_data/1.in/db/secret/v9.radia.run/mlflow_admin_username new file mode 100644 index 00000000..3d66ee1b --- /dev/null +++ b/tests/pkcli/build_data/1.in/db/secret/v9.radia.run/mlflow_admin_username @@ -0,0 +1 @@ +oyVQh9UIGyYK1rmL \ No newline at end of file diff --git a/tests/pkcli/build_data/1.out/srv/host/v9.radia.run/000.sh b/tests/pkcli/build_data/1.out/srv/host/v9.radia.run/000.sh deleted file mode 100644 index a46c9c36..00000000 --- a/tests/pkcli/build_data/1.out/srv/host/v9.radia.run/000.sh +++ /dev/null @@ -1,39 +0,0 @@ -#!/bin/bash -export install_channel=dev -rsconf_require base_os -rsconf_require network -rsconf_require base_users -rsconf_require logrotate -rsconf_require base_all -rsconf_require bkp -rsconf_require postgresql -rsconf_require nginx -rsconf_require postgrey -rsconf_require spamd -rsconf_require postfix -rsconf_require db_bkp -rsconf_require petshop -rsconf_require bop -rsconf_require bop_timer -rsconf_require btest -rsconf_require docker -rsconf_require devbox_devtech3 -rsconf_require devbox_fullstackdude -rsconf_require devbox_custom_image -rsconf_require devbox_rsivizcoder -rsconf_require devbox -rsconf_require dovecot -rsconf_require github_bkp -rsconf_require jupyterhub -rsconf_require jupyterhub_proxy -rsconf_require nfs_client -rsconf_require nfs_server -rsconf_require rsaccounting -rsconf_require rsiviz -rsconf_require raydata_scan_monitor -rsconf_require sirepo_jupyterhub -rsconf_require sirepo_job_supervisor -rsconf_require sirepo -rsconf_require sirepo_test_http -rsconf_require vm_devbox_user-1 -rsconf_require vm_devbox diff --git a/tests/pkcli/build_data/1.out/srv/host/v9.radia.run/mlflow.sh b/tests/pkcli/build_data/1.out/srv/host/v9.radia.run/mlflow.sh new file mode 100644 index 00000000..cd455ce2 --- /dev/null +++ b/tests/pkcli/build_data/1.out/srv/host/v9.radia.run/mlflow.sh @@ -0,0 +1,23 @@ +#!/bin/bash +mlflow_rsconf_component() { +rsconf_service_prepare 'mlflow' '/etc/systemd/system/mlflow.service' '/etc/systemd/system/mlflow.service.d' '/srv/mlflow' +rsconf_install_access '700' 'vagrant' 'vagrant' +rsconf_install_directory '/srv/mlflow' +rsconf_install_access '500' 'vagrant' 'vagrant' +rsconf_install_file '/srv/mlflow/cmd' '883ef20d348691bc33dd992112ed9985' +rsconf_install_file '/srv/mlflow/env' '01f7ee885371f9fd9a8042017614adf9' +rsconf_install_file '/srv/mlflow/remove' 'e8253886ccf08377b23c8cc32c4b64fa' +rsconf_install_file '/srv/mlflow/start' '1ad20b9401cf3eaf5191245a26475dc0' +rsconf_install_file '/srv/mlflow/stop' 'fe05a526f775d8c47ebca4412dc6b432' +rsconf_install_access '444' 'root' 'root' +rsconf_install_file '/etc/systemd/system/mlflow.service' 'b370658c6db4685dd6c38153694b8786' +rsconf_service_docker_pull 'v3.radia.run:5000/radiasoft/mlops:dev' 'mlflow' 'mlflow' '' +rsconf_install_access '700' 'vagrant' 'vagrant' +rsconf_install_directory '/srv/mlflow/db' +rsconf_install_access '400' 'vagrant' 'vagrant' +rsconf_install_file '/srv/mlflow/basic-auth.ini' '0e4ac81b2bd4cc356905d3b25f0989b3' +rsconf_install_access '400' 'root' 'root' +rsconf_install_file '/etc/nginx/conf.d/mlflow.v9.radia.run.key' '97cef7711a10368c80d469627acf4814' +rsconf_install_file '/etc/nginx/conf.d/mlflow.v9.radia.run.crt' 'c832c6b9d565a6ceb76b15bde9a24e52' +rsconf_install_file '/etc/nginx/conf.d/mlflow.v9.radia.run.conf' '69dd677938428b9501c0b3c4c447bba5' +} diff --git a/tests/pkcli/build_data/1.out/srv/host/v9.radia.run/srv/mlflow/basic-auth.ini b/tests/pkcli/build_data/1.out/srv/host/v9.radia.run/srv/mlflow/basic-auth.ini new file mode 100644 index 00000000..bebe8257 --- /dev/null +++ b/tests/pkcli/build_data/1.out/srv/host/v9.radia.run/srv/mlflow/basic-auth.ini @@ -0,0 +1,5 @@ +[mlflow] +admin_password = T3W4dhNjaSlwqjqZ +admin_username = oyVQh9UIGyYK1rmL +database_uri = sqlite:////srv/mlflow/basic-auth.db +default_permission = EDIT diff --git a/tests/pkcli/build_data/1.out/srv/host/v9.radia.run/srv/mlflow/env b/tests/pkcli/build_data/1.out/srv/host/v9.radia.run/srv/mlflow/env new file mode 100644 index 00000000..7ecce43e --- /dev/null +++ b/tests/pkcli/build_data/1.out/srv/host/v9.radia.run/srv/mlflow/env @@ -0,0 +1,4 @@ +#!/bin/bash +export 'MLFLOW_AUTH_CONFIG_PATH=/srv/mlflow/basic-auth.ini' +export 'PYTHONUNBUFFERED=1' +export 'TZ=:/etc/localtime'