diff --git a/rsconf/component/mlflow.py b/rsconf/component/mlflow.py new file mode 100644 index 00000000..a8315619 --- /dev/null +++ b/rsconf/component/mlflow.py @@ -0,0 +1,68 @@ +"""MLFlow tracking server + +:copyright: Copyright (c) 2024 RadiaSoft LLC. All Rights Reserved. +:license: http://www.apache.org/licenses/LICENSE-2.0.html +""" + +from pykern import pkconfig +from pykern.pkcollections import PKDict +from pykern.pkdebug import pkdp +from rsconf import component + + +_DB_SUBDIR = "db" +_MLFLOW_IP = "127.0.0.1" +_BASIC_AUTH_CONF_F = "basic-auth.ini" +_BASIC_AUTH_DB_F = "basic-auth.db" +_SECRETS = {"mlflow_admin_password", "mlflow_admin_username"} + + +class T(component.T): + def internal_build_compile(self): + from rsconf import db + from rsconf import systemd + + self.buildt.require_component("docker", "network") + jc, z = self.j2_ctx_init() + z.run_d = systemd.unit_run_d(jc, self.name) + z.db_d = z.run_d.join(_DB_SUBDIR) + z.auth_conf_f = z.run_d.join(_BASIC_AUTH_CONF_F) + z.auth_db_f = z.run_d.join(_BASIC_AUTH_DB_F) + systemd.docker_unit_prepare( + self, + jc, + docker_exec=f"mlflow server --host {_MLFLOW_IP} --port {z.service_port} --backend-store-uri 'sqlite:///{z.db_d}/backend-store.db' --no-serve-artifacts --default-artifact-root '{z.db_d}' --app-name basic-auth", + ) + for s in _SECRETSz: + z[s] = self.secret_path_value( + s, + gen_secret=lambda: db.random_string(length=16), + visibility="host", + )[0] + + def internal_build_write(self): + from rsconf import systemd + from rsconf.component import docker_registry + from rsconf.component import nginx + + jc = self.j2_ctx + z = jc[self.name] + systemd.docker_unit_enable( + self, + jc, + env=self.python_service_env( + values=PKDict(mlflow_auth_config_path=z.auth_conf_f) + ), + image=docker_registry.absolute_image(self), + volumes=[z.db_d], + ) + self.install_access(mode="700", owner=jc.rsconf_db.run_u) + self.install_directory(z.db_d) + self.install_resource2(_BASIC_AUTH_CONF_F, z.auth_conf_f.dirname, access="400") + nginx.install_vhost( + self, + vhost=z.vhost, + backend_host=_MLFLOW_IP, + backend_port=z.service_port, + j2_ctx=jc, + ) diff --git a/rsconf/package_data/mlflow/basic-auth.ini.jinja b/rsconf/package_data/mlflow/basic-auth.ini.jinja new file mode 100644 index 00000000..ecb720f9 --- /dev/null +++ b/rsconf/package_data/mlflow/basic-auth.ini.jinja @@ -0,0 +1,5 @@ +[mlflow] +admin_password = {{ this.mlflow_admin_password }} +admin_username = {{ this.mlflow_admin_username }} +database_uri = sqlite:///{{ this.auth_db_f }} +default_permission = EDIT diff --git a/rsconf/package_data/mlflow/nginx.conf.jinja b/rsconf/package_data/mlflow/nginx.conf.jinja new file mode 100644 index 00000000..b4896721 --- /dev/null +++ b/rsconf/package_data/mlflow/nginx.conf.jinja @@ -0,0 +1,11 @@ +server { + listen {{ nginx.listen_ip }}:443 ssl; + server_name {{ nginx.vhost }}; + root {{ nginx.default_root }}; + ssl_certificate {{ nginx.tls_crt }}; + ssl_certificate_key {{ nginx.tls_key }}; + + location / { + proxy_pass http://{{nginx.backend_host}}:{{ nginx.backend_port }}/; + } +} diff --git a/tests/pkcli/build_data/1.in/db/000.yml b/tests/pkcli/build_data/1.in/db/000.yml index 1bd9c4e9..cae81ba2 100644 --- a/tests/pkcli/build_data/1.in/db/000.yml +++ b/tests/pkcli/build_data/1.in/db/000.yml @@ -502,6 +502,10 @@ host: v9.radia.run: jupyter.v9.radia.run jupyterhub_proxy: listen_any: True + mlflow: + docker_image: radiasoft/mlops + service_port: 8999 + vhost: mlflow.v9.radia.run network: restricted_public_tcp_ports: https: [ 192.168.1.0/24, 127.0.0.1 ] @@ -553,6 +557,7 @@ host: - github_bkp - jupyterhub - jupyterhub_proxy + - mlflow - nfs_client - nfs_server - rsaccounting diff --git a/tests/pkcli/build_data/1.in/db/secret/tls/mlflow.v9.radia.run.crt b/tests/pkcli/build_data/1.in/db/secret/tls/mlflow.v9.radia.run.crt new file mode 100644 index 00000000..63f362e7 --- /dev/null +++ b/tests/pkcli/build_data/1.in/db/secret/tls/mlflow.v9.radia.run.crt @@ -0,0 +1,20 @@ +-----BEGIN CERTIFICATE----- +MIIDRzCCAi+gAwIBAgIJAJYNdpgLaB/AMA0GCSqGSIb3DQEBCwUAMD4xCzAJBgNV +BAYTAlVTMREwDwYDVQQIDAhDb2xvcmFkbzEQMA4GA1UEBwwHQm91bGRlcjEKMAgG +A1UEAwwBbTAgFw0yNDAzMjIxNzUyMjRaGA8yMDUxMDgwNzE3NTIyNFowPjELMAkG +A1UEBhMCVVMxETAPBgNVBAgMCENvbG9yYWRvMRAwDgYDVQQHDAdCb3VsZGVyMQow +CAYDVQQDDAFtMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAzlR/whd4 +f1Z1E7x3XgSAobyT5ckv1/XBtnQkhTL3SmhTuukzp09I1S8Zw8y6W7yOenvumMlH +Alx8NU8PnU+NEge3pk16V0UB0gSXrCknzK0yB8sB2kPl5AWm/AWskV18TPEWaz3Z +9D6ZB7MZUX0ihvcI8uv1GF+UDfYXVYRH/stAzdO0iLwwqsTuRwqB1/6Fy0+4TEP/ +fns4UyuyiOQK1uOFZzlvkchxlz/MftUcwMUfMTvVdkW1AoGQ8XSDDBXJGFKbI8Bo +SBM2DUPz2Ni+3xPcDCZiBZYJCMu3b2x5itHTJOZATW2D9c4GOY7MZQaE5Vwx8DYX +2mgRji1Kk1vwtQIDAQABo0YwRDBCBgNVHREEOzA5ggFtggFsggFmggFsggFvggF3 +ggEuggF2ggE5ggEuggFyggFhggFkggFpggFhggEuggFyggF1ggFuMA0GCSqGSIb3 +DQEBCwUAA4IBAQAs2i/gzrW2IfzSUoPx6N6uL+p8oG4ElZ6AcH0E62kZA1xUvRxj +WxVWZ+u+VTJr8zWe6sDopRL7G//UUgT/E7EUs2bb3rATB2/T/MJrhkEFw1xCAMnV ++4Czh+SJ4RU0CXcmya3k8uj2+eJi3EPuu1k9yfLI11gzh+O7DxJVhqS8TwcGRnBe +iVUdGYvH1ekA4UnH8iYYtfehGafY9DC/AUv79o9Lynadan4/lz3iduY45GADMGjC +KiJptEOfyfvQ6L4vRch7U69i6RbxK2TzbgpgiXqzLHvW3KZB4yiYqOl3G/9k8MFN +yWiMu3F7T+bvJ1zOFEYNMC2YbIjV728adVKP +-----END CERTIFICATE----- diff --git a/tests/pkcli/build_data/1.in/db/secret/tls/mlflow.v9.radia.run.key b/tests/pkcli/build_data/1.in/db/secret/tls/mlflow.v9.radia.run.key new file mode 100644 index 00000000..7917b7b9 --- /dev/null +++ b/tests/pkcli/build_data/1.in/db/secret/tls/mlflow.v9.radia.run.key @@ -0,0 +1,28 @@ +-----BEGIN PRIVATE KEY----- +MIIEvAIBADANBgkqhkiG9w0BAQEFAASCBKYwggSiAgEAAoIBAQDOVH/CF3h/VnUT +vHdeBIChvJPlyS/X9cG2dCSFMvdKaFO66TOnT0jVLxnDzLpbvI56e+6YyUcCXHw1 +Tw+dT40SB7emTXpXRQHSBJesKSfMrTIHywHaQ+XkBab8BayRXXxM8RZrPdn0PpkH +sxlRfSKG9wjy6/UYX5QN9hdVhEf+y0DN07SIvDCqxO5HCoHX/oXLT7hMQ/9+ezhT +K7KI5ArW44VnOW+RyHGXP8x+1RzAxR8xO9V2RbUCgZDxdIMMFckYUpsjwGhIEzYN +Q/PY2L7fE9wMJmIFlgkIy7dvbHmK0dMk5kBNbYP1zgY5jsxlBoTlXDHwNhfaaBGO +LUqTW/C1AgMBAAECggEAUKzXe3Oj6N5MTtg9MCTAivzqM0nUDDQKdyW4deB0ssJS +It42FTA6ASk2gMmXAHGcoCW/KDxjKHgzFMECEPde6HLeCwd2U5Mm4BBtoaJB7pS1 +4DgvVEGCLQNMxHQPgpM0G/2UT4BSrV6ghVMxDhzImE8VT66VGd+dS0wy2XwsjmIF +DWXUFXXLD6QPvOegPS2sxU1x07rKej7f4LBohJvPlNo1DkAiC1TgQPX/BcEvRFcy +eZydxVs3wMt0C3G462Mn83ne4HV+qBDnbzN29f/AXbdt6T5cBEAgAzrUI15d2ahd +Cx32ZcR++NGsAolsYHMslh1sOGLuq1EJB74KzEmjLQKBgQDl59+AxznGiuYVEJVC +DwOK9v1fxg+Y8s7gXpaNb8awe44+83eSRSfMpEMK66Qs9zaMTyNw/GfMDRsql9z3 +VGWM+02ZKGVKHxcxXU4ui+2Y5kXAYxU15B2BD6f7Q7WJydPhkBCpXu1CRYJvweeo +lZC3LLUtSoZ5BTDP4VSAyc9atwKBgQDlv5zQDOWkdpMZQrFCVaTVna7Qb3Xxygx9 +CyXPwuK1VM/EOYrQtPyEzVyQENisMrWoTd5+xUiGWELqgw3G1CZdttdULaSihnUi +kS5dIzZPr70O9MKtKyuA9N2HqPorL4ML82lzNZALNW3GraA3rqgdNfW/d3g5VumC +TeYZjlzT8wKBgBsU/VcGIOAqbH/+vBZT/m7YOXCWvC+gKo5zlfiDOjx/0XnI5ETu +sImCsAMRTv2dAWf1yo78rJ10zcZurTDVWEELOVDZWVUp5GmEPDlvckWYmo7XHSCE +BpW1amxGxZO9mp/vgIbzD6/G0F6p0MgjFjD9qznylXScD+vs9y9UloBxAoGALMcE +pBPvafdmQgUakSgqASxDVwJyRVawymoyy2RbVSYbfE4OvRCZKrTvtnBiGmWjvBK5 +K5H9yZlIKXazZ64PcDJjk1d1/+sex0bud16Whj9lJJYVnzSLvQ18Y6VOZL5U1y5w +vfCRi221YISUdmXHBDJxHAkH2H0U872E/DQf6XsCgYAw5/xBoXcBafwVYBzCMziF +xllrICXrpxGJEU1+MMnTJCtWiSYq6sOvZQTacNa9Pyr/SmNfSmuzJAO3yIXxo8WP +4ja2NxSgMGuomN8zMzRLiZhm5s1+BDIfh2kg9DA8Adv/0wctiCM8gtWfKm4l9Yix +NA7bqCalMR+wcbGgvoOnAQ== +-----END PRIVATE KEY----- diff --git a/tests/pkcli/build_data/1.in/db/secret/v9.radia.run/mlflow_admin_password b/tests/pkcli/build_data/1.in/db/secret/v9.radia.run/mlflow_admin_password new file mode 100644 index 00000000..6fe534b9 --- /dev/null +++ b/tests/pkcli/build_data/1.in/db/secret/v9.radia.run/mlflow_admin_password @@ -0,0 +1 @@ +T3W4dhNjaSlwqjqZ \ No newline at end of file diff --git a/tests/pkcli/build_data/1.in/db/secret/v9.radia.run/mlflow_admin_username b/tests/pkcli/build_data/1.in/db/secret/v9.radia.run/mlflow_admin_username new file mode 100644 index 00000000..3d66ee1b --- /dev/null +++ b/tests/pkcli/build_data/1.in/db/secret/v9.radia.run/mlflow_admin_username @@ -0,0 +1 @@ +oyVQh9UIGyYK1rmL \ No newline at end of file diff --git a/tests/pkcli/build_data/1.out/srv/host/v9.radia.run/000.sh b/tests/pkcli/build_data/1.out/srv/host/v9.radia.run/000.sh index a46c9c36..61057217 100644 --- a/tests/pkcli/build_data/1.out/srv/host/v9.radia.run/000.sh +++ b/tests/pkcli/build_data/1.out/srv/host/v9.radia.run/000.sh @@ -26,6 +26,7 @@ rsconf_require dovecot rsconf_require github_bkp rsconf_require jupyterhub rsconf_require jupyterhub_proxy +rsconf_require mlflow rsconf_require nfs_client rsconf_require nfs_server rsconf_require rsaccounting diff --git a/tests/pkcli/build_data/1.out/srv/host/v9.radia.run/mlflow.sh b/tests/pkcli/build_data/1.out/srv/host/v9.radia.run/mlflow.sh new file mode 100644 index 00000000..cd455ce2 --- /dev/null +++ b/tests/pkcli/build_data/1.out/srv/host/v9.radia.run/mlflow.sh @@ -0,0 +1,23 @@ +#!/bin/bash +mlflow_rsconf_component() { +rsconf_service_prepare 'mlflow' '/etc/systemd/system/mlflow.service' '/etc/systemd/system/mlflow.service.d' '/srv/mlflow' +rsconf_install_access '700' 'vagrant' 'vagrant' +rsconf_install_directory '/srv/mlflow' +rsconf_install_access '500' 'vagrant' 'vagrant' +rsconf_install_file '/srv/mlflow/cmd' '883ef20d348691bc33dd992112ed9985' +rsconf_install_file '/srv/mlflow/env' '01f7ee885371f9fd9a8042017614adf9' +rsconf_install_file '/srv/mlflow/remove' 'e8253886ccf08377b23c8cc32c4b64fa' +rsconf_install_file '/srv/mlflow/start' '1ad20b9401cf3eaf5191245a26475dc0' +rsconf_install_file '/srv/mlflow/stop' 'fe05a526f775d8c47ebca4412dc6b432' +rsconf_install_access '444' 'root' 'root' +rsconf_install_file '/etc/systemd/system/mlflow.service' 'b370658c6db4685dd6c38153694b8786' +rsconf_service_docker_pull 'v3.radia.run:5000/radiasoft/mlops:dev' 'mlflow' 'mlflow' '' +rsconf_install_access '700' 'vagrant' 'vagrant' +rsconf_install_directory '/srv/mlflow/db' +rsconf_install_access '400' 'vagrant' 'vagrant' +rsconf_install_file '/srv/mlflow/basic-auth.ini' '0e4ac81b2bd4cc356905d3b25f0989b3' +rsconf_install_access '400' 'root' 'root' +rsconf_install_file '/etc/nginx/conf.d/mlflow.v9.radia.run.key' '97cef7711a10368c80d469627acf4814' +rsconf_install_file '/etc/nginx/conf.d/mlflow.v9.radia.run.crt' 'c832c6b9d565a6ceb76b15bde9a24e52' +rsconf_install_file '/etc/nginx/conf.d/mlflow.v9.radia.run.conf' '69dd677938428b9501c0b3c4c447bba5' +} diff --git a/tests/pkcli/build_data/1.out/srv/host/v9.radia.run/srv/mlflow/basic-auth.ini b/tests/pkcli/build_data/1.out/srv/host/v9.radia.run/srv/mlflow/basic-auth.ini new file mode 100644 index 00000000..bebe8257 --- /dev/null +++ b/tests/pkcli/build_data/1.out/srv/host/v9.radia.run/srv/mlflow/basic-auth.ini @@ -0,0 +1,5 @@ +[mlflow] +admin_password = T3W4dhNjaSlwqjqZ +admin_username = oyVQh9UIGyYK1rmL +database_uri = sqlite:////srv/mlflow/basic-auth.db +default_permission = EDIT diff --git a/tests/pkcli/build_data/1.out/srv/host/v9.radia.run/srv/mlflow/env b/tests/pkcli/build_data/1.out/srv/host/v9.radia.run/srv/mlflow/env new file mode 100644 index 00000000..7ecce43e --- /dev/null +++ b/tests/pkcli/build_data/1.out/srv/host/v9.radia.run/srv/mlflow/env @@ -0,0 +1,4 @@ +#!/bin/bash +export 'MLFLOW_AUTH_CONFIG_PATH=/srv/mlflow/basic-auth.ini' +export 'PYTHONUNBUFFERED=1' +export 'TZ=:/etc/localtime'