Fix #452: create mlflow component (#462)

MLFlow Tracking Server as a service with basic-auth. It supports
experiment parameter tracking but does not yet support artifact
tracking.

rsconf/component/bivio_named.py

@@ -11,6 +11,7 @@
 
 class T(component.T):
     def internal_build(self):
+        from rsconf import db
         from rsconf import systemd
         from rsconf.component import bop
 
@@ -26,7 +27,7 @@ def internal_build(self):
             etc=run_d.join("etc"),
         )
         z = jc.bivio_named
-        z.listen_on = "127.0.0.1;"
+        z.listen_on = f"{db.LOCAL_IP};"
         nc = self.buildt.get_component("network")
         nc.add_public_tcp_ports(["domain"])
         nc.add_public_udp_ports(["domain"])

rsconf/component/devbox.py

@@ -128,6 +128,8 @@ def _gen_secrets(self, jc):
         self.secrets = PKDict({k: s[k] for k in ("host_key_f", "identity_pub_f")})
 
     def _jupyter_bashrc(self, jc, z, path):
+        from rsconf import db
+
         self.install_access(mode="600")
         self.install_ensure_file_exists(path)
         for n in ("package_path", "sim_types"):
@@ -144,7 +146,7 @@ def _jupyter_bashrc(self, jc, z, path):
         for n in ("DRIVER_LOCAL", "API"):
             self._env(
                 f"SIREPO_JOB_{n}_SUPERVISOR_URI",
-                f"http://127.0.0.1:{z.job_supervisor_port}",
+                f"http://{db.LOCAL_IP}:{z.job_supervisor_port}",
                 path,
             )
         self._rsiviz(jc, z, path)

rsconf/component/mlflow.py

@@ -0,0 +1,68 @@
+"""MLFlow tracking server
+
+:copyright: Copyright (c) 2024 RadiaSoft LLC.  All Rights Reserved.
+:license: http://www.apache.org/licenses/LICENSE-2.0.html
+"""
+
+from pykern import pkconfig
+from pykern.pkcollections import PKDict
+from pykern.pkdebug import pkdp
+from rsconf import component
+
+
+_DB_SUBDIR = "db"
+_BASIC_AUTH_CONF_F = "basic-auth.ini"
+_BASIC_AUTH_DB_F = "basic-auth.db"
+_SECRETS = {"mlflow_admin_password", "mlflow_admin_username"}
+
+
+class T(component.T):
+    def internal_build_compile(self):
+        from rsconf import db
+        from rsconf import systemd
+
+        self.buildt.require_component("docker", "network")
+        jc, z = self.j2_ctx_init()
+        z.run_d = systemd.unit_run_d(jc, self.name)
+        z.db_d = z.run_d.join(_DB_SUBDIR)
+        z.auth_conf_f = z.run_d.join(_BASIC_AUTH_CONF_F)
+        z.auth_db_f = z.run_d.join(_BASIC_AUTH_DB_F)
+        z.mlflow_ip = db.LOCAL_IP
+        systemd.docker_unit_prepare(
+            self,
+            jc,
+            docker_exec=f"mlflow server --host {z.mlflow_ip} --port {z.service_port} --backend-store-uri 'sqlite:///{z.db_d}/backend-store.db' --no-serve-artifacts --default-artifact-root '{z.db_d}' --app-name basic-auth",
+        )
+        for s in _SECRETS:
+            z[s] = self.secret_path_value(
+                s,
+                gen_secret=lambda: db.random_string(length=16),
+                visibility="host",
+            )[0]
+
+    def internal_build_write(self):
+        from rsconf import systemd
+        from rsconf.component import docker_registry
+        from rsconf.component import nginx
+
+        jc = self.j2_ctx
+        z = jc[self.name]
+        systemd.docker_unit_enable(
+            self,
+            jc,
+            env=self.python_service_env(
+                values=PKDict(mlflow_auth_config_path=z.auth_conf_f)
+            ),
+            image=docker_registry.absolute_image(self),
+            volumes=[z.db_d],
+        )
+        self.install_access(mode="700", owner=jc.rsconf_db.run_u)
+        self.install_directory(z.db_d)
+        self.install_resource2(_BASIC_AUTH_CONF_F, z.auth_conf_f.dirname, access="400")
+        nginx.install_vhost(
+            self,
+            vhost=z.vhost,
+            backend_host=z.mlflow_ip,
+            backend_port=z.service_port,
+            j2_ctx=jc,
+        )

rsconf/package_data/mlflow/basic-auth.ini.jinja

@@ -0,0 +1,5 @@
+[mlflow]
+admin_password = {{ this.mlflow_admin_password }}
+admin_username = {{ this.mlflow_admin_username }}
+database_uri = sqlite:///{{ this.auth_db_f }}
+default_permission = EDIT

rsconf/package_data/mlflow/nginx.conf.jinja

@@ -0,0 +1,11 @@
+server {
+    listen {{ nginx.listen_ip }}:443 ssl;
+    server_name {{ nginx.vhost }};
+    root {{ nginx.default_root }};
+    ssl_certificate {{ nginx.tls_crt }};
+    ssl_certificate_key {{ nginx.tls_key }};
+
+    location / {
+        proxy_pass http://{{nginx.backend_host}}:{{ nginx.backend_port }}/;
+    }
+}

tests/pkcli/build_data/1.in/db/000.yml

@@ -502,6 +502,10 @@ host:
           v9.radia.run: jupyter.v9.radia.run
       jupyterhub_proxy:
         listen_any: True
+      mlflow:
+        docker_image: radiasoft/mlops
+        service_port: 8999
+        vhost: mlflow.v9.radia.run
       network:
         restricted_public_tcp_ports:
           https: [ 192.168.1.0/24, 127.0.0.1 ]
@@ -553,6 +557,7 @@ host:
           - github_bkp
           - jupyterhub
           - jupyterhub_proxy
+          - mlflow
           - nfs_client
           - nfs_server
           - rsaccounting

tests/pkcli/build_data/1.in/db/secret/tls/mlflow.v9.radia.run.crt

@@ -0,0 +1,20 @@
+-----BEGIN CERTIFICATE-----
+MIIDRzCCAi+gAwIBAgIJAJYNdpgLaB/AMA0GCSqGSIb3DQEBCwUAMD4xCzAJBgNV
+BAYTAlVTMREwDwYDVQQIDAhDb2xvcmFkbzEQMA4GA1UEBwwHQm91bGRlcjEKMAgG
+A1UEAwwBbTAgFw0yNDAzMjIxNzUyMjRaGA8yMDUxMDgwNzE3NTIyNFowPjELMAkG
+A1UEBhMCVVMxETAPBgNVBAgMCENvbG9yYWRvMRAwDgYDVQQHDAdCb3VsZGVyMQow
+CAYDVQQDDAFtMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAzlR/whd4
+f1Z1E7x3XgSAobyT5ckv1/XBtnQkhTL3SmhTuukzp09I1S8Zw8y6W7yOenvumMlH
+Alx8NU8PnU+NEge3pk16V0UB0gSXrCknzK0yB8sB2kPl5AWm/AWskV18TPEWaz3Z
+9D6ZB7MZUX0ihvcI8uv1GF+UDfYXVYRH/stAzdO0iLwwqsTuRwqB1/6Fy0+4TEP/
+fns4UyuyiOQK1uOFZzlvkchxlz/MftUcwMUfMTvVdkW1AoGQ8XSDDBXJGFKbI8Bo
+SBM2DUPz2Ni+3xPcDCZiBZYJCMu3b2x5itHTJOZATW2D9c4GOY7MZQaE5Vwx8DYX
+2mgRji1Kk1vwtQIDAQABo0YwRDBCBgNVHREEOzA5ggFtggFsggFmggFsggFvggF3
+ggEuggF2ggE5ggEuggFyggFhggFkggFpggFhggEuggFyggF1ggFuMA0GCSqGSIb3
+DQEBCwUAA4IBAQAs2i/gzrW2IfzSUoPx6N6uL+p8oG4ElZ6AcH0E62kZA1xUvRxj
+WxVWZ+u+VTJr8zWe6sDopRL7G//UUgT/E7EUs2bb3rATB2/T/MJrhkEFw1xCAMnV
++4Czh+SJ4RU0CXcmya3k8uj2+eJi3EPuu1k9yfLI11gzh+O7DxJVhqS8TwcGRnBe
+iVUdGYvH1ekA4UnH8iYYtfehGafY9DC/AUv79o9Lynadan4/lz3iduY45GADMGjC
+KiJptEOfyfvQ6L4vRch7U69i6RbxK2TzbgpgiXqzLHvW3KZB4yiYqOl3G/9k8MFN
+yWiMu3F7T+bvJ1zOFEYNMC2YbIjV728adVKP
+-----END CERTIFICATE-----

tests/pkcli/build_data/1.in/db/secret/tls/mlflow.v9.radia.run.key

@@ -0,0 +1,28 @@
+-----BEGIN PRIVATE KEY-----
+MIIEvAIBADANBgkqhkiG9w0BAQEFAASCBKYwggSiAgEAAoIBAQDOVH/CF3h/VnUT
+vHdeBIChvJPlyS/X9cG2dCSFMvdKaFO66TOnT0jVLxnDzLpbvI56e+6YyUcCXHw1
+Tw+dT40SB7emTXpXRQHSBJesKSfMrTIHywHaQ+XkBab8BayRXXxM8RZrPdn0PpkH
+sxlRfSKG9wjy6/UYX5QN9hdVhEf+y0DN07SIvDCqxO5HCoHX/oXLT7hMQ/9+ezhT
+K7KI5ArW44VnOW+RyHGXP8x+1RzAxR8xO9V2RbUCgZDxdIMMFckYUpsjwGhIEzYN
+Q/PY2L7fE9wMJmIFlgkIy7dvbHmK0dMk5kBNbYP1zgY5jsxlBoTlXDHwNhfaaBGO
+LUqTW/C1AgMBAAECggEAUKzXe3Oj6N5MTtg9MCTAivzqM0nUDDQKdyW4deB0ssJS
+It42FTA6ASk2gMmXAHGcoCW/KDxjKHgzFMECEPde6HLeCwd2U5Mm4BBtoaJB7pS1
+4DgvVEGCLQNMxHQPgpM0G/2UT4BSrV6ghVMxDhzImE8VT66VGd+dS0wy2XwsjmIF
+DWXUFXXLD6QPvOegPS2sxU1x07rKej7f4LBohJvPlNo1DkAiC1TgQPX/BcEvRFcy
+eZydxVs3wMt0C3G462Mn83ne4HV+qBDnbzN29f/AXbdt6T5cBEAgAzrUI15d2ahd
+Cx32ZcR++NGsAolsYHMslh1sOGLuq1EJB74KzEmjLQKBgQDl59+AxznGiuYVEJVC
+DwOK9v1fxg+Y8s7gXpaNb8awe44+83eSRSfMpEMK66Qs9zaMTyNw/GfMDRsql9z3
+VGWM+02ZKGVKHxcxXU4ui+2Y5kXAYxU15B2BD6f7Q7WJydPhkBCpXu1CRYJvweeo
+lZC3LLUtSoZ5BTDP4VSAyc9atwKBgQDlv5zQDOWkdpMZQrFCVaTVna7Qb3Xxygx9
+CyXPwuK1VM/EOYrQtPyEzVyQENisMrWoTd5+xUiGWELqgw3G1CZdttdULaSihnUi
+kS5dIzZPr70O9MKtKyuA9N2HqPorL4ML82lzNZALNW3GraA3rqgdNfW/d3g5VumC
+TeYZjlzT8wKBgBsU/VcGIOAqbH/+vBZT/m7YOXCWvC+gKo5zlfiDOjx/0XnI5ETu
+sImCsAMRTv2dAWf1yo78rJ10zcZurTDVWEELOVDZWVUp5GmEPDlvckWYmo7XHSCE
+BpW1amxGxZO9mp/vgIbzD6/G0F6p0MgjFjD9qznylXScD+vs9y9UloBxAoGALMcE
+pBPvafdmQgUakSgqASxDVwJyRVawymoyy2RbVSYbfE4OvRCZKrTvtnBiGmWjvBK5
+K5H9yZlIKXazZ64PcDJjk1d1/+sex0bud16Whj9lJJYVnzSLvQ18Y6VOZL5U1y5w
+vfCRi221YISUdmXHBDJxHAkH2H0U872E/DQf6XsCgYAw5/xBoXcBafwVYBzCMziF
+xllrICXrpxGJEU1+MMnTJCtWiSYq6sOvZQTacNa9Pyr/SmNfSmuzJAO3yIXxo8WP
+4ja2NxSgMGuomN8zMzRLiZhm5s1+BDIfh2kg9DA8Adv/0wctiCM8gtWfKm4l9Yix
+NA7bqCalMR+wcbGgvoOnAQ==
+-----END PRIVATE KEY-----

tests/pkcli/build_data/1.in/db/secret/v9.radia.run/mlflow_admin_password

@@ -0,0 +1 @@
+T3W4dhNjaSlwqjqZ

tests/pkcli/build_data/1.in/db/secret/v9.radia.run/mlflow_admin_username

@@ -0,0 +1 @@
+oyVQh9UIGyYK1rmL

tests/pkcli/build_data/1.out/srv/host/v9.radia.run/mlflow.sh

@@ -0,0 +1,23 @@
+#!/bin/bash
+mlflow_rsconf_component() {
+rsconf_service_prepare 'mlflow' '/etc/systemd/system/mlflow.service' '/etc/systemd/system/mlflow.service.d' '/srv/mlflow'
+rsconf_install_access '700' 'vagrant' 'vagrant'
+rsconf_install_directory '/srv/mlflow'
+rsconf_install_access '500' 'vagrant' 'vagrant'
+rsconf_install_file '/srv/mlflow/cmd' '883ef20d348691bc33dd992112ed9985'
+rsconf_install_file '/srv/mlflow/env' '01f7ee885371f9fd9a8042017614adf9'
+rsconf_install_file '/srv/mlflow/remove' 'e8253886ccf08377b23c8cc32c4b64fa'
+rsconf_install_file '/srv/mlflow/start' '1ad20b9401cf3eaf5191245a26475dc0'
+rsconf_install_file '/srv/mlflow/stop' 'fe05a526f775d8c47ebca4412dc6b432'
+rsconf_install_access '444' 'root' 'root'
+rsconf_install_file '/etc/systemd/system/mlflow.service' 'b370658c6db4685dd6c38153694b8786'
+rsconf_service_docker_pull 'v3.radia.run:5000/radiasoft/mlops:dev' 'mlflow' 'mlflow' ''
+rsconf_install_access '700' 'vagrant' 'vagrant'
+rsconf_install_directory '/srv/mlflow/db'
+rsconf_install_access '400' 'vagrant' 'vagrant'
+rsconf_install_file '/srv/mlflow/basic-auth.ini' '0e4ac81b2bd4cc356905d3b25f0989b3'
+rsconf_install_access '400' 'root' 'root'
+rsconf_install_file '/etc/nginx/conf.d/mlflow.v9.radia.run.key' '97cef7711a10368c80d469627acf4814'
+rsconf_install_file '/etc/nginx/conf.d/mlflow.v9.radia.run.crt' 'c832c6b9d565a6ceb76b15bde9a24e52'
+rsconf_install_file '/etc/nginx/conf.d/mlflow.v9.radia.run.conf' '69dd677938428b9501c0b3c4c447bba5'
+}

tests/pkcli/build_data/1.out/srv/host/v9.radia.run/srv/mlflow/basic-auth.ini

@@ -0,0 +1,5 @@
+[mlflow]
+admin_password = T3W4dhNjaSlwqjqZ
+admin_username = oyVQh9UIGyYK1rmL
+database_uri = sqlite:////srv/mlflow/basic-auth.db
+default_permission = EDIT

tests/pkcli/build_data/1.out/srv/host/v9.radia.run/srv/mlflow/env

@@ -0,0 +1,4 @@
+#!/bin/bash
+export 'MLFLOW_AUTH_CONFIG_PATH=/srv/mlflow/basic-auth.ini'
+export 'PYTHONUNBUFFERED=1'
+export 'TZ=:/etc/localtime'