Fix #370: standardize rsiviz port configuration and enable devbox for rsiviz

rsconf/component/__init__.py

@@ -251,25 +251,27 @@ def j2_ctx_pksetdefault(self, defaults):
         defaults is nested dicts with dotted keys that
         will be turned into nested `PKDict` if not defined.
 
-        Uses PKDictpksetdefault to set values so can use
+        Uses PKDict.pksetdefault to set values so can use
         callables as initializers.
 
         Args:
             defaults (dict): nested values
         """
 
-        def f(prefix, defaults):
-            for k, v in defaults.items():
-                k = prefix + k.split(".")
-                if isinstance(v, dict):
-                    f(k, v)
-                    continue
-                n = self.j2_ctx
-                for y in k[:-1]:
-                    n = n.setdefault(y, PKDict())
-                n.pksetdefault(k[-1], v)
+        return self._j2_ctx_set(defaults, "pksetdefault")
 
-        f([], defaults)
+    def j2_ctx_pkupdate(self, updates):
+        """Set updates on self.j2_ctx
+
+        defaults is nested dicts with dotted keys that
+        will be turned into nested `PKDict` if not defined.
+
+        Uses PKDict.pkudpate to set values.
+
+        Args:
+            updates (dict): nested values to set
+        """
+        return self._j2_ctx_set(updates, "__setitem__")
 
     def python_service_env(self, values, exclude_re=None):
         e = pkconfig.to_environ(
@@ -395,6 +397,20 @@ def _bash_append_and_dst(
         self._bash_append(host_path, md5=md5)
         return dst
 
+    def _j2_ctx_set(self, values, method):
+        def f(prefix, values, method):
+            for k, v in values.items():
+                k = prefix + k.split(".")
+                if isinstance(v, dict):
+                    f(k, v, method)
+                    continue
+                n = self.j2_ctx
+                for y in k[:-1]:
+                    n = n.setdefault(y, PKDict())
+                getattr(n, method)(k[-1], v)
+
+        f([], values, method)
+
     def _render_file(self, path, j2_ctx):
         from pykern import pkjinja
 

rsconf/component/devbox.py

@@ -7,6 +7,9 @@
 from pykern.pkcollections import PKDict
 from pykern.pkdebug import pkdp
 from rsconf import component
+import ipaddress
+
+_RSIVIZ_DICE_NETWORK_DISCOVERY_PORT = 5555
 
 
 class T(component.T):
@@ -94,6 +97,16 @@ def internal_build_write(self):
         for k, v in self.secrets.items():
             self.install_abspath(v, z.host[k])
 
+    def _env(self, name, value, path):
+        v = str(value)
+        # TODO probably could recursively quote
+        if "'" in v:
+            raise ValueError(f"single quote in value={v} of bash variable name={name}")
+        self.rsconf_append(
+            path,
+            f"export {name}='{v}'",
+        )
+
     def _gen_paths(self, z, db_d, ssh_d):
         res = PKDict(ssh_d=db_d.join(ssh_d))
         res.pkupdate(sshd_config=res.ssh_d.join("sshd_config"))
@@ -108,34 +121,47 @@ def _gen_secrets(self, jc):
         self.secrets = PKDict({k: s[k] for k in ("host_key_f", "identity_pub_f")})
 
     def _jupyter_bashrc(self, z, path):
-        def _env(name, value):
-            v = str(value)
-            # TODO probably could recursively quote
-            if "'" in v:
-                raise ValueError(
-                    f"single quote in value={v} of bash variable name={name}"
-                )
-            self.rsconf_append(
-                path,
-                f"export {name}='{v}'",
-            )
-
         self.install_access(mode="600")
         self.install_ensure_file_exists(path)
         for n in ("package_path", "sim_types"):
             if n in z:
-                _env(f"SIREPO_FEATURE_CONFIG_{n.upper()}", ":".join(z[n]))
+                self._env(f"SIREPO_FEATURE_CONFIG_{n.upper()}", ":".join(z[n]), path)
         z.service_port = z.ssh_port + z.ssh_service_port_difference
         z.job_supervisor_port = z.service_port + z.ssh_service_port_difference
         for n in ("service_port", "job_supervisor_port"):
-            _env(f"SIREPO_PKCLI_{n.upper()}", z[n])
+            self._env(f"SIREPO_PKCLI_{n.upper()}", z[n], path)
         for n in ("DRIVER_LOCAL", "API"):
-            _env(
+            self._env(
                 f"SIREPO_JOB_{n}_SUPERVISOR_URI",
                 f"http://127.0.0.1:{z.job_supervisor_port}",
+                path,
             )
+        self._rsiviz(z, path)
 
     def _network(self, jc, z):
         n = self.buildt.get_component("network")
         z.ip = n.unchecked_public_ip() or n.ip_and_net_for_host(jc.rsconf_db.host)[0]
         n.add_public_tcp_ports([str(z.ssh_port)])
+
+    def _rsiviz(self, z, path):
+        u = z.users[self.user_name]
+        if not isinstance(u, PKDict) or not "rsiviz" in u:
+            return
+        self._env(
+            "PYKERN_PKASYNCIO_SERVER_PORT",
+            u.rsiviz.port_base,
+            path,
+        )
+        self._env(
+            "RSIVIZ_PKCLI_SERVICE_DICE_NETWORK_CLUSTER_INTERFACE_ADDRESS",
+            ipaddress.ip_address(u.rsiviz.ip_base) + 1,
+            path,
+        )
+        self._env(
+            "RSIVIZ_PKCLI_SERVICE_DICE_NETWORK_DISCOVERY_ADDRESS",
+            f"{u.rsiviz.ip_base}:{_RSIVIZ_DICE_NETWORK_DISCOVERY_PORT}",
+            path,
+        )
+        self._env(
+            "RSIVIZ_PKCLI_SERVICE_INDEX_IFRAME_PORT", u.rsiviz.port_base + 1, path
+        )

rsconf/component/rsiviz.py

@@ -11,36 +11,47 @@
 from rsconf import systemd
 
 _DB_SUBDIR = "db"
+_PORTS = PKDict(
+    index_port=8880,
+    flask_port=8882,
+)
 
 
 class T(component.T):
     def internal_build_compile(self):
         self.buildt.require_component("docker", "nginx")
         jc, z = self.j2_ctx_init()
         z._run_u = jc.rsconf_db.run_u
-        self.j2_ctx_pksetdefault(
-            PKDict(pykern=PKDict(pkconfig=PKDict(channel=jc.rsconf_db.channel)))
+        self.j2_ctx_pkupdate(
+            PKDict(
+                nginx=PKDict(
+                    docker_index_port=z.index_iframe_port,
+                    docker_flask_port=z.server_port,
+                    **_PORTS,
+                ),
+                pykern=PKDict(
+                    pkconfig=PKDict(channel=jc.rsconf_db.channel),
+                    pkasyncio=PKDict(server_port=z.server_port),
+                ),
+                rsiviz=PKDict(
+                    pkcli=PKDict(service=PKDict(index_iframe_port=z.index_iframe_port))
+                ),
+            ),
         )
         self.__run_d = systemd.docker_unit_prepare(
             self,
             jc,
+            # TODO(e-carlin): This is the default command (set by build_docker_cmd)
+            # is there a way to just use it and not specify cmd?
             docker_exec=f"bash {db.user_home_path(z._run_u)}/.radia-run/start",
         )
 
     def internal_build_write(self):
-        from rsconf.component import db_bkp
         from rsconf.component import nginx
-        from rsconf.component import docker
 
         jc = self.j2_ctx
         z = jc[self.name]
         d = self.__run_d.join(_DB_SUBDIR)
-        jc.nginx.pkupdate(
-            index_port=8880,
-            flask_port=8882,
-            docker_index_port=8000,
-            docker_flask_port=8002,
-        )
         nginx.install_vhost(
             self,
             vhost=self.hdb.rsconf_db.host,
@@ -54,10 +65,6 @@ def internal_build_write(self):
                 exclude_re=r"^rsiviz(?:__|_docker_image|_url_secret)",
             ),
             image=z.docker_image,
-            ports=(
-                (jc.nginx.docker_index_port, 8080),
-                (jc.nginx.docker_flask_port, 8082),
-            ),
         )
         self.install_access(mode="700", owner=z._run_u)
         self.install_directory(d)

tests/pkcli/build_data/1.in/db/000.yml

@@ -460,6 +460,15 @@ host:
           custom_image:
             docker_image: radiasoft/custom-image
             ssh_port: 3102
+          rsivizcoder:
+            rsiviz:
+              ip_base: 127.2.0.0
+              port_base: 20000
+            ssh_port: 3103
+      systemd:
+        extra_run_flags:
+          rsivizdevbox:
+            gpus: "all"
       jupyterhub:
         jupyter_docker_image: radiasoft/custom-jupyter:latest
         jupyter_docker_image_is_local: true
@@ -530,6 +539,8 @@ host:
           - sirepo_test_http
       rsiviz:
         docker_image: radiasoft/rsiviz
+        server_port: 8002
+        index_iframe_port: 8000
       sirepo:
         activait_redirect: false
         auth:

tests/pkcli/build_data/1.out/srv/host/v9.radia.run/000.sh

@@ -20,6 +20,7 @@ rsconf_require docker
 rsconf_require devbox_devtech3
 rsconf_require devbox_fullstackdude
 rsconf_require devbox_custom_image
+rsconf_require devbox_rsivizcoder
 rsconf_require devbox
 rsconf_require dovecot
 rsconf_require github_bkp

tests/pkcli/build_data/1.out/srv/host/v9.radia.run/etc/sysconfig/iptables

@@ -9,7 +9,7 @@
     -A INPUT -i lo -j ACCEPT
     -A INPUT -i em1 -j ACCEPT
 -A INPUT -i em2 -m state --state RELATED,ESTABLISHED -j ACCEPT
-    -A INPUT -i em2 -p tcp -m state --state NEW -m tcp --match multiport --dports 3100,3101,3102,9999,http,pop3s,smtp,submission -j ACCEPT
+    -A INPUT -i em2 -p tcp -m state --state NEW -m tcp --match multiport --dports 3100,3101,3102,3103,9999,http,pop3s,smtp,submission -j ACCEPT
         -A INPUT -i em2 -s 192.168.1.0/24 -p tcp -m state --state NEW -m tcp --dport https  -j ACCEPT
         -A INPUT -i em2 -s 127.0.0.1 -p tcp -m state --state NEW -m tcp --dport https  -j ACCEPT
 -A INPUT -i em2 -m state --state INVALID -j REJECT --reject-with icmp-port-unreachable

tests/pkcli/build_data/1.out/srv/host/v9.radia.run/network.sh

@@ -6,7 +6,7 @@ rsconf_install_access '444' 'root' 'root'
 rsconf_install_file '/etc/resolv.conf' 'c333a7a816c03062d4e61effd6d2a8ce'
 rsconf_install_file '/etc/sysconfig/network-scripts/ifcfg-em1' 'e225a3f7b4b071e5c204b1182c17ab94'
 rsconf_install_file '/etc/sysconfig/network-scripts/ifcfg-em2' '4f8c01335a3bf1a8a89ba5d09fdf28df'
-rsconf_install_file '/etc/sysconfig/iptables' '422d9959bcaf8e047c266a168ba2638e'
+rsconf_install_file '/etc/sysconfig/iptables' '797fdace2e6dd01371697aa815251bc5'
 network_main
 }
 #!/bin/bash