From 3ace76e319e296057c79b018ad52acfbc8e1aa62 Mon Sep 17 00:00:00 2001 From: e-carlin Date: Mon, 21 Aug 2023 23:09:27 +0000 Subject: [PATCH 1/7] Fix #370: standardize rsiviz port configuration and enable devbox for rsiviz --- rsconf/component/devbox.py | 54 +++++++++++++------ rsconf/component/rsiviz.py | 20 +++---- tests/pkcli/build_data/1.in/db/000.yml | 9 ++++ .../1.out/srv/host/v9.radia.run/000.sh | 1 + .../host/v9.radia.run/etc/sysconfig/iptables | 2 +- .../1.out/srv/host/v9.radia.run/network.sh | 2 +- 6 files changed, 62 insertions(+), 26 deletions(-) diff --git a/rsconf/component/devbox.py b/rsconf/component/devbox.py index e0b7ff06..edcec5f8 100644 --- a/rsconf/component/devbox.py +++ b/rsconf/component/devbox.py @@ -7,6 +7,7 @@ from pykern.pkcollections import PKDict from pykern.pkdebug import pkdp from rsconf import component +import ipaddress class T(component.T): @@ -94,6 +95,16 @@ def internal_build_write(self): for k, v in self.secrets.items(): self.install_abspath(v, z.host[k]) + def _env(self, name, value, path): + v = str(value) + # TODO probably could recursively quote + if "'" in v: + raise ValueError(f"single quote in value={v} of bash variable name={name}") + self.rsconf_append( + path, + f"export {name}='{v}'", + ) + def _gen_paths(self, z, db_d, ssh_d): res = PKDict(ssh_d=db_d.join(ssh_d)) res.pkupdate(sshd_config=res.ssh_d.join("sshd_config")) @@ -108,34 +119,47 @@ def _gen_secrets(self, jc): self.secrets = PKDict({k: s[k] for k in ("host_key_f", "identity_pub_f")}) def _jupyter_bashrc(self, z, path): - def _env(name, value): - v = str(value) - # TODO probably could recursively quote - if "'" in v: - raise ValueError( - f"single quote in value={v} of bash variable name={name}" - ) - self.rsconf_append( - path, - f"export {name}='{v}'", - ) - self.install_access(mode="600") self.install_ensure_file_exists(path) for n in ("package_path", "sim_types"): if n in z: - _env(f"SIREPO_FEATURE_CONFIG_{n.upper()}", ":".join(z[n])) + self._env(f"SIREPO_FEATURE_CONFIG_{n.upper()}", ":".join(z[n]), path) z.service_port = z.ssh_port + z.ssh_service_port_difference z.job_supervisor_port = z.service_port + z.ssh_service_port_difference for n in ("service_port", "job_supervisor_port"): - _env(f"SIREPO_PKCLI_{n.upper()}", z[n]) + self._env(f"SIREPO_PKCLI_{n.upper()}", z[n], path) for n in ("DRIVER_LOCAL", "API"): - _env( + self._env( f"SIREPO_JOB_{n}_SUPERVISOR_URI", f"http://127.0.0.1:{z.job_supervisor_port}", + path, ) + self._rsiviz(z, path) def _network(self, jc, z): n = self.buildt.get_component("network") z.ip = n.unchecked_public_ip() or n.ip_and_net_for_host(jc.rsconf_db.host)[0] n.add_public_tcp_ports([str(z.ssh_port)]) + + def _rsiviz(self, z, path): + u = z.users[self.user_name] + if not isinstance(u, PKDict) or not "rsiviz" in u: + return + self._env( + "PYKERN_PKASYNCIO_SERVER_PORT", + u.rsiviz.port_base, + path, + ) + self._env( + "RSIVIZ_PKCLI_SERVICE_DICE_NETWORK_CLUSTER_INTERFACE_ADDRESS", + ipaddress.ip_address(u.rsiviz.ip_base) + 1, + path, + ) + self._env( + "RSIVIZ_PKCLI_SERVICE_DICE_NETWORK_NETWORK_DISCOVERY_ADDRESS", + f"{u.rsiviz.ip_base}:5555", + path, + ) + self._env( + "RSIVIZ_PKCLI_SERVICE_INDEX_IFRAME_PORT", u.rsiviz.port_base + 1, path + ) diff --git a/rsconf/component/rsiviz.py b/rsconf/component/rsiviz.py index 392ad826..ed2eb08a 100644 --- a/rsconf/component/rsiviz.py +++ b/rsconf/component/rsiviz.py @@ -19,18 +19,24 @@ def internal_build_compile(self): jc, z = self.j2_ctx_init() z._run_u = jc.rsconf_db.run_u self.j2_ctx_pksetdefault( - PKDict(pykern=PKDict(pkconfig=PKDict(channel=jc.rsconf_db.channel))) + PKDict( + pykern=PKDict( + pkconfig=PKDict(channel=jc.rsconf_db.channel), + pkasyncio=PKDict(server_port=8002), + ), + rsiviz=PKDict(pkcli=PKDict(service=PKDict(index_iframe_port=8000))), + ) ) self.__run_d = systemd.docker_unit_prepare( self, jc, + # TODO(e-carlin): This is the default command (set by build_docker_cmd) + # is there a way to just use it and not specify cmd? docker_exec=f"bash {db.user_home_path(z._run_u)}/.radia-run/start", ) def internal_build_write(self): - from rsconf.component import db_bkp from rsconf.component import nginx - from rsconf.component import docker jc = self.j2_ctx z = jc[self.name] @@ -38,8 +44,8 @@ def internal_build_write(self): jc.nginx.pkupdate( index_port=8880, flask_port=8882, - docker_index_port=8000, - docker_flask_port=8002, + docker_index_port=z.pkcli.service.index_iframe_port, + docker_flask_port=jc.pykern.pkasyncio.server_port, ) nginx.install_vhost( self, @@ -54,10 +60,6 @@ def internal_build_write(self): exclude_re=r"^rsiviz(?:__|_docker_image|_url_secret)", ), image=z.docker_image, - ports=( - (jc.nginx.docker_index_port, 8080), - (jc.nginx.docker_flask_port, 8082), - ), ) self.install_access(mode="700", owner=z._run_u) self.install_directory(d) diff --git a/tests/pkcli/build_data/1.in/db/000.yml b/tests/pkcli/build_data/1.in/db/000.yml index 4385e220..71db5bea 100644 --- a/tests/pkcli/build_data/1.in/db/000.yml +++ b/tests/pkcli/build_data/1.in/db/000.yml @@ -460,6 +460,15 @@ host: custom_image: docker_image: radiasoft/custom-image ssh_port: 3102 + rsivizdevbox: + rsiviz: + ip_base: 127.2.0.0 + port_base: 20000 + ssh_port: 3103 + systemd: + extra_run_flags: + rsivizdevbox: + gpus: "all" jupyterhub: jupyter_docker_image: radiasoft/custom-jupyter:latest jupyter_docker_image_is_local: true diff --git a/tests/pkcli/build_data/1.out/srv/host/v9.radia.run/000.sh b/tests/pkcli/build_data/1.out/srv/host/v9.radia.run/000.sh index fc6f3b9f..899072f3 100644 --- a/tests/pkcli/build_data/1.out/srv/host/v9.radia.run/000.sh +++ b/tests/pkcli/build_data/1.out/srv/host/v9.radia.run/000.sh @@ -20,6 +20,7 @@ rsconf_require docker rsconf_require devbox_devtech3 rsconf_require devbox_fullstackdude rsconf_require devbox_custom_image +rsconf_require devbox_rsivizdevbox rsconf_require devbox rsconf_require dovecot rsconf_require github_bkp diff --git a/tests/pkcli/build_data/1.out/srv/host/v9.radia.run/etc/sysconfig/iptables b/tests/pkcli/build_data/1.out/srv/host/v9.radia.run/etc/sysconfig/iptables index 82f19b05..1495e1c1 100644 --- a/tests/pkcli/build_data/1.out/srv/host/v9.radia.run/etc/sysconfig/iptables +++ b/tests/pkcli/build_data/1.out/srv/host/v9.radia.run/etc/sysconfig/iptables @@ -9,7 +9,7 @@ -A INPUT -i lo -j ACCEPT -A INPUT -i em1 -j ACCEPT -A INPUT -i em2 -m state --state RELATED,ESTABLISHED -j ACCEPT - -A INPUT -i em2 -p tcp -m state --state NEW -m tcp --match multiport --dports 3100,3101,3102,9999,http,pop3s,smtp,submission -j ACCEPT + -A INPUT -i em2 -p tcp -m state --state NEW -m tcp --match multiport --dports 3100,3101,3102,3103,9999,http,pop3s,smtp,submission -j ACCEPT -A INPUT -i em2 -s 192.168.1.0/24 -p tcp -m state --state NEW -m tcp --dport https -j ACCEPT -A INPUT -i em2 -s 127.0.0.1 -p tcp -m state --state NEW -m tcp --dport https -j ACCEPT -A INPUT -i em2 -m state --state INVALID -j REJECT --reject-with icmp-port-unreachable diff --git a/tests/pkcli/build_data/1.out/srv/host/v9.radia.run/network.sh b/tests/pkcli/build_data/1.out/srv/host/v9.radia.run/network.sh index 4c4de443..1c7a9125 100644 --- a/tests/pkcli/build_data/1.out/srv/host/v9.radia.run/network.sh +++ b/tests/pkcli/build_data/1.out/srv/host/v9.radia.run/network.sh @@ -6,7 +6,7 @@ rsconf_install_access '444' 'root' 'root' rsconf_install_file '/etc/resolv.conf' 'c333a7a816c03062d4e61effd6d2a8ce' rsconf_install_file '/etc/sysconfig/network-scripts/ifcfg-em1' 'e225a3f7b4b071e5c204b1182c17ab94' rsconf_install_file '/etc/sysconfig/network-scripts/ifcfg-em2' '4f8c01335a3bf1a8a89ba5d09fdf28df' -rsconf_install_file '/etc/sysconfig/iptables' '422d9959bcaf8e047c266a168ba2638e' +rsconf_install_file '/etc/sysconfig/iptables' '797fdace2e6dd01371697aa815251bc5' network_main } #!/bin/bash From ff62898b22c0ff2f17944f6539b2a4e4c67805b9 Mon Sep 17 00:00:00 2001 From: e-carlin Date: Thu, 24 Aug 2023 20:45:17 +0000 Subject: [PATCH 2/7] pr comments --- rsconf/component/devbox.py | 4 +++- rsconf/component/rsiviz.py | 17 +++++++++++------ tests/pkcli/build_data/1.in/db/000.yml | 4 +++- 3 files changed, 17 insertions(+), 8 deletions(-) diff --git a/rsconf/component/devbox.py b/rsconf/component/devbox.py index edcec5f8..e59fb39e 100644 --- a/rsconf/component/devbox.py +++ b/rsconf/component/devbox.py @@ -9,6 +9,8 @@ from rsconf import component import ipaddress +_RSIVIZ_DICE_NETWORK_DISCOVERY_PORT = 5555 + class T(component.T): def internal_build_compile(self): @@ -157,7 +159,7 @@ def _rsiviz(self, z, path): ) self._env( "RSIVIZ_PKCLI_SERVICE_DICE_NETWORK_NETWORK_DISCOVERY_ADDRESS", - f"{u.rsiviz.ip_base}:5555", + f"{u.rsiviz.ip_base}:{_RSIVIZ_DICE_NETWORK_DISCOVERY_PORT}", path, ) self._env( diff --git a/rsconf/component/rsiviz.py b/rsconf/component/rsiviz.py index ed2eb08a..8ad1d4af 100644 --- a/rsconf/component/rsiviz.py +++ b/rsconf/component/rsiviz.py @@ -11,6 +11,10 @@ from rsconf import systemd _DB_SUBDIR = "db" +_PORTS = PKDict( + index_port=8880, + flask_port=8882, +) class T(component.T): @@ -22,9 +26,11 @@ def internal_build_compile(self): PKDict( pykern=PKDict( pkconfig=PKDict(channel=jc.rsconf_db.channel), - pkasyncio=PKDict(server_port=8002), + pkasyncio=PKDict(server_port=z.server_port), + ), + rsiviz=PKDict( + pkcli=PKDict(service=PKDict(index_iframe_port=z.index_iframe_port)) ), - rsiviz=PKDict(pkcli=PKDict(service=PKDict(index_iframe_port=8000))), ) ) self.__run_d = systemd.docker_unit_prepare( @@ -42,10 +48,9 @@ def internal_build_write(self): z = jc[self.name] d = self.__run_d.join(_DB_SUBDIR) jc.nginx.pkupdate( - index_port=8880, - flask_port=8882, - docker_index_port=z.pkcli.service.index_iframe_port, - docker_flask_port=jc.pykern.pkasyncio.server_port, + docker_index_port=z.index_iframe_port, + docker_flask_port=z.server_port, + **_PORTS, ) nginx.install_vhost( self, diff --git a/tests/pkcli/build_data/1.in/db/000.yml b/tests/pkcli/build_data/1.in/db/000.yml index 71db5bea..7c47e123 100644 --- a/tests/pkcli/build_data/1.in/db/000.yml +++ b/tests/pkcli/build_data/1.in/db/000.yml @@ -460,7 +460,7 @@ host: custom_image: docker_image: radiasoft/custom-image ssh_port: 3102 - rsivizdevbox: + rsivizcoder: rsiviz: ip_base: 127.2.0.0 port_base: 20000 @@ -539,6 +539,8 @@ host: - sirepo_test_http rsiviz: docker_image: radiasoft/rsiviz + server_port: 8002 + index_iframe_port: 8000 sirepo: activait_redirect: false auth: From fc1f236a02f3087395313865adc4ddc7e7858dce Mon Sep 17 00:00:00 2001 From: e-carlin Date: Thu, 24 Aug 2023 20:46:27 +0000 Subject: [PATCH 3/7] fix tests --- tests/pkcli/build_data/1.out/srv/host/v9.radia.run/000.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/pkcli/build_data/1.out/srv/host/v9.radia.run/000.sh b/tests/pkcli/build_data/1.out/srv/host/v9.radia.run/000.sh index 899072f3..13e4b6d3 100644 --- a/tests/pkcli/build_data/1.out/srv/host/v9.radia.run/000.sh +++ b/tests/pkcli/build_data/1.out/srv/host/v9.radia.run/000.sh @@ -20,7 +20,7 @@ rsconf_require docker rsconf_require devbox_devtech3 rsconf_require devbox_fullstackdude rsconf_require devbox_custom_image -rsconf_require devbox_rsivizdevbox +rsconf_require devbox_rsivizcoder rsconf_require devbox rsconf_require dovecot rsconf_require github_bkp From f3f0175e9a61818421528ec196b91fe0568e61c0 Mon Sep 17 00:00:00 2001 From: e-carlin Date: Thu, 24 Aug 2023 22:44:46 +0000 Subject: [PATCH 4/7] typo --- rsconf/component/devbox.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rsconf/component/devbox.py b/rsconf/component/devbox.py index e59fb39e..d9e7ee1b 100644 --- a/rsconf/component/devbox.py +++ b/rsconf/component/devbox.py @@ -158,7 +158,7 @@ def _rsiviz(self, z, path): path, ) self._env( - "RSIVIZ_PKCLI_SERVICE_DICE_NETWORK_NETWORK_DISCOVERY_ADDRESS", + "RSIVIZ_PKCLI_SERVICE_DICE_NETWORK_DISCOVERY_ADDRESS", f"{u.rsiviz.ip_base}:{_RSIVIZ_DICE_NETWORK_DISCOVERY_PORT}", path, ) From 3cba2329356ed2370bfb9917fcfe19b809c53a4f Mon Sep 17 00:00:00 2001 From: e-carlin Date: Fri, 25 Aug 2023 19:12:13 +0000 Subject: [PATCH 5/7] pr comments --- rsconf/component/rsiviz.py | 31 +++++++++++++++---------------- 1 file changed, 15 insertions(+), 16 deletions(-) diff --git a/rsconf/component/rsiviz.py b/rsconf/component/rsiviz.py index 8ad1d4af..d22d951f 100644 --- a/rsconf/component/rsiviz.py +++ b/rsconf/component/rsiviz.py @@ -22,17 +22,21 @@ def internal_build_compile(self): self.buildt.require_component("docker", "nginx") jc, z = self.j2_ctx_init() z._run_u = jc.rsconf_db.run_u - self.j2_ctx_pksetdefault( - PKDict( - pykern=PKDict( - pkconfig=PKDict(channel=jc.rsconf_db.channel), - pkasyncio=PKDict(server_port=z.server_port), - ), - rsiviz=PKDict( - pkcli=PKDict(service=PKDict(index_iframe_port=z.index_iframe_port)) - ), - ) - ) + for k, v in PKDict( + nginx=PKDict( + docker_index_port=z.index_iframe_port, + docker_flask_port=z.server_port, + **_PORTS, + ), + pykern=PKDict( + pkconfig=PKDict(channel=jc.rsconf_db.channel), + pkasyncio=PKDict(server_port=z.server_port), + ), + rsiviz=PKDict( + pkcli=PKDict(service=PKDict(index_iframe_port=z.index_iframe_port)) + ), + ).items(): + jc.setdefault(k, PKDict()).pkupdate(v) self.__run_d = systemd.docker_unit_prepare( self, jc, @@ -47,11 +51,6 @@ def internal_build_write(self): jc = self.j2_ctx z = jc[self.name] d = self.__run_d.join(_DB_SUBDIR) - jc.nginx.pkupdate( - docker_index_port=z.index_iframe_port, - docker_flask_port=z.server_port, - **_PORTS, - ) nginx.install_vhost( self, vhost=self.hdb.rsconf_db.host, From d66e32485931198ef4af7c95da9a0abb6e01a41c Mon Sep 17 00:00:00 2001 From: robnagler Date: Fri, 25 Aug 2023 13:28:31 -0600 Subject: [PATCH 6/7] add j2_ctx_pkupdate --- rsconf/component/__init__.py | 40 +++++++++++++++++++++++++----------- rsconf/component/rsiviz.py | 29 +++++++++++++------------- 2 files changed, 43 insertions(+), 26 deletions(-) diff --git a/rsconf/component/__init__.py b/rsconf/component/__init__.py index 8aa537fe..7acc943e 100644 --- a/rsconf/component/__init__.py +++ b/rsconf/component/__init__.py @@ -251,25 +251,27 @@ def j2_ctx_pksetdefault(self, defaults): defaults is nested dicts with dotted keys that will be turned into nested `PKDict` if not defined. - Uses PKDictpksetdefault to set values so can use + Uses PKDict.pksetdefault to set values so can use callables as initializers. Args: defaults (dict): nested values """ - def f(prefix, defaults): - for k, v in defaults.items(): - k = prefix + k.split(".") - if isinstance(v, dict): - f(k, v) - continue - n = self.j2_ctx - for y in k[:-1]: - n = n.setdefault(y, PKDict()) - n.pksetdefault(k[-1], v) + return self._j2_ctx_set(updates, "pksetdefault") + + def j2_ctx_pkupdate(self, updates): + """Set updates on self.j2_ctx + + defaults is nested dicts with dotted keys that + will be turned into nested `PKDict` if not defined. - f([], defaults) + Uses PKDict.pkudpate to set values. + + Args: + updates (dict): nested values to set + """ + return self._j2_ctx_set(updates, "pkupdate") def python_service_env(self, values, exclude_re=None): e = pkconfig.to_environ( @@ -395,6 +397,20 @@ def _bash_append_and_dst( self._bash_append(host_path, md5=md5) return dst + def _j2_ctx_set(self, values, method): + def f(prefix, values, method): + for k, v in values.items(): + k = prefix + k.split(".") + if isinstance(v, dict): + f(k, v) + continue + n = self.j2_ctx + for y in k[:-1]: + n = n.setdefault(y, PKDict()) + getattr(n, method)(k[-1], v) + + f([], values) + def _render_file(self, path, j2_ctx): from pykern import pkjinja diff --git a/rsconf/component/rsiviz.py b/rsconf/component/rsiviz.py index d22d951f..b9728cf1 100644 --- a/rsconf/component/rsiviz.py +++ b/rsconf/component/rsiviz.py @@ -22,21 +22,22 @@ def internal_build_compile(self): self.buildt.require_component("docker", "nginx") jc, z = self.j2_ctx_init() z._run_u = jc.rsconf_db.run_u - for k, v in PKDict( - nginx=PKDict( - docker_index_port=z.index_iframe_port, - docker_flask_port=z.server_port, - **_PORTS, + self.j2_ctx_pkupdate( + PKDict( + nginx=PKDict( + docker_index_port=z.index_iframe_port, + docker_flask_port=z.server_port, + **_PORTS, + ), + pykern=PKDict( + pkconfig=PKDict(channel=jc.rsconf_db.channel), + pkasyncio=PKDict(server_port=z.server_port), + ), + rsiviz=PKDict( + pkcli=PKDict(service=PKDict(index_iframe_port=z.index_iframe_port)) + ), ), - pykern=PKDict( - pkconfig=PKDict(channel=jc.rsconf_db.channel), - pkasyncio=PKDict(server_port=z.server_port), - ), - rsiviz=PKDict( - pkcli=PKDict(service=PKDict(index_iframe_port=z.index_iframe_port)) - ), - ).items(): - jc.setdefault(k, PKDict()).pkupdate(v) + ) self.__run_d = systemd.docker_unit_prepare( self, jc, From 01b076c7412b0fbba16965c5b947ea61983f7ae9 Mon Sep 17 00:00:00 2001 From: robnagler Date: Fri, 25 Aug 2023 19:35:08 +0000 Subject: [PATCH 7/7] fpc --- rsconf/component/__init__.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/rsconf/component/__init__.py b/rsconf/component/__init__.py index 7acc943e..88baf95f 100644 --- a/rsconf/component/__init__.py +++ b/rsconf/component/__init__.py @@ -258,7 +258,7 @@ def j2_ctx_pksetdefault(self, defaults): defaults (dict): nested values """ - return self._j2_ctx_set(updates, "pksetdefault") + return self._j2_ctx_set(defaults, "pksetdefault") def j2_ctx_pkupdate(self, updates): """Set updates on self.j2_ctx @@ -271,7 +271,7 @@ def j2_ctx_pkupdate(self, updates): Args: updates (dict): nested values to set """ - return self._j2_ctx_set(updates, "pkupdate") + return self._j2_ctx_set(updates, "__setitem__") def python_service_env(self, values, exclude_re=None): e = pkconfig.to_environ( @@ -402,14 +402,14 @@ def f(prefix, values, method): for k, v in values.items(): k = prefix + k.split(".") if isinstance(v, dict): - f(k, v) + f(k, v, method) continue n = self.j2_ctx for y in k[:-1]: n = n.setdefault(y, PKDict()) getattr(n, method)(k[-1], v) - f([], values) + f([], values, method) def _render_file(self, path, j2_ctx): from pykern import pkjinja