From 22d3e20b7eb2543611dc45d57f0766e4be000867 Mon Sep 17 00:00:00 2001 From: GeorgianaElena Date: Wed, 28 Jul 2021 22:16:10 +0300 Subject: [PATCH 1/6] Use extraFiles featur to cull idle kernels --- hub-templates/basehub/values.yaml | 33 ++++++++++++++++--------------- 1 file changed, 17 insertions(+), 16 deletions(-) diff --git a/hub-templates/basehub/values.yaml b/hub-templates/basehub/values.yaml index 5bca6ffb7..6e69d01cc 100644 --- a/hub-templates/basehub/values.yaml +++ b/hub-templates/basehub/values.yaml @@ -1,19 +1,3 @@ -etcJupyter: - jupyter_notebook_config.json: - # if a user leaves a notebook with a running kernel, - # the effective idle timeout will typically be CULL_TIMEOUT + CULL_KERNEL_TIMEOUT - # as culling the kernel will register activity, - # resetting the no_activity timer for the server as a whole - MappingKernelManager: - # shutdown kernels after no activity - cull_idle_timeout: 3600 - # check for idle kernels this often - cull_interval: 300 - # a kernel with open connections but no activity still counts as idle - # this is what allows us to shutdown servers - # when people leave a notebook open and wander off - cull_connected: true - nfsPVC: enabled: true shareCreator: @@ -100,6 +84,23 @@ jupyterhub: letsencrypt: contactEmail: yuvipanda@gmail.com singleuser: + extraFiles: + jupyter_notebook_config.json: + mountPath: /usr/local/etc/jupyter/jupyter_notebook_config.json + # if a user leaves a notebook with a running kernel, + # the effective idle timeout will typically be CULL_TIMEOUT + CULL_KERNEL_TIMEOUT + # as culling the kernel will register activity, + # resetting the no_activity timer for the server as a whole + data: + MappingKernelManager: + # shutdown kernels after no activity + cull_idle_timeout: 3600 + # check for idle kernels this often + cull_interval: 300 + # a kernel with open connections but no activity still counts as idle + # this is what allows us to shutdown servers + # when people leave a notebook open and wander off + cull_connected: true startTimeout: 600 # 10 mins, because sometimes we have too many new nodes coming up together defaultUrl: /tree nodeSelector: From 8fd26a018238053d1c551133ceb027f6b53c8ea7 Mon Sep 17 00:00:00 2001 From: GeorgianaElena Date: Tue, 17 Aug 2021 16:08:01 +0300 Subject: [PATCH 2/6] Document culling behaviour --- docs/howto/configure/culling.md | 51 +++++++++++++++++++++++++++++++++ 1 file changed, 51 insertions(+) create mode 100644 docs/howto/configure/culling.md diff --git a/docs/howto/configure/culling.md b/docs/howto/configure/culling.md new file mode 100644 index 000000000..2fd2f7079 --- /dev/null +++ b/docs/howto/configure/culling.md @@ -0,0 +1,51 @@ +# Manage resource culling + +To improve resource management, every user server that's not actively being used, it's shut down by the [jupyterhub-idle-culler](https://github.com/jupyterhub/jupyterhub-idle-culler) Hub service. Thus, any user pod, will be taken down by the idle culler when they are in an idle state. + +Since the server's kernel activity counts as server activity, the idle-culler also operates at a kernel level. This means that if a user leaves a notebook with a running kernel, the kernel will be shut down, if idle for the specified `timeout` period. + +## User server culling configuration + +To configure the server's different culling options, these options must be specified on a per-hub basis, under the appropriate configuration file in `config/hubs`. + +Example: + +```yaml +config: + jupyterhub: + cull: + # Cull after 30min of inactivity + every: 300 + timeout: 1800 + # No pods over 12h long + maxAge: 43200 +``` + +More culling options and info about them, can be found in the [idle-culler documentation](https://github.com/jupyterhub/jupyterhub-idle-culler#readme). + +## Kernel culling configuration + +The kernel culling options are configured through the `jupyter_notebook_config.json` file, located at `/usr/local/etc/jupyter/jupyter_notebook_config.json` in the user pod. This file is injected into the pod’s container on startup, by defining its location and content under [`singleuser.extraFiles`](https://zero-to-jupyterhub.readthedocs.io/en/latest/resources/reference.html#singleuser-extrafiles) dictionary. + +You can modify the current culling options values, under `singleuser.extraFiles.data`, in the `hub-templates/basehub/values.yaml` file. + +Example: + +```yaml +singleuser: + extraFiles: + jupyter_notebook_config.json: + mountPath: /usr/local/etc/jupyter/jupyter_notebook_config.json + data: + MappingKernelManager: + # shutdown kernels after no activity + cull_idle_timeout: 3600 + # check for idle kernels this often + cull_interval: 300 + # a kernel with open connections but no activity still counts as idle + cull_connected: true +``` + +### Note +If a user leaves a notebook with a running kernel, the idle timeout will typically be `CULL_TIMEOUT` + `CULL_KERNEL_TIMEOUT`, as culling the kernel will register activity, resetting the `no_activity` timer for the server as a whole. + From e38d2372916f379e8a7e5fa5e2da74335aa4605f Mon Sep 17 00:00:00 2001 From: Georgiana Elena Date: Tue, 24 Aug 2021 12:31:38 +0300 Subject: [PATCH 3/6] Update docs/howto/configure/culling.md Co-authored-by: Damian Avila --- docs/howto/configure/culling.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/howto/configure/culling.md b/docs/howto/configure/culling.md index 2fd2f7079..e59ac8856 100644 --- a/docs/howto/configure/culling.md +++ b/docs/howto/configure/culling.md @@ -21,7 +21,7 @@ config: maxAge: 43200 ``` -More culling options and info about them, can be found in the [idle-culler documentation](https://github.com/jupyterhub/jupyterhub-idle-culler#readme). +More culling options and information about them can be found in the [idle-culler documentation](https://github.com/jupyterhub/jupyterhub-idle-culler#readme). ## Kernel culling configuration From 1c4f8fa2db0aa76722ee32285ed0ea6f9c60f088 Mon Sep 17 00:00:00 2001 From: GeorgianaElena Date: Tue, 24 Aug 2021 13:08:16 +0300 Subject: [PATCH 4/6] Be more clear about what cull timeouts the docs are about --- docs/howto/configure/culling.md | 2 +- hub-templates/basehub/values.yaml | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/docs/howto/configure/culling.md b/docs/howto/configure/culling.md index e59ac8856..4e53b9ac8 100644 --- a/docs/howto/configure/culling.md +++ b/docs/howto/configure/culling.md @@ -47,5 +47,5 @@ singleuser: ``` ### Note -If a user leaves a notebook with a running kernel, the idle timeout will typically be `CULL_TIMEOUT` + `CULL_KERNEL_TIMEOUT`, as culling the kernel will register activity, resetting the `no_activity` timer for the server as a whole. +If a user leaves a notebook with a running kernel, the idle timeout will typically be the cull idle timeout of the server + the cull idle timeout set for the kernel, as culling the kernel will register activity, resetting the `no_activity` timer for the server as a whole. diff --git a/hub-templates/basehub/values.yaml b/hub-templates/basehub/values.yaml index 6e69d01cc..fbb3968f9 100644 --- a/hub-templates/basehub/values.yaml +++ b/hub-templates/basehub/values.yaml @@ -88,7 +88,8 @@ jupyterhub: jupyter_notebook_config.json: mountPath: /usr/local/etc/jupyter/jupyter_notebook_config.json # if a user leaves a notebook with a running kernel, - # the effective idle timeout will typically be CULL_TIMEOUT + CULL_KERNEL_TIMEOUT + # the effective idle timeout will typically be cull idle timeout + # of the server + the cull idle timeout of the kernel, # as culling the kernel will register activity, # resetting the no_activity timer for the server as a whole data: From b05032819f160303edbaad501aea9475e1627b32 Mon Sep 17 00:00:00 2001 From: GeorgianaElena Date: Tue, 24 Aug 2021 13:08:35 +0300 Subject: [PATCH 5/6] Add the cull docs to the index --- docs/howto/configure/index.md | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/howto/configure/index.md b/docs/howto/configure/index.md index 417bf262c..61eab4ccc 100644 --- a/docs/howto/configure/index.md +++ b/docs/howto/configure/index.md @@ -4,4 +4,5 @@ auth-management.md add-hub-domains.md update-env.md +culling.md ``` \ No newline at end of file From f0726dd6dc1cf82a28b05d296355a2fff801115f Mon Sep 17 00:00:00 2001 From: Chris Holdgraf Date: Thu, 26 Aug 2021 09:33:14 -0700 Subject: [PATCH 6/6] Update docs/howto/configure/culling.md --- docs/howto/configure/culling.md | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/howto/configure/culling.md b/docs/howto/configure/culling.md index 4e53b9ac8..9f3627143 100644 --- a/docs/howto/configure/culling.md +++ b/docs/howto/configure/culling.md @@ -1,3 +1,4 @@ +(configure:culling)= # Manage resource culling To improve resource management, every user server that's not actively being used, it's shut down by the [jupyterhub-idle-culler](https://github.com/jupyterhub/jupyterhub-idle-culler) Hub service. Thus, any user pod, will be taken down by the idle culler when they are in an idle state.