For reference: work towards adding JMTE hub (AWS, eksctl, cloudformation)

config/clusters/jmte/cluster.yaml

@@ -0,0 +1,20 @@
+name: jmte
+provider: none
+hubs:
+  - name: staging
+    domain: staging.hub.jupytearth.org
+    helm_chart: daskhub
+    auth0:
+      connection: github
+    helm_chart_values_files:
+      - common.values.yaml
+      - staging.values.yaml
+  - name: prod
+    display_name: "Jupyter Meets the Earth"
+    domain: hub.jupytearth.org
+    auth0:
+      connection: github
+    helm_chart: daskhub
+    helm_chart_values_files:
+      - common.values.yaml
+      - prod.values.yaml

config/clusters/jmte/prod.values.yaml

@@ -0,0 +1,131 @@
+basehub:
+  nfs:
+    enabled: true
+
+  jupyterhub:
+    # Reverts changes in basehub configuration to the z2jh defaults and ensures
+    # 1 pod is used as a placeholder pod, sized as the smallest node in the JMTE
+    # cluster. We also update singleuser.nodeSelector to ensure we default to
+    # have a placeholder for the smallest nodes only.
+    #
+    prePuller:
+      continuous:
+        enabled: true
+    scheduling:
+      userPlaceholder:
+        enabled: true
+        replicas: 1
+        resources:
+          requests:
+            cpu: 2.5
+            memory: 14G
+
+    singleuser:
+      # This default value will be relevant for the userPlaceholder
+      # configuration, but irrelevant for the defaults we override in our
+      # profileList configuration.
+      #
+      nodeSelector:
+        2i2c.org/node-cpu: "4"
+
+      # Eksctl: The service account was created by eksctl.
+      #
+      serviceAccountName: &user-sa s3-full-access
+
+      extraEnv:
+        # SCRATCH_BUCKET / PANGEO_SCRATCH are environment variables that
+        # help users write notebooks and such referencing this environment
+        # variable in a way that will work between users.
+        #
+        # $(ENV_VAR) will by evaluated by k8s automatically
+        #
+        # Cloudformation: The s3 bucket was created by cloudformation.
+        #
+        SCRATCH_BUCKET: s3://jmte-scratch/$(JUPYTERHUB_USER)
+        PANGEO_SCRATCH: s3://jmte-scratch/$(JUPYTERHUB_USER)
+        # An Amazon RDS postgresql 14 database server has been setup on a
+        # machine with 4 cores and 32 GB memory. See
+        # https://us-west-2.console.aws.amazon.com/rds/home?region=us-west-2#modify-instance:id=jmte-db.https://us-west-2.console.aws.amazon.com/rds/home?region=us-west-2#modify-instance:id=jmte-db
+        #
+        # I created a postgresql user and database for use by some like this:
+        #
+        # CREATE USER proj WITH ENCRYPTED PASSWORD '***';
+        # CREATE DATABASE proj;
+        # GRANT ALL PRIVILEGES ON DATABASE proj TO proj;
+        #
+        JMTE_DB_HOST: jmte-db.cqf1ngjal8bq.us-west-2.rds.amazonaws.com
+
+      initContainers:
+        # Need to explicitly fix ownership here, since EFS doesn't do anonuid
+        - name: volume-mount-ownership-fix
+          image: busybox
+          command:
+            [
+              "sh",
+              "-c",
+              "id && chown 1000:1000 /home/jovyan /home/jovyan/shared /home/jovyan/shared-public && ls -lhd /home/jovyan",
+            ]
+          securityContext:
+            runAsUser: 0
+          volumeMounts:
+            - name: home
+              mountPath: /home/jovyan
+              subPath: "{username}"
+            - name: home
+              mountPath: /home/jovyan/shared
+              subPath: _shared
+            - name: home
+              mountPath: /home/jovyan/shared-public
+              subPath: _shared_public
+
+    proxy:
+      https:
+        hosts:
+          - hub.jupytearth.org
+      traefik:
+        # jupyterhub-ssh/sftp integration part 3/3:
+        #
+        # We must let traefik know it should listen for traffic (traefik entrypoint)
+        # and route it (traefik router) onwards to the jupyterhub-ssh k8s Service
+        # (traefik service).
+        #
+        extraStaticConfig:
+          entryPoints:
+            ssh-entrypoint:
+              address: :8022
+            sftp-entrypoint:
+              address: :2222
+        extraDynamicConfig:
+          tcp:
+            services:
+              ssh-service:
+                loadBalancer:
+                  servers:
+                    - address: jupyterhub-ssh:22
+              sftp-service:
+                loadBalancer:
+                  servers:
+                    - address: jupyterhub-sftp:22
+            routers:
+              ssh-router:
+                entrypoints: [ssh-entrypoint]
+                rule: HostSNI(`*`)
+                service: ssh-service
+              sftp-router:
+                entrypoints: [sftp-entrypoint]
+                rule: HostSNI(`*`)
+                service: sftp-service
+
+dask-gateway:
+  gateway:
+    backend:
+      scheduler:
+        extraPodConfig:
+          serviceAccountName: *user-sa
+      worker:
+        extraPodConfig:
+          serviceAccountName: *user-sa
+
+jupyterhub-ssh:
+  sftp:
+    enabled: true

config/clusters/jmte/staging.values.yaml

@@ -0,0 +1,34 @@
+basehub:
+  nfs:
+    enabled: false
+
+  jupyterhub:
+    custom:
+      singleuserAdmin:
+        extraVolumeMounts: []
+
+    singleuser:
+      storage:
+        type: none
+        extraVolumeMounts: []
+
+      # cmd: Note the default in z2jh is jupyterhub-singleuser.
+      cmd:
+        - jupyterhub-singleuser
+        # WARNING: Collaborative mode is enabled in the staging hub specifically
+        #          to debug a critical issue leading to a loss of data.
+        #
+        #          ref: https://github.com/jupyterlab/jupyterlab/issues/12154#issuecomment-1069352840
+        #          ref: https://discourse.jupyter.org/t/plans-on-bringing-rtc-to-jupyterhub/9813/13
+        #          ref: https://github.com/jupyterlab/jupyterlab/pull/11599
+        #
+        - --LabApp.collaborative=True
+
+    proxy:
+      https:
+        hosts:
+          - staging.hub.jupytearth.org
+
+jupyterhub-ssh:
+  sftp:
+    enabled: false

debug-pod-node-fs.yml

@@ -0,0 +1,32 @@
+apiVersion: v1
+kind: Pod
+metadata:
+  name: node-fs-inspection
+spec:
+  containers:
+    - name: node-fs-inspection
+      image: ubuntu:22.04
+      command: ["sh", "-c", "sleep infinity"]
+      resources:
+        requests:
+          cpu: 225m
+          memory: "939524096"
+      volumeMounts:
+        - name: node-root-fs
+          mountPath: /node-root-fs
+  terminationGracePeriodSeconds: 1
+  nodeSelector:
+    2i2c.org/node-gpu: "1"
+  tolerations:
+    - effect: NoSchedule
+      key: hub.jupyter.org/dedicated
+      operator: Equal
+      value: user
+    - effect: NoSchedule
+      key: hub.jupyter.org_dedicated
+      operator: Equal
+      value: user
+  volumes:
+    - name: node-root-fs
+      hostPath:
+        path: /

debug-pod.yaml

@@ -0,0 +1,63 @@
+apiVersion: v1
+kind: Pod
+metadata:
+  name: jupyter-debugging
+  namespace: prod
+spec:
+  affinity:
+    nodeAffinity:
+      preferredDuringSchedulingIgnoredDuringExecution:
+        - preference:
+            matchExpressions:
+              - key: hub.jupyter.org/node-purpose
+                operator: In
+                values:
+                  - user
+          weight: 100
+  containers:
+    - name: busybox
+      image: busybox
+      command: ["sh", "-c", "sleep infinity"]
+      securityContext:
+        runAsUser: 0
+      volumeMounts:
+        - mountPath: /nfs
+          name: home
+        - mountPath: /home/jovyan
+          name: home
+          subPath: fperez
+        - mountPath: /home/jovyan/shared
+          name: home
+          subPath: _shared
+        - mountPath: /home/jovyan/shared-public
+          name: home
+          subPath: _shared_public
+      resources:
+        requests:
+          cpu: 225m
+          memory: "939524096"
+  nodeSelector:
+    2i2c.org/node-cpu: "4"
+  schedulerName: prod-user-scheduler
+  serviceAccountName: s3-full-access
+  tolerations:
+    - effect: NoSchedule
+      key: hub.jupyter.org/dedicated
+      operator: Equal
+      value: user
+    - effect: NoSchedule
+      key: hub.jupyter.org_dedicated
+      operator: Equal
+      value: user
+    - effect: NoExecute
+      key: node.kubernetes.io/not-ready
+      operator: Exists
+      tolerationSeconds: 300
+    - effect: NoExecute
+      key: node.kubernetes.io/unreachable
+      operator: Exists
+      tolerationSeconds: 300
+  volumes:
+    - name: home
+      persistentVolumeClaim:
+        claimName: home-nfs

deployer/cluster.py

@@ -31,6 +31,8 @@ def auth(self):
             yield from self.auth_azure()
         elif self.spec["provider"] == "kubeconfig":
             yield from self.auth_kubeconfig()
+        elif self.spec["provider"] == "none":
+            yield
         else:
             raise ValueError(f'Provider {self.spec["provider"]} not supported')
 

deployer/cluster.schema.yaml

@@ -28,6 +28,7 @@ properties:
       authentication against the cluster. Currently supports gcp, aws, azure,
       and raw kubeconfig files.
     enum:
+      - none
       - gcp
       - kubeconfig
       - aws

eksctl/README.md

@@ -0,0 +1,74 @@
+### Setup of k8s cluster via eksctl
+
+TODO describe...
+
+### Setup of extras via cloudformation
+
+TODO describe...
+
+### Setup of cluster-autoscaler in the k8s cluster
+
+`eksctl` doesn't automatically install a cluster-autoscaler and it is not part
+of a EKS based k8s cluster by itself, so it needs to be manually installed. The
+cluster-autoscaler will need permissions to do its job though, and for that we
+use some flags in our eksctl config file and then we install it with a Helm
+chart.
+
+#### eksctl configuration for cluster-autoscaler
+
+We need our eksctl-cluster-config.yaml to:
+
+1. Declare `nodeGroups.*.iam.withAddonPolicies.autoScaler=true`.
+
+   I believe doing so is what makes the following tags automatically be applied
+   on node groups, which is required by the cluster-autoscaler to detect them.
+
+   ```
+   k8s.io/cluster-autoscaler/<cluster-name>
+   k8s.io/cluster-autoscaler/enabled
+   ```
+
+2. Declare additional tags for labels/taints.
+
+   ```yaml
+   nodeGroups:
+     - name: worker-xlarge
+       labels:
+         k8s.dask.org/node-purpose: worker
+       taints:
+         k8s.dask.org_dedicated: worker:NoSchedule
+
+       # IMPORTANT: we also provide these tags alongside the labels/taints
+       #            to help the cluster-autoscaler do its job.
+       #
+       tags:
+         k8s.io/cluster-autoscaler/node-template/label/k8s.dask.org/node-purpose: worker
+         k8s.io/cluster-autoscaler/node-template/taint/k8s.dask.org_dedicated: worker:NoSchedule
+   ```
+
+
+#### Installation of cluster-autoscaler
+
+We rely on the [cluster-autoscaler Helm chart](https://github.com/kubernetes/autoscaler/tree/master/charts/cluster-autoscaler) to manage the k8s resources for the cluster-autoscaler we need to manually complement the k8s cluster with.
+
+```
+helm upgrade cluster-autocaler cluster-autoscaler \
+    --install \
+    --repo https://kubernetes.github.io/autoscaler \
+    --version 9.9.2 \
+    --namespace kube-system \
+    --set autoDiscovery.clusterName=jmte \
+    --set awsRegion=us-west-2
+```
+
+### Misc
+
+- Create a auth0 application for github
+- Update dns record ([jupytearth.org is managed on GCP by Erik](https://console.cloud.google.com/net-services/dns/zones/jupytearth-org/details?folder=&organizationId=&project=domains-sos))
+
+### FIXME: Open questions
+
+- How is cluster-autoscaler acquiring the permissions it needs? Is it by being
+  located on the node where we have
+  `nodeGroups.*.iam.withAddonPolicies.autoScaler=true`? Then we have ended up
+  granting permission to all pods on all nodes that are too high.