From a97ab8b307f2c167cfd4bde59d9355870d26ca35 Mon Sep 17 00:00:00 2001 From: YuviPanda Date: Tue, 11 May 2021 03:59:42 +0530 Subject: [PATCH 01/12] Add carbonplan cluster + hubs - staging and prod clusters that are exactly the same, with just domain differences - Uses traditional autohttps + LoadBalancer to get traffic into the cluster. Could be nginx-ingress later on if necessary. - Manual DNS entries for staging.carbonplan.2i2c.cloud and carbonplan.2i2c.cloud. Initial manual deploy with `proxy.https.enabled` set to false to complete deployment, fetch externalIP of `proxy-public` service, setup DNS, then re-deploy with `proxy.https.enabled` set to true. Ref https://github.com/2i2c-org/pilot-hubs/issues/291 --- config/hubs/carbonplan.cluster.yaml | 210 ++++++++++++++++++++++++++++ secrets/carbonplan.yaml | 33 +++++ 2 files changed, 243 insertions(+) create mode 100644 config/hubs/carbonplan.cluster.yaml create mode 100644 secrets/carbonplan.yaml diff --git a/config/hubs/carbonplan.cluster.yaml b/config/hubs/carbonplan.cluster.yaml new file mode 100644 index 0000000000..4edebad35b --- /dev/null +++ b/config/hubs/carbonplan.cluster.yaml @@ -0,0 +1,210 @@ +name: carbonplan +provider: kubeconfig +kubeconfig: + file: secrets/carbonplan.yaml +hubs: + - name: staging + domain: staging.carbonplan.2i2c.cloud + template: daskhub + auth0: + connection: github + config: &carbonPlanHubConfig + scratchBucket: + enabled: false + basehub: + nfsPVC: + nfs: + # from https://docs.aws.amazon.com/efs/latest/ug/mounting-fs-nfs-mount-settings.html + mountOptions: + - rsize=1048576 + - wsize=1048576 + - timeo=600 + - soft # We pick soft over hard, so NFS lockups don't lead to hung processes + - retrans=2 + - noresvport + serverIP: fs-2897912f.efs.us-west-2.amazonaws.com + baseShareName: / + shareCreator: + tolerations: + - key: node-role.kubernetes.io/master + operator: "Exists" + effect: "NoSchedule" + jupyterhub: + homepage: + templateVars: + org: + name: Carbon Plan + logo_url: https://pbs.twimg.com/profile_images/1262387945971101697/5q_X3Ruk_400x400.jpg + url: https://carbonplan.org + designed_by: + name: 2i2c + url: https://2i2c.org + operated_by: + name: 2i2c + url: https://2i2c.org + funded_by: + name: Carbon Plan + url: https://carbonplan.org + singleuser: + initContainers: + # Need to explicitly fix ownership here, since EFS doesn't do anonuid + - name: volume-mount-ownership-fix + image: busybox + command: ["sh", "-c", "id && chown 1000:1000 /home/jovyan && ls -lhd /home/jovyan"] + securityContext: + runAsUser: 0 + volumeMounts: + - name: home + mountPath: /home/jovyan + subPath: "{username}" + image: + name: carbonplan/r-retro-notebook + tag: latest + profileList: + # The mem-guarantees are here so k8s doesn't schedule other pods + # on these nodes. + - display_name: "Small: r5.large" + description: "~2 CPU, ~15G RAM" + kubespawner_override: + # Expllicitly unset mem_limit, so it overrides the default memory limit we set in + # basehub/values.yaml + mem_limit: null + mem_guarantee: 12G + node_selector: + hub.jupyter.org/pool-name: notebook-r5-large + - display_name: "Medium: r5.xlarge" + description: "~4 CPU, ~30G RAM" + kubespawner_override: + mem_limit: null + mem_guarantee: 29G + node_selector: + hub.jupyter.org/pool-name: notebook-r5-xlarge + - display_name: "Large: r5.2xlarge" + description: "~8 CPU, ~60G RAM" + kubespawner_override: + mem_limit: null + mem_guarantee: 60G + node_selector: + hub.jupyter.org/pool-name: notebook-r5-2xlarge + - display_name: "Huge: r5.8xlarge" + description: "~32 CPU, ~256G RAM" + kubespawner_override: + mem_limit: null + mem_guarantee: 250G + node_selector: + hub.jupyter.org/pool-name: notebook-r5-8xlarge + scheduling: + userPlaceholder: + enabled: false + replicas: 0 + userScheduler: + enabled: false + proxy: + service: + type: LoadBalancer + https: + enabled: true + chp: + nodeSelector: {} + tolerations: + - key: "node-role.kubernetes.io/master" + effect: "NoSchedule" + traefik: + nodeSelector: {} + tolerations: + - key: "node-role.kubernetes.io/master" + effect: "NoSchedule" + hub: + allowNamedServers: true + networkPolicy: + # FIXME: For dask gateway + enabled: false + readinessProbe: + enabled: false + nodeSelector: {} + tolerations: + - key: "node-role.kubernetes.io/master" + effect: "NoSchedule" + dask-gateway: + traefik: + tolerations: + - key: "node-role.kubernetes.io/master" + effect: "NoSchedule" + controller: + tolerations: + - key: "node-role.kubernetes.io/master" + effect: "NoSchedule" + gateway: + tolerations: + - key: "node-role.kubernetes.io/master" + effect: "NoSchedule" + backend: + scheduler: + extraPodConfig: + nodeSelector: + hub.jupyter.org/pool-name: dask-worker + tolerations: + - key: "k8s.dask.org/dedicated" + operator: "Equal" + value: "worker" + effect: "NoSchedule" + - key: "k8s.dask.org_dedicated" + operator: "Equal" + value: "worker" + effect: "NoSchedule" + worker: + extraPodConfig: + nodeSelector: + hub.jupyter.org/pool-name: dask-worker + tolerations: + - key: "k8s.dask.org/dedicated" + operator: "Equal" + value: "worker" + effect: "NoSchedule" + - key: "k8s.dask.org_dedicated" + operator: "Equal" + value: "worker" + effect: "NoSchedule" + + # TODO: figure out a replacement for userLimits. + extraConfig: + optionHandler: | + from dask_gateway_server.options import Options, Integer, Float, String + def cluster_options(user): + def option_handler(options): + if ":" not in options.image: + raise ValueError("When specifying an image you must also provide a tag") + extra_annotations = { + "hub.jupyter.org/username": user.name, + "prometheus.io/scrape": "true", + "prometheus.io/port": "8787", + } + extra_labels = { + "hub.jupyter.org/username": user.name, + } + return { + "worker_cores_limit": options.worker_cores, + "worker_cores": min(options.worker_cores / 2, 1), + "worker_memory": "%fG" % options.worker_memory, + "image": options.image, + "scheduler_extra_pod_annotations": extra_annotations, + "worker_extra_pod_annotations": extra_annotations, + "scheduler_extra_pod_labels": extra_labels, + "worker_extra_pod_labels": extra_labels, + } + return Options( + Integer("worker_cores", 2, min=1, max=16, label="Worker Cores"), + Float("worker_memory", 4, min=1, max=32, label="Worker Memory (GiB)"), + String("image", default="pangeo/pangeo-notebook:latest", label="Image"), + handler=option_handler, + ) + c.Backend.cluster_options = cluster_options + idle: | + # timeout after 30 minutes of inactivity + c.KubeClusterConfig.idle_timeout = 1800 + - name: prod + domain: carbonplan.2i2c.cloud + template: daskhub + auth0: + connection: github + config: *carbonPlanHubConfig \ No newline at end of file diff --git a/secrets/carbonplan.yaml b/secrets/carbonplan.yaml new file mode 100644 index 0000000000..afbaf8e201 --- /dev/null +++ b/secrets/carbonplan.yaml @@ -0,0 +1,33 @@ +apiVersion: ENC[AES256_GCM,data:Jpo=,iv:lyP0+xfZQ++dLycJ8oU5iFLkpxQsXzvnLBUJKDU+u1c=,tag:KWxZ+98SrWvzNe0LmdKYkQ==,type:str] +clusters: + - cluster: + certificate-authority-data: ENC[AES256_GCM,data:3AwnbHYxXx+6R4j2KUSMGz/9yZ3Ls63yJE4rlXkJqmUFmXTMQyFXzCwoaMzynhqpp2T5IeodUFPSkbXNj7G12bO8o06Wk81M6Ad5mz0BUgBQCUkEMA6pgqVaWrkKa5swNh4YkBkNEG3Z9U0VDOlNi9pO+8T7OhEwtewRHu4WYPUzYfk2hwweHG0al7DsHANT83CQdxL0clX+1H5z2VcKu6OBucdmGOGe8S2z5VJ/4/vOk9NpWZmU1ieoi+wiwjsfSCchQX3MsdzgHWjwtLCyf3pFDgEkyNTKCIZ4JlIyrshKhepKRH33eNLbnFFrIFMIoxIWnfPSm1ek+xosCel507ZYx4d+85auVRXU4XBXsPphPCDbaf5AGdE6Mn6/IaByOt+z4wm+5fzVuSKqwLaITBe/RMiPScqUA4vfI8MfljlSEaphXU7OQMyftm/zs5boCliydBcKD/f4plCDKi9wZ2kgaZfWirEZ5s+BRD+m5lqyg2HqXAtXYnpf4f/njm2/vvNop8+l1zNm2rXspa3Wn6MpoTZ/8ZHBaTuVKl8Y0M807YOGQVHSDxzisNi9l+Ldj2o0vfv2T6do6JCQPXuUautoQSSubXTZa+q7f+deQ5l3t+0m44tF2mNwQ4lJKvSB6NeoEXo4dDQ9MDHRMQ5BMHAjsxBWnxLFCEMUDde2LdF4EYHZhoRAXT/IJyCsIRcL7ppNxO1Os3afDokFhNyQBtAfYHD0Lwel2xK0QyzNHravHbo8GLdHxDw0He/UCPWWhgygEu9btlJ7WRfINlmQvp7eG88AJFuKOPlzPAtgfOGQBxCoY/HG7hNcVcOF3DG42FAWLpqn3TY+QBJk/UjUrrpAgnWSQeWoqWzdOC+lsHwbTV/6AjYa1bncDQySKYQ0hr+gLDTaGqRizeSwElZq3X3tbolvKouKDbvNuX6Oe72OWJzHL+1n5EVM7AvdRrAlSjERiwBM9ensSbzV2jBnROabfrGLn8rppN6kObsxYsVhSDqTh+bmmUjfabgv2yEyiXdSRsMIbQw+T9r7ojW0JGQ1mqCT7V01c19R2uDlTZdZt4WUTZmSLS0GdnbNAY5TB6z0gkDyAujwEvdGtfsKCwm0gnXEuoF2/zOSTqRsCwkYE3RN7G93tK7ntf+sOolBb08u7uWTbta0zhKLKKI/J2Nv6AGolR4j2ohxIcg7BPJqgy+JxSEJhM8bsiSUMWhoErlMHkkZieUbZlcXr9aWOqWKRCcbM0UopkTLwSvly4i88qfMGqibDEBgGJX7TtTijwpPfRSYar+ShuXjoTYNcT6m1hZbQSfSgammWp3ORvX4Hwu6coImrd+N16Z3FH8LMPfhMcuWpiKgW1ORQTxG1e2MQUWaamtWHePel59yf+Zz1YZchEVjmc5F3P88fLea/CsrqpdI52e0B3q6RUs5exboQWCTVcSETrUjk1bZNkXP5eMwACQyfdm3Ryq6tFfN81D27dhpHNcDsiwQ6Gjau6vAsbW5rKOGXG3NFxK6+ASqUdre7m8YdplxdRuDjkK0VL6f3FzOFtwbM5ki90skVom8+EoXLEfySY0ayheuyH1QcJUKUr8LTf87ewEQ+c3qwnWlh20S1aGGzHSHFPhxqhZ5JFTiI5xgp9F3Qli2gpHzsLY84reqwlaR/CK/DKxTG95tWSzxWG1n9MnGgEMn4kaQKSrpnTm15k8CRPIpS1lsg4ZY/+pUJBC0XKLReQ6UXpOCryG9Hmuufg8r460gm6FiXmnUWpZCMIaU451k5/T41YJWvcHqLjvcSkBw+1V45x39swoROsouOhMm+DzQjVcd2Joq814cQZTg/3rL9pFKiMut71WinpTRogvNDgi+DPz2NREQKOUpeHe1u/MwcdQHHxn4DW2xp/x9pNQk53JDP4EhdcH6k9hT83DUcAq3o8ZFDw==,iv:w5OZN0GZZcfam1EZZFPmodhHLtuM7DJHBcYTbHL5YkA=,tag:SJMcHsd2gMWPgjIZLfhLmw==,type:str] + server: ENC[AES256_GCM,data:6syyGxOJVVKJuafONJLvDGcxjsNZamwpM3nlai/XzjwEVCcL2AU0aotTITq7dRxq3UzTFDFfHlIOoOttDCLM/2ML1hDksvKXv0XVDJW7TA==,iv:8JkMVUULGwPAr1ex0541ML7mtzsFWASskxCoG40d7B0=,tag:1TDOlMl1MOSuA2ixUCPvMg==,type:str] + name: ENC[AES256_GCM,data:Tn/6Zhy+CsRngO9/k0uLYKuldAHl8Bk=,iv:3wdHmttch2xna41q5U77rMURb6YVjcfpruoCJBJ0bck=,tag:DI09Rph0WUHzK/Ti4NWgqg==,type:str] +contexts: + - context: + cluster: ENC[AES256_GCM,data:wPo4tztl6bOkhVLd54hwn9WSj0vfBk8=,iv:paJp95y/YA8NgzsrPToNruQmGQOeeMOG38i1aj7OJfQ=,tag:HoeiurynOpqfljXY2opjLg==,type:str] + user: ENC[AES256_GCM,data:7iPDh5EKQ8LHaztvAIzGMfVy0wQF6yE=,iv:urVMXazL+ZYn1GKGEwTI/yhMPyEVa8VZOUkjFb+eojE=,tag:ov1rFT7/IHvAF5J5f11fng==,type:str] + name: ENC[AES256_GCM,data:MOCzSvUViwVZjFHAjMKtHPC0diuFxFw=,iv:CEIHGCt+/IWpXH+6eeHsckKne3c8cnpa6CQ31Es+OSM=,tag:MoREm99ZyOlrBpi7xI3AQg==,type:str] +current-context: ENC[AES256_GCM,data:+XE07HU+QDfHqJ0o2LbY+D3aN3o0XQU=,iv:mbraHPQfBopbpebbzmizIHNiJbPUPZyuQDVDyIYTxuY=,tag:4ab/QvwdZdbJVv9PErXZ3g==,type:str] +kind: ENC[AES256_GCM,data:r/FzqNYQ,iv:FOogwzFq4fZd/KRfM5z52cl5tW3BOodobU5rAbxH8Lw=,tag:OnKZJPEh9AShHjyvaD3NYA==,type:str] +preferences: {} +users: + - name: ENC[AES256_GCM,data:eR7e+4X2GrnyCLTPMaYesDyjHcPPAns=,iv:gYg8ETXenwsGSRjOGwsTTyw5QllkXQ2B8u6Vdgrfvog=,tag:DQMW/N7cAyeg8yzwSzajyw==,type:str] + user: + client-certificate-data: ENC[AES256_GCM,data:J46C7tCbr4LiHkA4K1KV8hWOPGejCzHtAJF1KgH17lTREhD8lw4qDGoKJ6/KWUzbNreaMZv26UNde0s5PwOUt3FW50jbm4bKE3jypdAelHaFUnCkP9Eqg1/bGHV6CoflUWAiedD/4lNV9bfyApDi/h2sqsEYJSyHYae8cbNhCo/1ha4ppGbPZE1TxVv3OyVpV3PJBtrAcesx5GX5HpU0Ale8oPZqPUL6zcuqLTNTerUOKdNeoSpXlpR929xwV7bSMLD0Cbst68b0u/wM266CFuwLk99L/u6Lc8lg2Gt28IR9jW3/tJApuAziTaRfdLWAyoIlX9kPBJo90HAVMBde8UJedXMzNYBZVb0zIwWu9y0/uHIlQLAi1S/OFUpfnxHugBcU2+CBBcA9Odd7vWRNcMuk9N6F034ePBj5F6Mlekvf1haNwTFTJrroLILSSmFni4a8dg3Nq3oCx6ORdIOVVYdCJBGvUa2TKrje3nQN4YTTlDYg84gIOrwNu9rvvrTBvvgAYESeoX07/6bDe1+HYAIAP9DmsZPOULvAkH6mI6g6ifNoJHGTN+YgPZyIIGMhb8TPsj6AihQl3BRTDJx2cHt1QVbDOswE2Tq+5lBen+2Zs1BZLRq4IZjTQVmkN4IDjUmqPzPnWcpqIsBgaHel4vdbXq22xr205GlqNm+cC45h5xYiUkHJNakBaIGGaoJG3znxJ4vTNshUq1s1l55COdNcK3mGfQlunbZRG4aNMl8kXT4p2WOEx3hrOzNarcwuToa6yWYoiiwhfp5CfIIg8LESsUPHtZu7TqOxdQS6B2YHEvQjzose8SAuM6IAmFXH+1Y3xssH86jPEH75lfIWrMOqc9n3sJgWHKAzcQdMDu4B30GBx2PJ4EVIjTNsFZ35GQiioXgYYkK6hQmCTWxwLMtyKI8rI63wzkhbL/zt+h1legFB9QBUqZ+GBaoSMbPFJQjc4dPbvmO/U37g0oWJRl2RTN2f48bFrsAM24CeB/p/o15eGpNnD5ptL2RQ8T2pqOBcDQJLgNPvFJyPnOG/1jOWv9OU0PZjJA6eabI+YB1fw4xkBUXspMM0WnYwI/DCTFyy3UWT5Y9MaXxspp5rHFr978W5kXRtCXRfkSJAm4lcxcMfkc/7XcuK2wMXlGXYuDO8EiM0H8xJ6cYh/IXnUz76f86IMbadlcqeDOFHfKFdgHkW+44J4+QncoyZVRlCdzqAz2luyAF0gkzLQGMkwLelKSmlx7nwUBfbmsFAm+jy3UgkN8Df3+9W+qMz8Op8CjDDXgldjN3W9uojFdRgLJTwuUrkoPiT9rZbXn4XRyNiIe3IvFvPYpHrrh0VVIi0pwBuOkMOLWpWOCAWQ99etKkm07H6l8x5HDZxFNRye3dtP2bz2YATV1Q5EmPuvAA+RI6lyUOOCP6F3IXqtS40/ML65UOGDzp7xf3BeR/2l7EnSA9R24Rw/1UmBBN+lAw4TGckk+y5ixHALlpBTRqhSt0tF6cTgbO12dM1EVfVyjHBCZaxe1c8WyE8XUB9jsHz+cUtLAdcO0I62x9XGrq6V2Jc+HNkGSkbUdEA1Oq853gDH6BxZuba94rAWTHoUN0OBcRBmFBsUL06JE5XWFZb/AfNi3enPMjBbFMKr02ctRGE8czhu8S8fo+Pkjv8YNCjDW/2gVapggHjyyiaLI8SewH0606ZPq9HTjxxONrBNeveePc/YY/5HnJE+lKj6oG/Ixk7J/tq6SOkz8WeJzBsgJ+42AAUPwpsg9hGVIga81C8ClNgZaX1ju4Jz587yL3sLaKaGl7KGEs9RGjLOx4UA6Myk8URwbw2gBaOiBPK7nb/x5x04jxWF1tgM6eYWJQ9Gi1QKt45LgYPdzYyCoOumcJl+NjWua2Q2U7x/xJcQbn5I31FH2YmSwfFNLfWPcrNSG9jLkB3J9uBT/k61/pZGmWuBKcf0GWX7fNpkH4CdpuYOhi9lfpI4lIARMJDqp/cRwQu5l+E7hWs3floD0qr2l1fCMWxxApjtwBV93NUe95XTbnKgp+Wh7gWEBKtSDmB6CXZOmop8OHPpfwS,iv:efCIgmdnzcW/0vXjMirln8t+NFRmboaVaRxme6H2XaM=,tag:sxMpJUlNNGk1fK+W/KtEcA==,type:str] + client-key-data: ENC[AES256_GCM,data:XljhSGl4WHMv7hpNDRAtu9kdUKH1rePyRwceMbicditzDq5K61pwbvncYWX6hYJ1x+9scvVoRuk3H22ai52xZty7FJb0DiWu6siNLNjTmQCWoGFneBprQRsy+SI7EWgVZoF7XIbVYC6rP8by9U3v5YDMQPMVTSdoxfuk7J0FE65SzCID5uns0Ia2VJkQP8cGbXZq4Us4lzA0L0R6hV4ak6Xt7hIPwi/Fd1zWAG4XmTXtZgh07iKORbDpkWN6UpgpirsgRTxkr9sRh3MVQjVBWr0rz35nvRjn4eA/gxZwCcvTyej98nN6AzFPdIm+2aTTsj5+1nLt4Pny9sDDj9cRPyz/OlKd5E95Yt1zfePaoUlQaH59aL4f0gleSmLlCc+xDPkv7N7B5bdyMGr+ROiDA26HmPxF/2Z65pFNz4sHXVLfCpFggRAkhO/N4As1Zt9RFkyCz/Iu1pBVom9TEPxjfsSrqOS/rOcLgepqE60wdFsFVIdwJUo7xPa2IWUFo0Duwe05yDphijYhd5ieLIr6V3qL5m3LQfDZhYcY8kE93Q8Cx1UsYWv0f+Uy96eDDxZBvhZvnMjGm0i9rV8hcxGj9O5YR2Kq+laPgRKKObh40qwTmlCkZKkp4dmTvQnZ4ZYFFEFcKgARN5hT1Ha1VbxwBAohHocce+cjSQg+J49QYNZe1Rh3ZuYvpFL52f00a0YK+93xGyodL2PxCqMaVMVmHhwU65ICtgrLgtAELSTuNOGT9FEqAURoJrc7Oc0YxP/kzl7YHSdjH/tTXQyHVVXfnY0WOLvahz09xyNN1ip2c5FWKVpAiJ+424lASlsvfQV6hYTB7VXsXT8AGofz7oGCuGMHqwcQZa83QAI8sr493SO3rDWZ0HqpvCEtI6vsI1kfp7l2ZyrMUs/C1x24RCILyhbYKqA0bB9Nqyk27d7vUZmhaSnJ8PiU5hZWZWv756eAM2YPfaJMG0FBpQfOFn0zXh+6Zn4v01X7k9voA9V9m383eYRq6Mejkf6Xl5gX6JUVGpIH9Nv5Rrr6LK67h7kqi6B+4AB9yhe1CzSTuQRy3IV8eTyZuWvGnsC4rNQumMK6EfjqNWD/4g5JalmucJymWVXxUpNvrKgYebVkcazglI1AiPlxiAccYpwcKdHpnogFZnlxpVVYKGNjCy1BrseJ2sfPJhIpJOLlAu330nrwgYhHErRhmV+rcrISVb1rKIkalExD882bFXBoZANByTieJVzs9FcbrH3/s6wXEkSgsMnRCmXQVPrx36xkNn5sVFeuxZaPAFnnSdhBhS41KBnLL09hEo87/KJNAYKIKX1KMaw3K08sdRHT2BLrRat8rSGDYd7qIaosdUg6hPevlBDixI3rsts7/dB1mxPmAPLiN2jZ+QBDMs3WVK52aae0Rt4kEkk6P9TIdLhQODjjyPl7JUapYzB42SrWDZNZL9fao2SwhI41yKawAf8QlSDFU+OFehYzO7pg/tBTRRcF+VS6E7G3/Dz2iZnC/iv+ie3PDP9AeyItp8ajTFS/rTDLpTPDQbzIZjypOSoydqBsr354Rp6ULx5Y+jF2TEeFGcbJYelN0VUjoTCcl5sOF6CsMtjnjcgTWECDzjNYekPuyjuztWoF0zlnzcVdnHwN10Gcyw8NGGTJ6HktJv3JLnwCEoSKO+VZ9/H8vemqrHVquLBpPFHWN7c4mE1R9dNv9otq7YE/eVDkYKxHCqfZ/P1MAfelGz+4gRc0b72iwlw/qweroBmeiGvX/sZUOBdvRvZoV1q1NOUPmlhlviZgmU4nNGKC5zfOv5kIShhWd8sTQCggZhOvuhOkjgUN6ODTMkAaOf9ymK49uQaCopphE/csOAmkLcSlsmOUrqyk8DpjNUoSQ3qth4o0UcHyC6He/V63LKsIlGwYDP3Y6wW/0GlqqrzkJ3q4IyBY4FF+MQVsNoZd1uKZH1emYF6rptt9PLAlKFW80JtES0Fl1Ls/vsvT48PmGPzHLGRnP7qUxEYaD+0bNHIy2OD8Io7646okuPs5sQcE6V9ngDU0K9ypD5GuAXTRN4Xt4pKBiTb9z2HkN92hzYMh+5Sclq50+0JD3/8NLC/PIxpXwsW3ZRo87I8DNwZAJnjd7G1nXglJlpxooLU07nLOfOYh7zcEc7ZYjbXsqq8p0yxdhpo5zPjLX/rRGv9guadKIp/nIR/N2BX9jXe/kr885m51njGd7fRxlH9ZRqkEYHEHou8f4OKCekcCvoL2AaamtWnASc5J8mYHIzVsX3mYbecrFqB780uuaoCLJC6RV15J8fi62ckh1KM41XuCDr9VW582FQl83+MBHzDlAocvYt1GTyZjW7j+N2/yh5YUyHwEgBs1w7zrBjPlOKW5S5WFJZK0rnDO7cEDwx7smX5yA2AYqfZpMQbfnIRlOBcqCcc5QVHXg7JPeY0uTGSt4nPlp0bHVyIiUjqdznyjGS7dIy6CQu97Rxe6910H4hRpsTd8frzNCO8Jk0Xw2klVAXgxtxAINK/o/2G2I0NAZVOlo9nMYMozcLAta3lcxxsNRk7AiSRLjEevsaIloLdacbtveMiIyAJBgBd7Pl1iFwa3onk21DypgmvPkwA4YJyO0Cu5T3MGsEUepnDAGUOqr65sqB25SMHOeIR1c1zUJmH5OU/tdS7nTAFqZmF49HgMhdWfHvP0XhwauI8esLZrH0XhTsaBsiuJsgVLZEODKmrROlx3CuLWSgtBzW2GrsXPctftAWhjLtv2vu9DvtU7MpRBwn16C59imgLnBVShm1S0Dr4c2UpS60pDGIecV+qjyKPqHwUYq8YdgF4OlMebk+5ejqqUiqShmWvzlbDcJLE1MIOB9mjecdlTjJg4SUN6rcpoTlief9YjKenkV2Q1NGjr/qVflqV1w+3YbI7G04+fig9d0OXuoDlSeMgsCVdKXAVaCKCGtMUp0xLzLxXCrBCtX13diLb9TQ2ZRk2E36KP8sei+jKeePX/ATZxQ6M=,iv:Yd0SBcdQbcA6AwEAr7rLYfX5XTPyITVVIJ/Lbexq6kc=,tag:XmTFiGg5tesV+jw5VTCLxg==,type:str] +sops: + kms: [] + gcp_kms: + - resource_id: projects/two-eye-two-see/locations/global/keyRings/sops-keys/cryptoKeys/similar-hubs + created_at: "2021-05-10T17:35:08Z" + enc: CiQA4OM7eLM6u6MlzoUjfIYlN5H/wC7Dr3n4xmiRNdOO6TKOd+YSSQBy9hCYancmxxKoD4g2+zDN2GU4aKKSHq2BwUiYNCBwLs8U5BV2yhzKSnjY5cEbM+9YMV2holXPYrthBCtola5bR69V0nORC9g= + azure_kv: [] + hc_vault: [] + age: [] + lastmodified: "2021-05-10T17:35:10Z" + mac: ENC[AES256_GCM,data:nIIqCgBY7lmTp0k5If5BXHFeeRUJ0OdkU66Z7/bZOc4EElu9JbXSrNPfjcllk6pzXB6XGOIU2RuIhDTcl/DZSvCAgYuFkorQLYZ2js/2yNxSZY2sYcRnQB+C/R7PgwpEyObpTQ2faRmcVEYPzGKTBml9GyZo1RfNllxOuUeykHM=,iv:oxKFojrFq7/AiX8ytiiwq/1lA3IuHg3aOh/EBfknsUo=,tag:KX7fhH78UOvzSPm/rjqXBQ==,type:str] + pgp: [] + unencrypted_suffix: _unencrypted + version: 3.7.1 From 3c344a4c6a52cb670ea1856dc60714d3082b79ee Mon Sep 17 00:00:00 2001 From: YuviPanda Date: Tue, 11 May 2021 21:00:12 +0530 Subject: [PATCH 02/12] Standardize labels used for our various nodes We have three sets of labels: 1. What components of a JupyterHub can run here? core / user 2. What components of a dask gateway can run here? core / scheduler / worker 3. What are the features of the node pool we care about? For example, if we want to be on an r5.xlarge node, we should target the existing node.kubernetes.io/instance-type label This gives us flexibility without adding too much overhead. --- config/hubs/carbonplan.cluster.yaml | 36 ++++------------------------- hub-templates/basehub/values.yaml | 10 ++++---- hub-templates/daskhub/values.yaml | 11 ++++----- kops/carbonplan.jsonnet | 34 +++++++++++++++++++++++---- 4 files changed, 44 insertions(+), 47 deletions(-) diff --git a/config/hubs/carbonplan.cluster.yaml b/config/hubs/carbonplan.cluster.yaml index 4edebad35b..1c11b99d55 100644 --- a/config/hubs/carbonplan.cluster.yaml +++ b/config/hubs/carbonplan.cluster.yaml @@ -71,28 +71,28 @@ hubs: mem_limit: null mem_guarantee: 12G node_selector: - hub.jupyter.org/pool-name: notebook-r5-large + node.kubernetes.io/instance-type: r5.large - display_name: "Medium: r5.xlarge" description: "~4 CPU, ~30G RAM" kubespawner_override: mem_limit: null mem_guarantee: 29G node_selector: - hub.jupyter.org/pool-name: notebook-r5-xlarge + node.kubernetes.io/instance-type: r5.xlarge - display_name: "Large: r5.2xlarge" description: "~8 CPU, ~60G RAM" kubespawner_override: mem_limit: null mem_guarantee: 60G node_selector: - hub.jupyter.org/pool-name: notebook-r5-2xlarge + node.kubernetes.io/instance-type: r5.2xlarge - display_name: "Huge: r5.8xlarge" description: "~32 CPU, ~256G RAM" kubespawner_override: mem_limit: null mem_guarantee: 250G node_selector: - hub.jupyter.org/pool-name: notebook-r5-8xlarge + node.kubernetes.io/instance-type: r5.8xlarge scheduling: userPlaceholder: enabled: false @@ -138,34 +138,6 @@ hubs: tolerations: - key: "node-role.kubernetes.io/master" effect: "NoSchedule" - backend: - scheduler: - extraPodConfig: - nodeSelector: - hub.jupyter.org/pool-name: dask-worker - tolerations: - - key: "k8s.dask.org/dedicated" - operator: "Equal" - value: "worker" - effect: "NoSchedule" - - key: "k8s.dask.org_dedicated" - operator: "Equal" - value: "worker" - effect: "NoSchedule" - worker: - extraPodConfig: - nodeSelector: - hub.jupyter.org/pool-name: dask-worker - tolerations: - - key: "k8s.dask.org/dedicated" - operator: "Equal" - value: "worker" - effect: "NoSchedule" - - key: "k8s.dask.org_dedicated" - operator: "Equal" - value: "worker" - effect: "NoSchedule" - # TODO: figure out a replacement for userLimits. extraConfig: optionHandler: | diff --git a/hub-templates/basehub/values.yaml b/hub-templates/basehub/values.yaml index bda8c5768c..92095cc2b1 100644 --- a/hub-templates/basehub/values.yaml +++ b/hub-templates/basehub/values.yaml @@ -52,7 +52,7 @@ jupyterhub: userScheduler: enabled: true nodeSelector: - hub.jupyter.org/pool-name: core-pool + hub.jupyter.org/node-purpose: core resources: requests: # FIXME: Just unset this? @@ -72,7 +72,7 @@ jupyterhub: type: ClusterIP chp: nodeSelector: - hub.jupyter.org/pool-name: core-pool + hub.jupyter.org/node-purpose: core resources: requests: # FIXME: We want no guarantees here!!! @@ -83,7 +83,7 @@ jupyterhub: memory: 1Gi traefik: nodeSelector: - hub.jupyter.org/pool-name: core-pool + hub.jupyter.org/node-purpose: core resources: requests: memory: 64Mi @@ -102,7 +102,7 @@ jupyterhub: startTimeout: 600 # 10 mins, because sometimes we have too many new nodes coming up together defaultUrl: /tree nodeSelector: - hub.jupyter.org/pool-name: user-pool + hub.jupyter.org/node-purpose: user image: name: set_automatically_by_automation tag: 1b83c4f @@ -183,7 +183,7 @@ jupyterhub: JupyterHub: authenticator_class: oauthenticator.generic.GenericOAuthenticator nodeSelector: - hub.jupyter.org/pool-name: core-pool + hub.jupyter.org/node-purpose: core networkPolicy: enabled: true ingress: diff --git a/hub-templates/daskhub/values.yaml b/hub-templates/daskhub/values.yaml index cef9db1a81..483648c820 100644 --- a/hub-templates/daskhub/values.yaml +++ b/hub-templates/daskhub/values.yaml @@ -123,10 +123,10 @@ dask-gateway: # See https://github.com/dask/dask-gateway/blob/master/resources/helm/dask-gateway/values.yaml controller: nodeSelector: - hub.jupyter.org/pool-name: core-pool + k8s.dask.org/node-purpose: core gateway: nodeSelector: - hub.jupyter.org/pool-name: core-pool + k8s.dask.org/node-purpose: core backend: scheduler: extraPodConfig: @@ -143,8 +143,7 @@ dask-gateway: value: "user" effect: "NoSchedule" nodeSelector: - # Schedulers should be in the user pool - hub.jupyter.org/pool-name: user-pool + k8s.dask.org/node-purpose: scheduler cores: request: 0.01 limit: 1 @@ -171,7 +170,7 @@ dask-gateway: effect: "NoSchedule" nodeSelector: # Dask workers get their own pre-emptible pool - hub.jupyter.org/pool-name: dask-worker-pool + k8s.dask.org/node-purpose: worker # TODO: figure out a replacement for userLimits. extraConfig: @@ -217,6 +216,6 @@ dask-gateway: type: jupyterhub # Use JupyterHub to authenticate with Dask Gateway traefik: nodeSelector: - hub.jupyter.org/pool-name: core-pool + k8s.dask.org/node-purpose: core service: type: ClusterIP # Access Dask Gateway through JupyterHub. To access the Gateway from outside JupyterHub, this must be changed to a `LoadBalancer`. diff --git a/kops/carbonplan.jsonnet b/kops/carbonplan.jsonnet index c266c3fe38..9cce3c07e4 100644 --- a/kops/carbonplan.jsonnet +++ b/kops/carbonplan.jsonnet @@ -33,7 +33,8 @@ local data = { machineType: "t3.medium", subnets: [zone], nodeLabels+: { - "hub.jupyter.org/pool-name": "core-pool" + "hub.jupyter.org/node-purpose": "core", + "k8s.dask.org/node-purpose": "core" }, // Needs to be at least 1 minSize: 1, @@ -41,7 +42,7 @@ local data = { role: "Master" }, }, - nodes: [ + notebookNodes: [ ig { local thisIg = self, metadata+: { @@ -56,7 +57,8 @@ local data = { maxSize: 20, role: "Node", nodeLabels+: { - "hub.jupyter.org/pool-name": thisIg.metadata.name + "hub.jupyter.org/node-purpose": "user", + "k8s.dask.org/node-purpose": "scheduler" }, taints: [ "hub.jupyter.org_dedicated=user:NoSchedule", @@ -64,10 +66,34 @@ local data = { ], }, } + n for n in nodes + ], + daskNodes: [ + ig { + local thisIg = self, + metadata+: { + labels+: { + "kops.k8s.io/cluster": data.cluster.metadata.name + }, + name: "dask-%s" % std.strReplace(thisIg.spec.machineType, ".", "-") + }, + spec+: { + machineType: n.machineType, + subnets: [zone], + maxSize: 20, + role: "Node", + nodeLabels+: { + "k8s.dask.org/node-purpose": "worker" + }, + taints: [ + "k8s.dask.org_dedicated=worker:NoSchedule", + "k8s.dask.org/dedicated=worker:NoSchedule" + ], + }, + } + n for n in nodes ] }; [ data.cluster, data.master -] + data.nodes \ No newline at end of file +] + data.notebookNodes + data.daskNodes \ No newline at end of file From 6553ab5a3b040b3b8ef1aa75d528b431a030ce41 Mon Sep 17 00:00:00 2001 From: YuviPanda Date: Tue, 11 May 2021 21:13:51 +0530 Subject: [PATCH 03/12] Use upstream pangeo image dask-gateway requires that the image used for it contains the `dask-gateway` package. The scheduler image is the same image as the user notebook image, to make sure that versions match. The previously used image did not have dask-gateway installed --- config/hubs/carbonplan.cluster.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/config/hubs/carbonplan.cluster.yaml b/config/hubs/carbonplan.cluster.yaml index 1c11b99d55..cfb3c22347 100644 --- a/config/hubs/carbonplan.cluster.yaml +++ b/config/hubs/carbonplan.cluster.yaml @@ -58,8 +58,8 @@ hubs: mountPath: /home/jovyan subPath: "{username}" image: - name: carbonplan/r-retro-notebook - tag: latest + name: pangeo/pangeo-notebook + tag: 2021.05.04 profileList: # The mem-guarantees are here so k8s doesn't schedule other pods # on these nodes. From 4593a11645f2ab5c47149cfc6cce7fc5170a3403 Mon Sep 17 00:00:00 2001 From: YuviPanda Date: Wed, 12 May 2021 00:02:04 +0530 Subject: [PATCH 04/12] Tell clusterautoscaler about node.kubernetes.io/machine-type Otherwise it doesn't know which instance group to scale up when a pod wants a node with that label --- kops/libsonnet/instancegroup.jsonnet | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/kops/libsonnet/instancegroup.jsonnet b/kops/libsonnet/instancegroup.jsonnet index e19dcdde7e..bea8ccc305 100644 --- a/kops/libsonnet/instancegroup.jsonnet +++ b/kops/libsonnet/instancegroup.jsonnet @@ -26,7 +26,11 @@ local makeCloudTaints(taints) = { }, spec: { image: "099720109477/ubuntu/images/hvm-ssd/ubuntu-focal-20.04-amd64-server-20210119.1", - cloudLabels: makeCloudLabels(self.nodeLabels) + makeCloudTaints(self.taints), + cloudLabels: { + // Tell autoscaler to scale up this instancegroup when something asks for a node with the label + // node.kubernetes.io/instance-type: + "k8s.io/cluster-autoscaler/node-template/label/node.kubernetes.io/instance-type": $.spec.machineType + } + makeCloudLabels(self.nodeLabels) + makeCloudTaints(self.taints), taints: [], nodeLabels: {}, machineType: "", From 09302c9e6e654f7f3bbe6a7ba72f55343b0e4118 Mon Sep 17 00:00:00 2001 From: YuviPanda Date: Wed, 12 May 2021 00:02:55 +0530 Subject: [PATCH 05/12] Use new cluster creds + EFS --- config/hubs/carbonplan.cluster.yaml | 2 +- secrets/carbonplan.yaml | 32 ++++++++++++++--------------- 2 files changed, 17 insertions(+), 17 deletions(-) diff --git a/config/hubs/carbonplan.cluster.yaml b/config/hubs/carbonplan.cluster.yaml index cfb3c22347..1c0eae7cbd 100644 --- a/config/hubs/carbonplan.cluster.yaml +++ b/config/hubs/carbonplan.cluster.yaml @@ -22,7 +22,7 @@ hubs: - soft # We pick soft over hard, so NFS lockups don't lead to hung processes - retrans=2 - noresvport - serverIP: fs-2897912f.efs.us-west-2.amazonaws.com + serverIP: fs-8a4e4f8d.efs.us-west-2.amazonaws.com baseShareName: / shareCreator: tolerations: diff --git a/secrets/carbonplan.yaml b/secrets/carbonplan.yaml index afbaf8e201..c7086b8dc7 100644 --- a/secrets/carbonplan.yaml +++ b/secrets/carbonplan.yaml @@ -1,33 +1,33 @@ -apiVersion: ENC[AES256_GCM,data:Jpo=,iv:lyP0+xfZQ++dLycJ8oU5iFLkpxQsXzvnLBUJKDU+u1c=,tag:KWxZ+98SrWvzNe0LmdKYkQ==,type:str] +apiVersion: ENC[AES256_GCM,data:Pbg=,iv:n1l/qA/aY7Vdl/9KgRQV90Yprvz14NFcm5Tpzvc2+HE=,tag:jlmp4qMo5APRUYlZOvaWFg==,type:str] clusters: - cluster: - certificate-authority-data: ENC[AES256_GCM,data:3AwnbHYxXx+6R4j2KUSMGz/9yZ3Ls63yJE4rlXkJqmUFmXTMQyFXzCwoaMzynhqpp2T5IeodUFPSkbXNj7G12bO8o06Wk81M6Ad5mz0BUgBQCUkEMA6pgqVaWrkKa5swNh4YkBkNEG3Z9U0VDOlNi9pO+8T7OhEwtewRHu4WYPUzYfk2hwweHG0al7DsHANT83CQdxL0clX+1H5z2VcKu6OBucdmGOGe8S2z5VJ/4/vOk9NpWZmU1ieoi+wiwjsfSCchQX3MsdzgHWjwtLCyf3pFDgEkyNTKCIZ4JlIyrshKhepKRH33eNLbnFFrIFMIoxIWnfPSm1ek+xosCel507ZYx4d+85auVRXU4XBXsPphPCDbaf5AGdE6Mn6/IaByOt+z4wm+5fzVuSKqwLaITBe/RMiPScqUA4vfI8MfljlSEaphXU7OQMyftm/zs5boCliydBcKD/f4plCDKi9wZ2kgaZfWirEZ5s+BRD+m5lqyg2HqXAtXYnpf4f/njm2/vvNop8+l1zNm2rXspa3Wn6MpoTZ/8ZHBaTuVKl8Y0M807YOGQVHSDxzisNi9l+Ldj2o0vfv2T6do6JCQPXuUautoQSSubXTZa+q7f+deQ5l3t+0m44tF2mNwQ4lJKvSB6NeoEXo4dDQ9MDHRMQ5BMHAjsxBWnxLFCEMUDde2LdF4EYHZhoRAXT/IJyCsIRcL7ppNxO1Os3afDokFhNyQBtAfYHD0Lwel2xK0QyzNHravHbo8GLdHxDw0He/UCPWWhgygEu9btlJ7WRfINlmQvp7eG88AJFuKOPlzPAtgfOGQBxCoY/HG7hNcVcOF3DG42FAWLpqn3TY+QBJk/UjUrrpAgnWSQeWoqWzdOC+lsHwbTV/6AjYa1bncDQySKYQ0hr+gLDTaGqRizeSwElZq3X3tbolvKouKDbvNuX6Oe72OWJzHL+1n5EVM7AvdRrAlSjERiwBM9ensSbzV2jBnROabfrGLn8rppN6kObsxYsVhSDqTh+bmmUjfabgv2yEyiXdSRsMIbQw+T9r7ojW0JGQ1mqCT7V01c19R2uDlTZdZt4WUTZmSLS0GdnbNAY5TB6z0gkDyAujwEvdGtfsKCwm0gnXEuoF2/zOSTqRsCwkYE3RN7G93tK7ntf+sOolBb08u7uWTbta0zhKLKKI/J2Nv6AGolR4j2ohxIcg7BPJqgy+JxSEJhM8bsiSUMWhoErlMHkkZieUbZlcXr9aWOqWKRCcbM0UopkTLwSvly4i88qfMGqibDEBgGJX7TtTijwpPfRSYar+ShuXjoTYNcT6m1hZbQSfSgammWp3ORvX4Hwu6coImrd+N16Z3FH8LMPfhMcuWpiKgW1ORQTxG1e2MQUWaamtWHePel59yf+Zz1YZchEVjmc5F3P88fLea/CsrqpdI52e0B3q6RUs5exboQWCTVcSETrUjk1bZNkXP5eMwACQyfdm3Ryq6tFfN81D27dhpHNcDsiwQ6Gjau6vAsbW5rKOGXG3NFxK6+ASqUdre7m8YdplxdRuDjkK0VL6f3FzOFtwbM5ki90skVom8+EoXLEfySY0ayheuyH1QcJUKUr8LTf87ewEQ+c3qwnWlh20S1aGGzHSHFPhxqhZ5JFTiI5xgp9F3Qli2gpHzsLY84reqwlaR/CK/DKxTG95tWSzxWG1n9MnGgEMn4kaQKSrpnTm15k8CRPIpS1lsg4ZY/+pUJBC0XKLReQ6UXpOCryG9Hmuufg8r460gm6FiXmnUWpZCMIaU451k5/T41YJWvcHqLjvcSkBw+1V45x39swoROsouOhMm+DzQjVcd2Joq814cQZTg/3rL9pFKiMut71WinpTRogvNDgi+DPz2NREQKOUpeHe1u/MwcdQHHxn4DW2xp/x9pNQk53JDP4EhdcH6k9hT83DUcAq3o8ZFDw==,iv:w5OZN0GZZcfam1EZZFPmodhHLtuM7DJHBcYTbHL5YkA=,tag:SJMcHsd2gMWPgjIZLfhLmw==,type:str] - server: ENC[AES256_GCM,data:6syyGxOJVVKJuafONJLvDGcxjsNZamwpM3nlai/XzjwEVCcL2AU0aotTITq7dRxq3UzTFDFfHlIOoOttDCLM/2ML1hDksvKXv0XVDJW7TA==,iv:8JkMVUULGwPAr1ex0541ML7mtzsFWASskxCoG40d7B0=,tag:1TDOlMl1MOSuA2ixUCPvMg==,type:str] - name: ENC[AES256_GCM,data:Tn/6Zhy+CsRngO9/k0uLYKuldAHl8Bk=,iv:3wdHmttch2xna41q5U77rMURb6YVjcfpruoCJBJ0bck=,tag:DI09Rph0WUHzK/Ti4NWgqg==,type:str] + certificate-authority-data: ENC[AES256_GCM,data:n4yDTdl1eD2JS8R/gesmA+A9y4tVkZJcHerXlnv9qMRxhY9G9OEwJWhzfoFcf9StjfutTDu34JxkLXkrJETLF/uuvybquas7VEYoG8+8oOrMlfTHa9rrD47VrxOTWBfOd7uBPJSz0A8IuXsI0gNdvb2UlM6HCdOvjF0ldqZudkzFc5R+gkThZqIPZqJGiLlF7s8t0t3mEYnKaKgaYUEuJKGjd9bdrzC944UuVa+bH5XpOG9L1yd1UOxtIucwc5EWUxpRbozSouGOxoBC7ovaGG2WD7KMaSg2Z0GELl6a354K4pCi8hX13VBLuY8Zup5tghLBBr2ADRFdr5J3Q51Fxs/Kr7pQIAs8D/D8zYY3WQRkkeT9OTRi60e3BeYid6s5qbMCk3yK6lyoOEealJy5/i8A5pWuAeqimqMrFpxiUVLHfZL9L7CZIVaQt+WZP8ywQ8OTYhvHp1JrUZvIkerUnzYt/c0b6exBn2E/F4maMTV63KeZUdPINKpplYYRhfeErzvv6cd4BWRMBMFwQOrFD0qtF1IAGpTfTdbB2LNx3jexzMVeJBUmeO1Pizx3VK38YvwLOumbBVwVzlLTtL1p0sr1lLp6qxAXxs6hp3rpMXpAY1rkj2GeYUzWotZhrJhXcUHdqJFl7AIO82AfCZD3q2HXVYwRGPQzTzKaWtUq89ku2ju/eIulwQiL9ktlWjv+12u8Icb1wKlOwO9iwant3qFVjsOrH/qovDx2dJFiANq7ttXOVvmdmmd7Jfp9LUAkyiqo7RmcVDv7bA0jCLRyopiujpFRKW9RjQ8TXtieLmQlmwjbQl08pzr0iApm6K8ldJQLqH4viWEdFnMSSmIB3WtJsJkTV1Srp/fivdhGAgVhTak7TuE9lnl6wAmcDW+A9XewuKCc6b4nm5OqFGscW/Gx0ZvsdgjOlxyr1cCbGj+opi6fgu03oZzq70i4vr4D0hIuXcFW5aC7Rqmg3xz4o2ZZSJ0hlW5I15Lly80otZ9Z0mNIk2koWq4Z1pxN83XhHszhjDiMUePf9Ey71egLKxSMgEJfd21QywXQZSjR9g7LsH6aY9AcNKB2VZcs82bQLakS/FLpXrj+O8ZDx/gnuQQOzaEkweIbSrs4a/cQ+A8DysKHDt1+ibS6wpqYVktwuJ/UgyAd3uwx88YqgRFtV/UDqfb+kpj62riYCyGwk8MIpaB4pyq3G5XxId1Bl4Xf2Or88r2WyKG4/0RROru/8/MFLCNVS2t/TNArdzoFYM0GP2ORP5SZBc+yp10CUtHGN8esAdXvEMvIUy1lSauH3e9nXrGy6MbcMIoy/dYdKah0gkfeLA/yTZFja3vLdHX06I3PL4LhwvYZLKIeLVufIxQM4dA1odchM3Xwdd0MPvUMyRwLEDcLTU2Vzh1ePnGEZfdSpbXqe2UqaHQ23H+0WiAZ1ziNhxFJ9AqJDffoXqhmhW0RVeIV3WFcpfWPFck/kWtHq+5kGWkD3on107HlRnnxr8FTDwRmmARi5rGBRCsxa6S0nC+R5mIVk7Zb3UQl2ihuFG0dpVDCNW5D+xcAxfqfj7SC8VFkXMZ8FidQ79ybymQxIs0TFIaHR7nfpdnlbVX3ww3iMmBNr29OofcdG0xA14cn/Zg21W87m8UNCI5XrCD2y7QUVWkDVsmLfcgulC002sdWKBSejF/wWnJvo4jDCrhtbEv7nulgnc6JEkOXJWkoJTAPKzn6JDgIdzaD+Pttc6GgT9tjYKktPwqyc4vu3UddvwPr+xryiVkppVEnC/dFe5YLQgcF2DNfl/QCRc0vkP4TVCxebDzTp6m5gOqUsiA56pbrt/6sxpvFnEHC3Almr9/S6WeZ1cy0ZVPZG6UwtMaqoPr74ifZM1hiOsya5WdPNss66i2YvwqidUg/ows1/WGHW2/lgwXFfowcpskMCQ==,iv:zx3p5elDZqZTLRDJ6o1ErzF/3V+MSSjSsnERXAAkzw4=,tag:pR1xurZ/FwpH0I9NFzHoBg==,type:str] + server: ENC[AES256_GCM,data:LjzyspIiCGHCzTtLBLIXMD19cKbvygWBePlyFma2rzNvXlzcSdwGI4/WWz8MBJIsGhaTGALGRX2thDBXhzlBWIXWUpvYYntignbRVkziDQ==,iv:t5P2kKIx2WbI4IbQ25hoeIBAwYV7zTG5IkxVWhpzAiQ=,tag:97p0meCGUsEayhYhfzpDfg==,type:str] + name: ENC[AES256_GCM,data:m/BXWox6HSwgZCfzWbVEn/ofhectLYc=,iv:EFwTN64w0HtaATtccTf2jbqEJzbaaJEKqwVhFXAX8B0=,tag:M716LDPkFLFHXE1U/zUlxA==,type:str] contexts: - context: - cluster: ENC[AES256_GCM,data:wPo4tztl6bOkhVLd54hwn9WSj0vfBk8=,iv:paJp95y/YA8NgzsrPToNruQmGQOeeMOG38i1aj7OJfQ=,tag:HoeiurynOpqfljXY2opjLg==,type:str] - user: ENC[AES256_GCM,data:7iPDh5EKQ8LHaztvAIzGMfVy0wQF6yE=,iv:urVMXazL+ZYn1GKGEwTI/yhMPyEVa8VZOUkjFb+eojE=,tag:ov1rFT7/IHvAF5J5f11fng==,type:str] - name: ENC[AES256_GCM,data:MOCzSvUViwVZjFHAjMKtHPC0diuFxFw=,iv:CEIHGCt+/IWpXH+6eeHsckKne3c8cnpa6CQ31Es+OSM=,tag:MoREm99ZyOlrBpi7xI3AQg==,type:str] -current-context: ENC[AES256_GCM,data:+XE07HU+QDfHqJ0o2LbY+D3aN3o0XQU=,iv:mbraHPQfBopbpebbzmizIHNiJbPUPZyuQDVDyIYTxuY=,tag:4ab/QvwdZdbJVv9PErXZ3g==,type:str] -kind: ENC[AES256_GCM,data:r/FzqNYQ,iv:FOogwzFq4fZd/KRfM5z52cl5tW3BOodobU5rAbxH8Lw=,tag:OnKZJPEh9AShHjyvaD3NYA==,type:str] + cluster: ENC[AES256_GCM,data:8ZoXepVShI+VJCwsyXQWsW6jDlrWFnQ=,iv:CWz4gPSdsmC1YSSUyaxeFl3kwAH8FlNji+4wPv8IpIM=,tag:IZnk69WKJm3TE/OyxhccGQ==,type:str] + user: ENC[AES256_GCM,data:WW05SXoMIvNCiWf8wyoisS4IZ+peARU=,iv:rl9CzzOJnEwOKWICxXn5TdSbJufALNtjwDgaH8BXYHc=,tag:9z5Yoy5zDH9UbzIEui7vXw==,type:str] + name: ENC[AES256_GCM,data:SwJGWxvcjagEThk196jML7l+AKP0KF0=,iv:j7j7hcdCq2N/h82nHasjX/cU8naNTtQxMvb19xjz2cY=,tag:R/f+lHVe80JGXXgLOFo89A==,type:str] +current-context: ENC[AES256_GCM,data:53+XhFN7WM5hJZ25YXdJzHzSosb5tys=,iv:MfY1K7R35s/gjRAkxkyrKUmuWjKPSyLl+OQOF/ZLQyQ=,tag:hbWJhcLDK62KM+63Xzu9Yg==,type:str] +kind: ENC[AES256_GCM,data:VkFwX3PV,iv:5p7fQVOi63lXt6RLlRlLt4s48feGCG5pc/t8UZbj75E=,tag:G2ZWqupWmWKxvGzlnybIag==,type:str] preferences: {} users: - - name: ENC[AES256_GCM,data:eR7e+4X2GrnyCLTPMaYesDyjHcPPAns=,iv:gYg8ETXenwsGSRjOGwsTTyw5QllkXQ2B8u6Vdgrfvog=,tag:DQMW/N7cAyeg8yzwSzajyw==,type:str] + - name: ENC[AES256_GCM,data:IzJTzad6TXkbuDuEQHfUHMbS3IexYo4=,iv:igfNhn+1x9FLLYvum2pN4UWu9cPKAvR6l9Az1oieILA=,tag:LmQ9ArMAsBjdPG6lOOZQqA==,type:str] user: - client-certificate-data: ENC[AES256_GCM,data:J46C7tCbr4LiHkA4K1KV8hWOPGejCzHtAJF1KgH17lTREhD8lw4qDGoKJ6/KWUzbNreaMZv26UNde0s5PwOUt3FW50jbm4bKE3jypdAelHaFUnCkP9Eqg1/bGHV6CoflUWAiedD/4lNV9bfyApDi/h2sqsEYJSyHYae8cbNhCo/1ha4ppGbPZE1TxVv3OyVpV3PJBtrAcesx5GX5HpU0Ale8oPZqPUL6zcuqLTNTerUOKdNeoSpXlpR929xwV7bSMLD0Cbst68b0u/wM266CFuwLk99L/u6Lc8lg2Gt28IR9jW3/tJApuAziTaRfdLWAyoIlX9kPBJo90HAVMBde8UJedXMzNYBZVb0zIwWu9y0/uHIlQLAi1S/OFUpfnxHugBcU2+CBBcA9Odd7vWRNcMuk9N6F034ePBj5F6Mlekvf1haNwTFTJrroLILSSmFni4a8dg3Nq3oCx6ORdIOVVYdCJBGvUa2TKrje3nQN4YTTlDYg84gIOrwNu9rvvrTBvvgAYESeoX07/6bDe1+HYAIAP9DmsZPOULvAkH6mI6g6ifNoJHGTN+YgPZyIIGMhb8TPsj6AihQl3BRTDJx2cHt1QVbDOswE2Tq+5lBen+2Zs1BZLRq4IZjTQVmkN4IDjUmqPzPnWcpqIsBgaHel4vdbXq22xr205GlqNm+cC45h5xYiUkHJNakBaIGGaoJG3znxJ4vTNshUq1s1l55COdNcK3mGfQlunbZRG4aNMl8kXT4p2WOEx3hrOzNarcwuToa6yWYoiiwhfp5CfIIg8LESsUPHtZu7TqOxdQS6B2YHEvQjzose8SAuM6IAmFXH+1Y3xssH86jPEH75lfIWrMOqc9n3sJgWHKAzcQdMDu4B30GBx2PJ4EVIjTNsFZ35GQiioXgYYkK6hQmCTWxwLMtyKI8rI63wzkhbL/zt+h1legFB9QBUqZ+GBaoSMbPFJQjc4dPbvmO/U37g0oWJRl2RTN2f48bFrsAM24CeB/p/o15eGpNnD5ptL2RQ8T2pqOBcDQJLgNPvFJyPnOG/1jOWv9OU0PZjJA6eabI+YB1fw4xkBUXspMM0WnYwI/DCTFyy3UWT5Y9MaXxspp5rHFr978W5kXRtCXRfkSJAm4lcxcMfkc/7XcuK2wMXlGXYuDO8EiM0H8xJ6cYh/IXnUz76f86IMbadlcqeDOFHfKFdgHkW+44J4+QncoyZVRlCdzqAz2luyAF0gkzLQGMkwLelKSmlx7nwUBfbmsFAm+jy3UgkN8Df3+9W+qMz8Op8CjDDXgldjN3W9uojFdRgLJTwuUrkoPiT9rZbXn4XRyNiIe3IvFvPYpHrrh0VVIi0pwBuOkMOLWpWOCAWQ99etKkm07H6l8x5HDZxFNRye3dtP2bz2YATV1Q5EmPuvAA+RI6lyUOOCP6F3IXqtS40/ML65UOGDzp7xf3BeR/2l7EnSA9R24Rw/1UmBBN+lAw4TGckk+y5ixHALlpBTRqhSt0tF6cTgbO12dM1EVfVyjHBCZaxe1c8WyE8XUB9jsHz+cUtLAdcO0I62x9XGrq6V2Jc+HNkGSkbUdEA1Oq853gDH6BxZuba94rAWTHoUN0OBcRBmFBsUL06JE5XWFZb/AfNi3enPMjBbFMKr02ctRGE8czhu8S8fo+Pkjv8YNCjDW/2gVapggHjyyiaLI8SewH0606ZPq9HTjxxONrBNeveePc/YY/5HnJE+lKj6oG/Ixk7J/tq6SOkz8WeJzBsgJ+42AAUPwpsg9hGVIga81C8ClNgZaX1ju4Jz587yL3sLaKaGl7KGEs9RGjLOx4UA6Myk8URwbw2gBaOiBPK7nb/x5x04jxWF1tgM6eYWJQ9Gi1QKt45LgYPdzYyCoOumcJl+NjWua2Q2U7x/xJcQbn5I31FH2YmSwfFNLfWPcrNSG9jLkB3J9uBT/k61/pZGmWuBKcf0GWX7fNpkH4CdpuYOhi9lfpI4lIARMJDqp/cRwQu5l+E7hWs3floD0qr2l1fCMWxxApjtwBV93NUe95XTbnKgp+Wh7gWEBKtSDmB6CXZOmop8OHPpfwS,iv:efCIgmdnzcW/0vXjMirln8t+NFRmboaVaRxme6H2XaM=,tag:sxMpJUlNNGk1fK+W/KtEcA==,type:str] - client-key-data: ENC[AES256_GCM,data:XljhSGl4WHMv7hpNDRAtu9kdUKH1rePyRwceMbicditzDq5K61pwbvncYWX6hYJ1x+9scvVoRuk3H22ai52xZty7FJb0DiWu6siNLNjTmQCWoGFneBprQRsy+SI7EWgVZoF7XIbVYC6rP8by9U3v5YDMQPMVTSdoxfuk7J0FE65SzCID5uns0Ia2VJkQP8cGbXZq4Us4lzA0L0R6hV4ak6Xt7hIPwi/Fd1zWAG4XmTXtZgh07iKORbDpkWN6UpgpirsgRTxkr9sRh3MVQjVBWr0rz35nvRjn4eA/gxZwCcvTyej98nN6AzFPdIm+2aTTsj5+1nLt4Pny9sDDj9cRPyz/OlKd5E95Yt1zfePaoUlQaH59aL4f0gleSmLlCc+xDPkv7N7B5bdyMGr+ROiDA26HmPxF/2Z65pFNz4sHXVLfCpFggRAkhO/N4As1Zt9RFkyCz/Iu1pBVom9TEPxjfsSrqOS/rOcLgepqE60wdFsFVIdwJUo7xPa2IWUFo0Duwe05yDphijYhd5ieLIr6V3qL5m3LQfDZhYcY8kE93Q8Cx1UsYWv0f+Uy96eDDxZBvhZvnMjGm0i9rV8hcxGj9O5YR2Kq+laPgRKKObh40qwTmlCkZKkp4dmTvQnZ4ZYFFEFcKgARN5hT1Ha1VbxwBAohHocce+cjSQg+J49QYNZe1Rh3ZuYvpFL52f00a0YK+93xGyodL2PxCqMaVMVmHhwU65ICtgrLgtAELSTuNOGT9FEqAURoJrc7Oc0YxP/kzl7YHSdjH/tTXQyHVVXfnY0WOLvahz09xyNN1ip2c5FWKVpAiJ+424lASlsvfQV6hYTB7VXsXT8AGofz7oGCuGMHqwcQZa83QAI8sr493SO3rDWZ0HqpvCEtI6vsI1kfp7l2ZyrMUs/C1x24RCILyhbYKqA0bB9Nqyk27d7vUZmhaSnJ8PiU5hZWZWv756eAM2YPfaJMG0FBpQfOFn0zXh+6Zn4v01X7k9voA9V9m383eYRq6Mejkf6Xl5gX6JUVGpIH9Nv5Rrr6LK67h7kqi6B+4AB9yhe1CzSTuQRy3IV8eTyZuWvGnsC4rNQumMK6EfjqNWD/4g5JalmucJymWVXxUpNvrKgYebVkcazglI1AiPlxiAccYpwcKdHpnogFZnlxpVVYKGNjCy1BrseJ2sfPJhIpJOLlAu330nrwgYhHErRhmV+rcrISVb1rKIkalExD882bFXBoZANByTieJVzs9FcbrH3/s6wXEkSgsMnRCmXQVPrx36xkNn5sVFeuxZaPAFnnSdhBhS41KBnLL09hEo87/KJNAYKIKX1KMaw3K08sdRHT2BLrRat8rSGDYd7qIaosdUg6hPevlBDixI3rsts7/dB1mxPmAPLiN2jZ+QBDMs3WVK52aae0Rt4kEkk6P9TIdLhQODjjyPl7JUapYzB42SrWDZNZL9fao2SwhI41yKawAf8QlSDFU+OFehYzO7pg/tBTRRcF+VS6E7G3/Dz2iZnC/iv+ie3PDP9AeyItp8ajTFS/rTDLpTPDQbzIZjypOSoydqBsr354Rp6ULx5Y+jF2TEeFGcbJYelN0VUjoTCcl5sOF6CsMtjnjcgTWECDzjNYekPuyjuztWoF0zlnzcVdnHwN10Gcyw8NGGTJ6HktJv3JLnwCEoSKO+VZ9/H8vemqrHVquLBpPFHWN7c4mE1R9dNv9otq7YE/eVDkYKxHCqfZ/P1MAfelGz+4gRc0b72iwlw/qweroBmeiGvX/sZUOBdvRvZoV1q1NOUPmlhlviZgmU4nNGKC5zfOv5kIShhWd8sTQCggZhOvuhOkjgUN6ODTMkAaOf9ymK49uQaCopphE/csOAmkLcSlsmOUrqyk8DpjNUoSQ3qth4o0UcHyC6He/V63LKsIlGwYDP3Y6wW/0GlqqrzkJ3q4IyBY4FF+MQVsNoZd1uKZH1emYF6rptt9PLAlKFW80JtES0Fl1Ls/vsvT48PmGPzHLGRnP7qUxEYaD+0bNHIy2OD8Io7646okuPs5sQcE6V9ngDU0K9ypD5GuAXTRN4Xt4pKBiTb9z2HkN92hzYMh+5Sclq50+0JD3/8NLC/PIxpXwsW3ZRo87I8DNwZAJnjd7G1nXglJlpxooLU07nLOfOYh7zcEc7ZYjbXsqq8p0yxdhpo5zPjLX/rRGv9guadKIp/nIR/N2BX9jXe/kr885m51njGd7fRxlH9ZRqkEYHEHou8f4OKCekcCvoL2AaamtWnASc5J8mYHIzVsX3mYbecrFqB780uuaoCLJC6RV15J8fi62ckh1KM41XuCDr9VW582FQl83+MBHzDlAocvYt1GTyZjW7j+N2/yh5YUyHwEgBs1w7zrBjPlOKW5S5WFJZK0rnDO7cEDwx7smX5yA2AYqfZpMQbfnIRlOBcqCcc5QVHXg7JPeY0uTGSt4nPlp0bHVyIiUjqdznyjGS7dIy6CQu97Rxe6910H4hRpsTd8frzNCO8Jk0Xw2klVAXgxtxAINK/o/2G2I0NAZVOlo9nMYMozcLAta3lcxxsNRk7AiSRLjEevsaIloLdacbtveMiIyAJBgBd7Pl1iFwa3onk21DypgmvPkwA4YJyO0Cu5T3MGsEUepnDAGUOqr65sqB25SMHOeIR1c1zUJmH5OU/tdS7nTAFqZmF49HgMhdWfHvP0XhwauI8esLZrH0XhTsaBsiuJsgVLZEODKmrROlx3CuLWSgtBzW2GrsXPctftAWhjLtv2vu9DvtU7MpRBwn16C59imgLnBVShm1S0Dr4c2UpS60pDGIecV+qjyKPqHwUYq8YdgF4OlMebk+5ejqqUiqShmWvzlbDcJLE1MIOB9mjecdlTjJg4SUN6rcpoTlief9YjKenkV2Q1NGjr/qVflqV1w+3YbI7G04+fig9d0OXuoDlSeMgsCVdKXAVaCKCGtMUp0xLzLxXCrBCtX13diLb9TQ2ZRk2E36KP8sei+jKeePX/ATZxQ6M=,iv:Yd0SBcdQbcA6AwEAr7rLYfX5XTPyITVVIJ/Lbexq6kc=,tag:XmTFiGg5tesV+jw5VTCLxg==,type:str] + client-certificate-data: ENC[AES256_GCM,data:7syq5NPzYsNNE3AnFlPncNqtQ8bliZQDvXUPTzzDFJcL03nrk0G7SBUKYvIjtXoGMzIje4HTblbsiL+IrYUcr+KO3rAOPYs8R6sjQfWtVlcMG7V9LR4mXjGSQ09Q6fFMbCOu8HUlPB45EHnBH5mm1wxtM543x4VAEgy3cjTBD1/mHpy9Cu5Qu4otoz3arzWeFlceOqL6avNrAMVp2urKOa1uVOS/fgYalLC1im2MtPfmFm4S4jG+zel505dU5Gjb4xIwMMlS5yNponZAP+pp+1nNZvUuq1DTS+NWKoEpSFgaEYbHsLFltt9nyW39+Csw+D/SVhzAfRip9yDzp197EbrV5W/lhFXFz0ZnyAPYS2SptMBRuqX7Viw77SlLbhVujDUU9phiFx+BCD1wVxhzoRMaP7zCdvh/yYsQgGrqByLsblM6zHu4M4ImARHyOWR+MdUDYXf6PnZ1SFg8h6Yhsg1dTPpoRnPqYrSiLDpdYRr6I186RNeXgvVK5xopip5eK/k0dptZWFLmJfaYxhQ23MhRNL5j3rM05eh5bFhDxU4T+ic8/dhVze1FoWoZJnYFFsjjLF8MOAeZ2DlGu5UWzCyfh0+GSPjkxo+bahrM8gyg8d9kxwwKJytMgk8yF8dVmk1kswi+HxF/oHo8NvZMjCLrNiEy4YO/DSLfSzVObOMy394Iny8+94T3v5ZFpWztUTGI8rgqpgv8ZFQw1ufNE/LRaae5yf0VOwXPXxFqDJi31mpm+xOohhd0qv4r89aQzPkwSRf23iNw2l9hJymRiPAJpCjZMNt9Uj9Ok4mQVbt3dK9UWmCwa1G8S3E2IUPxOiElDtBoDwNUOc4M7HbRZE5U0AW9Q6RMDgJFGDwqCwi3NfZpMfzYS/y8377YghRsrStfO/9DJzXEmeRB1PVaXgUvNykfcq2yw6c05/rpH1E/PNgvg3WK3e82AniBN+nIwxYEwgGPo5WlJjWmz8kOZRoTLWKfphr8eZhSFratq3rqtPr1Z5g71USZI6NUQbJRBM9BLz6VdOMmwEK7NcOh11ibVSgiAc4u1wT22sqBqZWnqK/bsgXWO1tP3oPyVRlFRD8vjUD3AR53IFpeJMLxeJCBldjIh+9EwJqgqNpozt3v5jkqgGag/7vYcOi2SOouvAAG9w6v9yKVftPIkpkUxslC8Fa9e0uNJuM2Bs9+2iPs3egnQe8hu0QhW+v5I/Ew127AMLEvwakerkriW5vr6wuDQg7gFD21cZEoUGAR2ZTLOmyXfez8n1TRLtIo9krNs/+BODj/dsTx3DY9KJGAhC/UtFe+FUCl4HSgJ+UgO5MOgxulVWG0GTuGcGztbQ99wSYJYXnJ45psVljqh0GMCaMuuB+xTYYEZMn92bngZfRBRBaL8C63Nvswg94T95zAwEKwPEr8Nbz3sxpqFUp5Zkp4aYOp+HxqgWZ/FvL7e/OavB0AUC3nL98A5RUDIq8B1TyK7Dzjn8rS7VJk1yHrln3KgJRe2dBiiVgEVyWcB16hTKXA6YZnobZOls2ccXTvHA13z1EKrAPDagZGOsAmSkrvVRTkuCgoC9lGokG849m1NM9aap+sUXcl+Yemf12rZnR2ZBHRE6m1A1xMbLwNkld8BE3qM1R94ZLgiTV2tT17XvXkP5PL5sSWm2YeuLx1OI+Kza+GbYdOanXxw+yjHpV+XJhaVuM6fTPwbZJ+Gc9ve4l4L4xFIzR8zhel8LOVRJL54PFjihKAH6g+SLyfwWhAbjYdEhx50Pg9tRTM/zkC/lvO5v4ZpGDhyJQXIto5+UGO/QZrPkmJLexC2tACMSC0BHVOSO6Mt4OVYjPVvfqHz49fQ7D9Ihkngt8LbxRG9QZy6JNkuO9ao961gC3LbjtKMkyfHHUcu4vdXso8MgzBMZWfqOD8E8IYOpHgd8d0tpmVe7fAM9xus/2K23O7QDadA3R48RqzK8CRuTvsl6LcW78srOUW2XHz27YUUbBtpwVe3NzYRu0yD0ZToGd7oyL5VazaZQwtauxdsq6WSLrET5I8xUaYxTtwYJFH1mclbHUgAeO+KbdLkHcH,iv:HGOaid3oUh1kWKKzuL+FGeVRpgsI+OQdrvzZVa7dbeI=,tag:jSAxEKErekVhm5I+Aggb2Q==,type:str] + client-key-data: ENC[AES256_GCM,data:rOTMcUUvgx5GcQWzPmMiy3K41AKS+3ydsv3sTnIhSH/xf/96m4E0fZTgBK9LzenelcQZPHubWWiyKg8+Q2ey7RsSPJNUTkBChkl/RAzCh23tSIiy6yqwyaAeE2REvjOnXYQ4X1mfKDhootc77dg8bbI7PMGY8vGAVPkse/ML04NYZAimT0tH5cKzB0dzTtHZRLKYgJyF85WzfHb9KulJBrgLk6mZzNDMdA6/aqKTx7uVUEU0ztaYyMXFIHwVx4c7rxv3b42xkPKzEQRU1kg0O0+2QGrAqfIPwJ8ds9VzUlcNVu17VZQlwOoS2xsWcz5Ohm3bhWcdVUvQwmm+L+2QzcPmyLRhmPudjmfUe7cf9eXxZQ0iln1/5eOe5WTQWC5Eek8u4LNFCTlIomj61imOpjEjpDFb7b/D222PjPT1s+ItzNH3ppcqdsDA1A1Vw0vKIpZMZgPf3QsomY6xvhPk5bsS1UFVNoVeRuzAR5apLfKUf88Ebc+La1Po+VGBxyl/InOUUW3tqg5orU62pC4109oS7/133ScW00997HFA7PFn1Xjb3Ff1opB6FAXygZSl2AvflhbCjWtx/rW80oJzTFvSM0OxML85Ebi1hMnA+JVCdju1ey874SRPZHr8YPlfouz7K4xAtlE2oXGtAFFdJL7Qlp8Us3VMz9JMv1O7zxq3mPC4lElgo+kmy3LAmsTSsZTu5qg6gzZm1276e1+6hOFPbXWwWsIctvETdAVAjVmp35fGwMxJN+N4PdgxTzJwUeN09sfp1z8BXskvlb1PfhjSRsOVlrbcj6lmlYzXiJYUqQ48YUYmK3/IIpSENaK6qvL2V2//LdOVfdDQ+MILeXInbnkNmk4INUs3SVefHyagIaxud9uQC1SM25Wz/jYh8r8n5l4XxF8K+TejKmQ1P3KoqtPbRBTk64gZFzS3yn2kEXUrIrT2CWdQsd277eIq6PTVwjDbbin50cZwqd2tqvzjhp16IMhYRTXyRf3ctp5K3gHfAIA/A/rPiMtKz3HzGmrXTNEOzQ1uKIf3GHRuzPsy+3yUd4dX+InszHhqMC1R6tZim8/ko9jk+4GydPfnaKfuocZf2EDU5FyHUHIVb0xIPCjGlUBl3xdRdnyALTbhsWl0Av1lbldtTFshrmyBgK6EAHysPyX/B1Pkg5/RvVg1sJwFapNpuQxdPh6tG7u7YIZiqY9KkFn51tJ68LX7uYqFJ0Fy5jRQBfiwiNqtHlPqbZr7EWD/8wkZJzmaybzykmbw2W4myB6N8jsUUIfrDkjYBpAw56380PlQLoPbAFvXtLG3FLhZ49IFu1jw5v9Om7LVYNgVMzIWA3Z8/N2ncodF5+Okb3UKNIhBCrpPSWe6jxnpFjC0CuSjarLygNOeYDPz5WeZoVS2FzKOU/UjWPvK1B9fVLas8fhQ0hT/WE0RXsO/SLnkJPXxS6F6tSktiB8Lypw9iD+VlleAOYaqhFqYKXksy5L6QZkpLPZuKInx7mCw7R+bOQS0eV9H0EUWTrCYprkO5vjFfYq5OeVATqbns/MaymDhbAvz2Lv4S2EjERbheNeRSSiKYe/m5GXF2x86mnnj9bdIc3LEpOF1/MNZe/V0uzs1jgSobj/yCU3PIXSbRn1t5VVam9IYZ0xz8hLjnyFooKEvW6ildM/XIUdWnQ2wWvPNuQBGmgI+uyI/KwhxC/k/bi4vCwFNgHoqCLjkWupYbWCm8pmXiypLETcb8lQKppLgee53GGlmxmvue5VufYJkzNOrykw3AY/ppDvwvVuNm7hZOsiMNGJ9lLLBntrk6S7VZRtCWxEOd/L2d3GffZrIryOKglKjECHkqWRj82RflaRif3TRoU2K+XDXtQFGKjp3jNHKxUzw6sesUsReIcmT1lKrrL3nATVNrS4oo3olaRWP4yYhO8plKzIS1RC5XVdHg0syJsEuIDalaEYVQfcjtiWz0zVfne4f88z789ZNsLbzYOKMBQVWTzRLITKQvKsnTVt3cCL8B55UIsP0mbG9PKKg+W8+P9RPyi6EBDqNwqNQWw/ichF38euJkQLtJ24yfC8tGKt1HTTH64yNayqoUTCrdFbeDhcFHXc7TmGieYaOIKvHgLjz8Gx1JqSI8Mq/MLjIeSkGp9FPSMEK5BQfyGFKtxIrkNLY88KZBnEmxmskuTlsjPmUlOyvDhvm8s3yRufVdGiL1YOx0l+MJ5wZNzgTYuRpHPGRcqX7iM157W9SCVHQMUJlSiV5fR7P90ok9NVcHswaFSkiUVXogmvuh5bGROQvC3g1n8ZonCuPj3dtRQrhhVlOCkyK7D/9v4AKtrxAz2f0/MgTWk/M8ItqC7/iH5qWcGwwGxnMr4QKOsXxYh41ImsSOgeA/Wj8BB0yCC9ivknlCzDxzMcYuCQY/YofC24qPnurlzov89oC4C29Q/A3N0V09DccYePdVdvvmfwTSKOhuSJwPoaFx2NnKi8/mFfNiFYNzZTgO4EVbRz2AqD0d5fBCSYeTup1yMHS+DddRaUYvm+SeVkYhg96xtdPiqWI+fvRTys4NcVUf/9DDU8mD5AOhGQJ0D9ZP9EPJJXRsSClHL5WQZeLRBX30wxpdZmi8qspmuGTBL60nwDm+esSxFqdwa4ERMuu12IunJLcHRUX2j7/eKYOj7xhHIIajLeNzHmItZ3ZcaWaB66o1kA1D3tgyMO4EHU0v7NofOY0ESp1+3jh1WpuVETfgEAU+8PlwiPufkbpps1KMaKVw2ZTu9mQ340io4zLQUNKsO1skJajxnRKqIOtf3jE76WYjww+2OmTrc1JU9b7nUXnMZstXJCroWDfEQYjXhTvKzwna5brUdjet5rNgqc5bhln6fhT2WFAl2U598X16L3w2hjW1P6P+IhG/xwYmkzUQ3rd85BMkWigyoCXi0+ARy5/68pXdbVppxaXelFfXiNAY3eZhX3SbQx49aunmoiNine1tF8TLsOW0eViqdgGCJVZVQ==,iv:2ikCP0ls3Fn3yMTU/L/ax3bSxteYxqk5FUocl6QAkzM=,tag:hvsG8Rqdx0sZ73+hN+MQUw==,type:str] sops: kms: [] gcp_kms: - resource_id: projects/two-eye-two-see/locations/global/keyRings/sops-keys/cryptoKeys/similar-hubs - created_at: "2021-05-10T17:35:08Z" - enc: CiQA4OM7eLM6u6MlzoUjfIYlN5H/wC7Dr3n4xmiRNdOO6TKOd+YSSQBy9hCYancmxxKoD4g2+zDN2GU4aKKSHq2BwUiYNCBwLs8U5BV2yhzKSnjY5cEbM+9YMV2holXPYrthBCtola5bR69V0nORC9g= + created_at: "2021-05-11T17:42:53Z" + enc: CiQA4OM7eAi+MGH3STjMg9uCkPhlJLinqBkbsd50JHoUbFJ1ya0SSQBy9hCYkSPMrZmzLIuLrzSfa8SqY11Xn4nGQJq7gBw6qArdSOOJe2iIoGyeBN4XftmGJQT2EB+5UeNQk0nO4Dw3FMehcacKCuA= azure_kv: [] hc_vault: [] age: [] - lastmodified: "2021-05-10T17:35:10Z" - mac: ENC[AES256_GCM,data:nIIqCgBY7lmTp0k5If5BXHFeeRUJ0OdkU66Z7/bZOc4EElu9JbXSrNPfjcllk6pzXB6XGOIU2RuIhDTcl/DZSvCAgYuFkorQLYZ2js/2yNxSZY2sYcRnQB+C/R7PgwpEyObpTQ2faRmcVEYPzGKTBml9GyZo1RfNllxOuUeykHM=,iv:oxKFojrFq7/AiX8ytiiwq/1lA3IuHg3aOh/EBfknsUo=,tag:KX7fhH78UOvzSPm/rjqXBQ==,type:str] + lastmodified: "2021-05-11T17:42:55Z" + mac: ENC[AES256_GCM,data:4IPo4SH3mjiS5iE9BthCDp31d5OB4YA7HAjHG21O57e5A3f7J/bTt4acA0fYmf6FiCtb6UxKwyIRxsRfgsjVfkBY0VgLMdqNDijn6nqTt1roObWwoXmI7KkpqlWPTuUz9doTwPLje35sY+kBaLuhv/QCPabkul4n9S7trb1wYQQ=,iv:khILySofUGqHZMIT7zRilnwJLFCpSAfY0oZcJzrgb40=,tag:l+UZr8L+i54YdKA9vh2EQg==,type:str] pgp: [] unencrypted_suffix: _unencrypted version: 3.7.1 From b3dd9e209d339d1b06d2a6ba9668f80ffc70de52 Mon Sep 17 00:00:00 2001 From: YuviPanda Date: Wed, 12 May 2021 00:08:49 +0530 Subject: [PATCH 06/12] Add hacky script to create EFS for a kops cluster --- kops/setup-efs.py | 63 +++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 63 insertions(+) create mode 100644 kops/setup-efs.py diff --git a/kops/setup-efs.py b/kops/setup-efs.py new file mode 100644 index 0000000000..d50e39dd92 --- /dev/null +++ b/kops/setup-efs.py @@ -0,0 +1,63 @@ +#!/usr/bin/env python3 +""" +Hacky script to automate EFS setup for a given kops cluster + +1. Create an EFS file system +2. Create a mount target with correct security groups & subnets for + a given kops cluster +""" + +import sys +import boto3 +import secrets +import time + +def find_subnets(cluster_name, region): + """ + Find all the subnets created by kops for this cluster + """ + ec2 = boto3.client('ec2', region_name=region) + return ec2.describe_subnets(Filters=[ + {'Name':'tag:KubernetesCluster', 'Values': [cluster_name]} + ])['Subnets'] + +def find_security_groups(cluster_name, region): + """ + Find security groups of master and nodes + """ + ec2 = boto3.client('ec2', region_name=region) + return ec2.describe_security_groups(Filters=[ + {'Name':'tag:Name', 'Values': [f'{t}.{cluster_name}' for t in ['masters', 'nodes']]} + ])['SecurityGroups'] + +def create_filesystem(token, name, region): + efs = boto3.client('efs', region_name=region) + subnets = find_subnets(name, region) + security_groups = find_security_groups(name, region) + fs = efs.create_file_system( + CreationToken=token, + Encrypted=True, + Backup=True, + Tags=[ + {'Key': 'KubernetesCluster', 'Value': name} + ] + ) + while True: + resp = efs.describe_file_systems( + CreationToken=token + ) + if resp['FileSystems'][0]['LifeCycleState'] == 'available': + break + time.sleep(5) + + for subnet in subnets: + efs.create_mount_target( + FileSystemId=fs['FileSystemId'], + SubnetId=subnet['SubnetId'], + SecurityGroups=[sg['GroupId'] for sg in security_groups] + ) + print(f'setup {fs["FileSystemId"]}') + +if __name__ == '__main__': + token = secrets.token_hex(16) + create_filesystem(token, sys.argv[1], sys.argv[2]) \ No newline at end of file From e898ae8dc30e522a4b8f8d3b38caa45d8acd5ca5 Mon Sep 17 00:00:00 2001 From: YuviPanda Date: Wed, 12 May 2021 00:16:44 +0530 Subject: [PATCH 07/12] Set configBase explicitly --- kops/carbonplan.jsonnet | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/kops/carbonplan.jsonnet b/kops/carbonplan.jsonnet index 9cce3c07e4..6f6e30bac8 100644 --- a/kops/carbonplan.jsonnet +++ b/kops/carbonplan.jsonnet @@ -15,7 +15,8 @@ local data = { name: "carbonplanhub.k8s.local" }, spec+: { - configBase: "s3://2i2c-carbonplan-kops-state" + // FIXME: Not sure if this is necessary? + configBase: "s3://2i2c-carbonplan-kops-state/%s" % data.cluster.metadata.name }, _config+:: { zone: zone, From a13443ade5ebf97c405c313d4a7559a18bfa0285 Mon Sep 17 00:00:00 2001 From: YuviPanda Date: Thu, 13 May 2021 00:13:43 +0530 Subject: [PATCH 08/12] Add (encrypted) ssh key used for carbonplan cluster --- .sops.yaml | 2 ++ kops/ssh-keys/carbonplan.key | 21 +++++++++++++++++++++ kops/ssh-keys/carbonplan.key.pub | 1 + 3 files changed, 24 insertions(+) create mode 100644 kops/ssh-keys/carbonplan.key create mode 100644 kops/ssh-keys/carbonplan.key.pub diff --git a/.sops.yaml b/.sops.yaml index 87b509914b..826c010aa2 100644 --- a/.sops.yaml +++ b/.sops.yaml @@ -1,5 +1,7 @@ creation_rules: - path_regex: .*/secrets/.* gcp_kms: projects/two-eye-two-see/locations/global/keyRings/sops-keys/cryptoKeys/similar-hubs + - path_regex: .*/kops/ssh-keys/.* + gcp_kms: projects/two-eye-two-see/locations/global/keyRings/sops-keys/cryptoKeys/similar-hubs - path_regex: config/secrets.yaml$ gcp_kms: projects/two-eye-two-see/locations/global/keyRings/sops-keys/cryptoKeys/similar-hubs \ No newline at end of file diff --git a/kops/ssh-keys/carbonplan.key b/kops/ssh-keys/carbonplan.key new file mode 100644 index 0000000000..028b248991 --- /dev/null +++ b/kops/ssh-keys/carbonplan.key @@ -0,0 +1,21 @@ +{ + "data": "ENC[AES256_GCM,data:KwDNN+NY8bfmgnehZrUhpi0NA77vZfP3+Uo8fssd6m31UG4TeKFqxLS5QxouT4rmRc3rWS9YJNWLMG9/wzWSby6GPnK3IrO5zvx/PNJs5YYUc0HLpWy0VYxNsqTdytIUXEBqymzArbiJhIkrXhXK+JGLbT7WABtbgY6BI/hWAIyUvPCUKewnmCX3pEShzHVY8hyzad45p2ap6utCIKhu4WJeQkcXaHgj/yzcTWlggKqoj8LYoeTX/x6j3G0hyhhugJFq9HVWNq6OjuXSaB8eKHulIkV1RTTly6uQOi98ReztHoggLuQnJjeUe/yaqR9JGBh2ahb91iqRsknNdQcsBg2EWr+m6LYrEA2XVnb2FEulN4vGJlSY+q/3kh+j/JER37nJ79128DSrClac/6rGUJTc4d3I00jYHh6mYt4799ammN5M+SpuYxtnSpiST4Kw3CVUFNcC/W/fG8n6Td1Koo4hoo2N9RXkz6j52Ft+jXaPyPV+1JDSGTO1Hnx5EeFIx016j1Scz1gBUXRSrgsU496hNMoEBTtXorh9lCBtYhv/GbLiHPnXhO0T6fW+vgnsGNJgUKR42ZjPPyZd2XV5w6vBXuQg12DdON6G3LO11agvKk2FJkKcMBpKW/fhd3lLluHX1Wj2Kvpy5sX/sTlebyiWyCvRb0Sn9hH3fEbrLEapIDlo/9WPxooZYC1j+t+BwhdrwUKNQyb0tD6JHCQGg8HNDffKZxzceSS0YDhZrP2O7i+GL/BxBGb6nBB/Oq6OPGYmwQYqYRjmmFWD+kfxKBPkDflaDGjPDjgpzbitNULcY2j/bAGtHPOxP6LNB20vy/oJ8cYNXhYC6AuFQXrz9jvhmUpjcRLrl3NGJKLaS6sc3VulEZFCkO5riBKVHrX+ndR1qJcXn2VtiKcAKiuzvKzeZAOaPeBDw/3ja1Pc8h6CDEX8yfx/CO2TFHPH9ljAI1wbmJlpGMw0hb2djNbO2nx/S04+0bJEYVByzMo9xWhh6z/zG2KcJd8OqCbKxJgb+HjtH6wu1d7AV5D0W1rDHlRpriMuCcmYhvCKG9hVL+//iEXf0j//WjxXR9fJTa19qqBYTaGWdR+AOeS7gwSOlb4LWWutCOPAtbFPXSnnJQxG4vEauDK7RlQideKukOkp//+3y5/YiMajRxUlK9kdEe5zyJKqxFbalWO0LYfvNnIAuXYVnYwQ2rlTQRkh121dG3RIuWsxPqgFgXp6MEMwMW1AhHf0s/qy8E1yS8MzxQjkuDuXJW/qAzUNOO4+AvGR1T7YCE/c2JQSZJddkI9y4OZfTcptnd+hl88K3y8DHaoRo/0h2RhfImW9DoVKk4CHWpnkzI2SnZuCfkZt9bI/tdwsMSX7B0gTyupbcvPK6Ywe8b8m6dp/4tj2GMdjNFn0B4hHui34qlBpoDLbeKmq8GX0Yc2mcprPPAFocemA5EqKlaXcIgZYWxEWNC5ZXdZElbroiINn1GzObVFP40i2zB/pFe+MNoWGG5ZLF11hAhsHL4H9amVsi6vWs/v2FIn2R/j681IOlPG6Dfm7xybVeDpCuUPi3L/IYlGsCd7UhHi7bFZIBVkNQUjmQ5/HIPgnudJodz3YFuMBREYtnSrX0z+o7l2lsr9FWIKTPpPutO4R5GUefnXK7UtwrXG3GCs7J71TQuNiwMxcj8V8Ex3xal9I2qdJM+/O/Fks0fksxjYm4kt8pEnG7yOnWvB71JoRy4xAfBeK+x2ZoVa9bOosszGlICMc9APcCFfPjFgjOyg8ucHb9dgPA2YuI+it0eS/xasJf7B42SDXydlWlklR8KbY/ZCYHO5/Ic6Z7MCmPFoGtFq0YTplp2/IrMDHyy8aK8ARg7E4KJgSnylHYn3PhNvZlywsaN2Eq1UnCMwbc9RlQxpzb9LhluUNVUtrRCmJUZwNplaUo1NqIQjOKbhurhSUMa5pIP/jFUXl1WTIC1mskQeCUN7G53OQRbZlR+NwVwW5USsBsREuy//AH+euJS+ZLayNkBmRQ4yEF6gNlD7+QBMOPvC75i7dX2LFAlKRe4cg0E4outxCAwwfJ6qLMJNhuv4d0DNoqn5As7sRcklUlAquvCEsTzG6MN1zPATdovbUB9y+0NHt9B7C+uXpgJrvnObb3CQkrk8T4Jom/hCOSjz6D+LqvxgBaaOvhv9xCLteO8r0sRszhrz1MpHjpHcksmdvMD3BSiVYle7u2lNncMJer+5yI4yyPhBrHpjWh93qmaCWcMUGJHc8KDtX3gJy4PrascVGKTtrTsQO5B+F9xh0J571Jz68NOGTDZUn/NY7i7pJfGQ0WJXiTxjBSebpEZFTGQGBuGZMfsrSe14nbdY1Rt8t842tvvrTL2wYVl0HwaAoJ07Oj0lBjk25mp56ZDIIYszJmeBh5ZyctjrtI7LgpmvREtS8PnF5sAKgdfYVya3SqephizaBdX/xX7A2/QRPRr21g3ZLttBzKF9tgqwbzw04VLCGeNDsKXripDNGXOAsb3dYOc5MxraZPqwEMpTksMtf7laBPQaxKVxeRndMrYq/e9ihjTo9Op08NWs/tG5+8S3L32DYbxM7ILFqA5GJj+776MBo9pZwLNlZWWlxEgU9ek4aw+oUpKzTivMJlf1l1UvaUVgcX4l3BunfvrQrVPYlN9AlGs+zqQJdK5LpDgWp3fWtOuwtMafYiSPkDHTcPUTirYryHGE+yBbhXJOPaOdHPlHIeigFupmbJZsKMvKwX5MGrBhchVSIILAlabnYAaCQrdwtAymzIzYpMk/A+kkYadVE2SWY404gw2acVyfYC+lAx3biQ6KqJFDzcOYMFuuxAV1Uy9hrt3e1NWlSbCF8GZsGix2VYnUVH0JSWN4rYiUhOSrgUK0K9c8b/Ye2pyPsTOwppxNPUAGVUQv3pt/fCVXe0pclhjUGqw1o7zzIYYPQI833AwUXEZNBcrj03mRW6HucuCSaw0AgpozRR0TdBGoPRMUO5g080ed5AZ+8cGYKhNfbKNmDSDIPJ14epA7t9T1XaLZ1F+ZyYTKR2Gx7mpDQdPzCtLO2xwmgGNgUlnDc/aZA2umaFyYADn/zEaSisp9M0cpp+dSy8+A26tVNXp9KD3RdWjjM5xK23xVijy7w8dvnQYjQOkBxZUwIRBXpv3hwwFDjeTBrTxsi3dHMTYWQthtelQDaMEwP5uGgO4y9Gksas6M+4xTSRwTb9RSExtCPcurVQdtDbx5/gPTd6YuJtYeIAnnWiu0JrfNZNBL5e/uaXxLbgLU8WpzXbjAIfteKxsTxP209dQ5Rpn3dHQDseOuH/TMVr63R5Z8gJ07CXib8z63EKjYdO8KhKBGz2WBSYEyt+B57N4AwNHSuA6EfYurjXix777BVsXg3CYQ8zNU3MFki2mtBKNrFePgBx0SWJD0dJ0lUPTfamKNOsSdGRbB0fJlmeoawhhV+PXHxevEuq366x0emCslhTokX+8GVS2SFXJw2kNQNPsxhNE0F4VZW,iv:4/eOiYZBFsG1c7bR0wkCYEKBmU53GNFmn+MFmul1GPM=,tag:zQd1K2lk+zXC3DprVbZi/w==,type:str]", + "sops": { + "kms": null, + "gcp_kms": [ + { + "resource_id": "projects/two-eye-two-see/locations/global/keyRings/sops-keys/cryptoKeys/similar-hubs", + "created_at": "2021-05-12T18:14:54Z", + "enc": "CiQA4OM7eEkMq9G2m3gLWS3mrl3HSyHobWRfIsCdb4CUrbySJzMSSQBy9hCY4tQtuh4Y7O3C7EDKKXrTWADp7JNmVKSrzOUyGeIHC7C30CSfpLkwSMBSRw6gfo4eogGFud/TipQ+zGBecVgqA62AA4M=" + } + ], + "azure_kv": null, + "hc_vault": null, + "age": null, + "lastmodified": "2021-05-12T18:14:56Z", + "mac": "ENC[AES256_GCM,data:cR/7XKrPY8TQImjJoeXNTuM+2Vs02Hnu8w8EUgt7nyr498+2brOZt4AeMtknuSN+IJ66qASu8AwP+z1Ofw+TZTRVxtx/BxlzRBI675J7LxlxcV75nNM1VhcZI7AnQTlW3Bi8vbk6BPoZU/IrmJ7ZR2eZohq0MpgSu7VnHgucrck=,iv:R9dg53E+edcAXROW7c4l7El19EWf9V5f14Uc+LCPigk=,tag:4piO7hPnRzyVxgAUfIrbpg==,type:str]", + "pgp": null, + "unencrypted_suffix": "_unencrypted", + "version": "3.7.1" + } +} \ No newline at end of file diff --git a/kops/ssh-keys/carbonplan.key.pub b/kops/ssh-keys/carbonplan.key.pub new file mode 100644 index 0000000000..189ee3a1dc --- /dev/null +++ b/kops/ssh-keys/carbonplan.key.pub @@ -0,0 +1 @@ +ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQDfwrdzo3LocJCE1aWU1wcu5BPpEYF/wBhjoRB4VrcGILhQfQjjnTx3fbhsBjiE6v2g0hxpk6LSMPIC86z+faB3c7eutrfLeLtd409xFlEY9RlxX7jYVoIDTrZtO/EyQiAIpNQj9ZBz+vSCReExXRHhy3M5iJm9k9mIQlIR1GCWuvVxDP7/unpGvqkr8GplziUUq9jp+CjtlT2jnUmin7hjEALbXDo0oJpCg3ecy2b3Wr4Pb9cCBCPV1jEoHuNr48qLPjb5mc0ZLoQTxHBmLENQFb+AcOrUhnGD9Yl5vc4Ca+c4kl6hzGXyRtHSxoLE6t/2woq36KqmyZpKlYQACkEmsEu7d8JqPTwWH/JlMwtpSCnvhmYE4L9JO5cnANfsZEBqAnkfaa8uKHCXoNHiI5n37/uLv+v2NaJjSrOEoJUAAERzQ2/xT9fGKhHNRUTZ4pN1EXIi4SdPaBXp368eRGmL3tXy8Qv7mEfwv+XpuBKPdTalEbUP3nxmusOGjT6IjA8= yuvipanda@do-the-work.local From ef2df6e11cc631f62dcda8176fb04c1b51f33204 Mon Sep 17 00:00:00 2001 From: YuviPanda Date: Thu, 13 May 2021 00:14:17 +0530 Subject: [PATCH 09/12] Specify carbonplan managed image --- config/hubs/carbonplan.cluster.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/config/hubs/carbonplan.cluster.yaml b/config/hubs/carbonplan.cluster.yaml index 1c0eae7cbd..66972b88e3 100644 --- a/config/hubs/carbonplan.cluster.yaml +++ b/config/hubs/carbonplan.cluster.yaml @@ -58,8 +58,8 @@ hubs: mountPath: /home/jovyan subPath: "{username}" image: - name: pangeo/pangeo-notebook - tag: 2021.05.04 + name: carbonplan/trace-python-notebook + tag: sha-da2d1c9 profileList: # The mem-guarantees are here so k8s doesn't schedule other pods # on these nodes. From 5c90a7641a272a5a18cf5692790d933ea5bdd2f7 Mon Sep 17 00:00:00 2001 From: YuviPanda Date: Thu, 13 May 2021 00:54:42 +0530 Subject: [PATCH 10/12] Document setup-efs.py script --- kops/setup-efs.py | 23 ++++++++++++++++++----- 1 file changed, 18 insertions(+), 5 deletions(-) diff --git a/kops/setup-efs.py b/kops/setup-efs.py index d50e39dd92..065893209f 100644 --- a/kops/setup-efs.py +++ b/kops/setup-efs.py @@ -5,6 +5,9 @@ 1. Create an EFS file system 2. Create a mount target with correct security groups & subnets for a given kops cluster + +Should be terraform that runs after kops cluster is created and +fetches VPC / SG / Subnets with data sources instead. """ import sys @@ -23,23 +26,33 @@ def find_subnets(cluster_name, region): def find_security_groups(cluster_name, region): """ - Find security groups of master and nodes + Find security groups of master and nodes of given cluster. + + The EFS mount target needs to be available to the master node of the + cluster too - this is where the hub, proxy, and other core pods live. + The hub-share-creator also runs there. """ ec2 = boto3.client('ec2', region_name=region) return ec2.describe_security_groups(Filters=[ {'Name':'tag:Name', 'Values': [f'{t}.{cluster_name}' for t in ['masters', 'nodes']]} ])['SecurityGroups'] -def create_filesystem(token, name, region): +def create_filesystem(token, cluster_name, region): + """ + Create an EFS filesystem for given cluster + + Sets up a mount target in the appropriate subnets with correct + security groups too. + """ efs = boto3.client('efs', region_name=region) - subnets = find_subnets(name, region) - security_groups = find_security_groups(name, region) + subnets = find_subnets(cluster_name, region) + security_groups = find_security_groups(cluster_name, region) fs = efs.create_file_system( CreationToken=token, Encrypted=True, Backup=True, Tags=[ - {'Key': 'KubernetesCluster', 'Value': name} + {'Key': 'KubernetesCluster', 'Value': cluster_name} ] ) while True: From 3b790557aead9c5d0dd4ac4b06ccf7f6b3ee6b97 Mon Sep 17 00:00:00 2001 From: YuviPanda Date: Thu, 13 May 2021 17:58:35 +0530 Subject: [PATCH 11/12] Document our lib/jsonnet files --- kops/libsonnet/cluster.jsonnet | 34 +++++++++++++++++++++++++++- kops/libsonnet/instancegroup.jsonnet | 22 ++++++++++++++---- 2 files changed, 50 insertions(+), 6 deletions(-) diff --git a/kops/libsonnet/cluster.jsonnet b/kops/libsonnet/cluster.jsonnet index 3384f14ff5..5d19ebcbb9 100644 --- a/kops/libsonnet/cluster.jsonnet +++ b/kops/libsonnet/cluster.jsonnet @@ -1,4 +1,33 @@ -// local cluster(name, configBase, zone, masterIgName, networkCIDR, subnets) = { +// Exports a customizable kops Cluster object. +// https://kops.sigs.k8s.io/cluster_spec/ lists available properties. +// +// The default configuration sets up the following: +// +// 1. One etcd cluster each on the master node for events & api, +// with minimal resource allocations +// 2. Calico for in-cluster networking https://kops.sigs.k8s.io/networking/calico/, +// with the default settings. Explicitly decided against AWS-VPC cluster networking +// due to pod density issues - see https://github.com/2i2c-org/pangeo-hubs/issues/28. +// 3. Nodes in only one subnet in one AZ. Ideally, the master would be multi-AZ but +// the nodes would be single AZ. Multi AZ workers run into problems attaching PVs +// from other AZs (for hub db PVC, for example), and incurs networking cost for no +// clear benefit in our use case. An opinionated set of IP ranges is picked here, +// and the subnet is created in _config.zone. +// 4. A /16 network for the entire cluster, with a /19 allocated to the one subnet +// currently in use. This allows for ~8000 currently active pods. +// FIXME: Consider a /18 instead? +// 5. Kubernetes API and SSH access allowed from everywhere. +// 6. IAM Permissions to pull from ECR. +// 7. Enables feature gates to allow hub services to run on master node as well. +// 8. Docker as the container runtime. +// +// Supports passing a hidden `_config` object that takes the following +// keys: +// 1. masterInstanceGroupName +// Name of the InstanceGroup that is the master. The etcd clusters will be +// put on this. +// 2. zone +// Zone where the cluster is to be set up { _config+:: { masterInstanceGroupName: "", @@ -63,6 +92,9 @@ anonymousAuth: false, featureGates: { // These boolean values need to be strings + // Without these, services can't target pods running on the master node. + // We want our hub core services to run on the master node, so we need + // to set these. LegacyNodeRoleBehavior: "false", ServiceNodeExclusion: "false" } diff --git a/kops/libsonnet/instancegroup.jsonnet b/kops/libsonnet/instancegroup.jsonnet index bea8ccc305..5ae4967400 100644 --- a/kops/libsonnet/instancegroup.jsonnet +++ b/kops/libsonnet/instancegroup.jsonnet @@ -1,8 +1,24 @@ +// Exports a customizable kops InstanceGroup object. +// https://kops.sigs.k8s.io/instance_groups/ lists available properties +// of the underlying object. On top of that, the following extra features +// are supported: +// 1. cloudLabels are automatically generated from nodeLabels and taints, +// provided in the appropriate kops format. This is required for the +// cluster autoscaler to function - see +// https://github.com/kubernetes/autoscaler/blob/master/cluster-autoscaler/cloudprovider/aws/README.md#auto-discovery-setup +// for more information. +// 2. A cloudLabel is added for the `node.kubernetes.io/instance-type` +// label, since we want to target nodes of different sizes with that +// label. Without the cloudLabel, the cluster autoscaler will not know +// which instancegroup to scale up. +// +// Everything else is just passed through to the kops InstanceGroup config local makeCloudLabels(labels) = { ["k8s.io/cluster-autoscaler/node-template/label/%s" % key]: labels[key] for key in std.objectFields(labels) }; + // Kops expects these as strings of form Key=Value:Effect in spec.taints, // but cloudlabels expects them to be key value pairs of Key: Value:Effect local makeCloudTaints(taints) = { @@ -11,16 +27,12 @@ local makeCloudTaints(taints) = { }; -// local instanceGroup(name, clusterName, nodeImage, labels, taints, machineType, subnets, minSize, maxSize, role) = { { - _config+:: { - clusterName: "" - }, apiVersion: "kops.k8s.io/v1alpha2", kind: "InstanceGroup", metadata: { labels+: { - "kops.k8s.io/cluster": $._config.clusterName, + "kops.k8s.io/cluster": "", }, name: "" }, From 302438f2f377cd1be3f0e07c94da863b7ff53afa Mon Sep 17 00:00:00 2001 From: YuviPanda Date: Thu, 13 May 2021 21:58:00 +0530 Subject: [PATCH 12/12] Specify where the networking comes from --- kops/libsonnet/cluster.jsonnet | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/kops/libsonnet/cluster.jsonnet b/kops/libsonnet/cluster.jsonnet index 5d19ebcbb9..e1c499cad3 100644 --- a/kops/libsonnet/cluster.jsonnet +++ b/kops/libsonnet/cluster.jsonnet @@ -13,9 +13,9 @@ // from other AZs (for hub db PVC, for example), and incurs networking cost for no // clear benefit in our use case. An opinionated set of IP ranges is picked here, // and the subnet is created in _config.zone. -// 4. A /16 network for the entire cluster, with a /19 allocated to the one subnet -// currently in use. This allows for ~8000 currently active pods. -// FIXME: Consider a /18 instead? +// 4. kops defaults for networking - a /16 network for the entire cluster, +// with a /19 allocated to the one subnet currently in use. This allows for +// ~8000 currently active pods. // 5. Kubernetes API and SSH access allowed from everywhere. // 6. IAM Permissions to pull from ECR. // 7. Enables feature gates to allow hub services to run on master node as well.