[pull] master from 2i2c-org:master

.github/workflows/deploy-hubs.yaml

@@ -205,6 +205,7 @@ jobs:
       failure_catalystproject-africa: "${{ env.failure_catalystproject-africa }}"
       failure_hhmi: "${{ env.failure_hhmi }}"
       failure_nasa-esdis: "${{ env.failure_nasa-esdis }}"
+      failure_earthscope: "${{ env.failure_earthscope }}"
 
     # Only run this job on pushes to the default branch and when the job output is not
     # an empty list

config/clusters/earthscope/cluster.yaml

@@ -0,0 +1,28 @@
+name: earthscope
+provider: aws
+aws:
+  key: enc-deployer-credentials.secret.json
+  clusterType: eks
+  clusterName: earthscope
+  region: us-east-2
+support:
+  helm_chart_values_files:
+    - support.values.yaml
+    - enc-support.secret.values.yaml
+hubs:
+  - name: staging
+    display_name: "EarthScope (staging)"
+    domain: staging.earthscope.2i2c.cloud
+    helm_chart: daskhub
+    helm_chart_values_files:
+      - common.values.yaml
+      - staging.values.yaml
+      - enc-staging.secret.values.yaml
+  - name: prod
+    display_name: "EarthScope (prod)"
+    domain: earthscope.2i2c.cloud
+    helm_chart: daskhub
+    helm_chart_values_files:
+      - common.values.yaml
+      - prod.values.yaml
+      - enc-prod.secret.values.yaml

config/clusters/earthscope/common.values.yaml

@@ -0,0 +1,115 @@
+basehub:
+  nfs:
+    pv:
+      # from https://docs.aws.amazon.com/efs/latest/ug/mounting-fs-nfs-mount-settings.html
+      mountOptions:
+        - rsize=1048576
+        - wsize=1048576
+        - timeo=600
+        - soft # We pick soft over hard, so NFS lockups don't lead to hung processes
+        - retrans=2
+        - noresvport
+      serverIP: fs-08e7747330d833d82.efs.us-east-2.amazonaws.com
+      baseShareName: /
+  jupyterhub:
+    custom:
+      2i2c:
+        add_staff_user_ids_to_admin_users: true
+        add_staff_user_ids_of_type: "google"
+      homepage:
+        templateVars:
+          org:
+            url: https://www.earthscope.org/
+            logo_url: https://drive.google.com/uc?export=view&id=1UUStqv7PBcxiIkzECUFKIdQKKIU8mXeb
+          designed_by:
+            name: "2i2c"
+            url: https://2i2c.org
+          operated_by:
+            name: "2i2c"
+            url: https://2i2c.org
+          funded_by:
+            name: "EarthScope Consortium"
+            url: https://www.earthscope.org/
+    hub:
+      config:
+        JupyterHub:
+          authenticator_class: cilogon
+        CILogonOAuthenticator:
+          allowed_idps:
+            http://github.com/login/oauth/authorize:
+              default: true
+              username_derivation:
+                username_claim: "preferred_username"
+            http://google.com/accounts/o8/id:
+              username_derivation:
+                username_claim: email
+        Authenticator:
+          admin_users:
+            - timdittmann
+            - chad-earthscope
+    singleuser:
+      profileList:
+        - display_name: "Shared Small: 1-4 CPU, 8-32 GB"
+          description: "A shared machine, the recommended option until you experience a limitation."
+          profile_options: &profile_options
+            image:
+              display_name: Image
+              unlisted_choice:
+                enabled: True
+                display_name: "Custom image"
+                validation_regex: "^.+:.+$"
+                validation_message: "Must be a publicly available docker image, of form <image-name>:<tag>"
+                kubespawner_override:
+                  image: "{value}"
+              choices:
+                jupyter-scipy:
+                  display_name: Jupyter
+                  slug: jupyter-scipy
+                  kubespawner_override:
+                    image: jupyter/scipy-notebook:2023-06-27
+                rocker-geospatial:
+                  display_name: RStudio
+                  slug: rocker-geospatial
+                  kubespawner_override:
+                    image: rocker/binder:4.3
+                    # Launch into RStudio after the user logs in
+                    default_url: /rstudio
+                    # Ensures container working dir is homedir
+                    # https://github.com/2i2c-org/infrastructure/issues/2559
+                    working_dir: /home/rstudio
+          kubespawner_override:
+            mem_guarantee: 7.234G
+            cpu_guarantee: 0.1
+            mem_limit: null
+            node_selector:
+              node.kubernetes.io/instance-type: r5.xlarge
+
+        - display_name: "Small: 4 CPU, 32 GB"
+          description: "A dedicated machine for you."
+          profile_options: *profile_options
+          kubespawner_override:
+            mem_guarantee: 28.937G
+            cpu_guarantee: 0.4
+            mem_limit: null
+            node_selector:
+              node.kubernetes.io/instance-type: r5.xlarge
+
+        - display_name: "Medium: 16 CPU, 128 GB"
+          description: "A dedicated machine for you."
+          profile_options: *profile_options
+          kubespawner_override:
+            mem_guarantee: 120.513G
+            cpu_guarantee: 1.6
+            mem_limit: null
+            node_selector:
+              node.kubernetes.io/instance-type: r5.4xlarge
+
+        - display_name: "Large: 64 CPU, 512 GB"
+          description: "A dedicated machine for you"
+          profile_options: *profile_options
+          kubespawner_override:
+            mem_guarantee: 489.13G
+            cpu_guarantee: 6.4
+            mem_limit: null
+            node_selector:
+              node.kubernetes.io/instance-type: r5.16xlarge

config/clusters/earthscope/enc-deployer-credentials.secret.json

@@ -0,0 +1,25 @@
+{
+	"AccessKey": {
+		"AccessKeyId": "ENC[AES256_GCM,data:3/SbbOw+wTiuR/pAQX7YlzqZ+A0=,iv:xLjEyZGkbaQ3eVcsfsoFInazu52pT++HtqmbMYFO830=,tag:h8u13XufpQEQzuntIERpfQ==,type:str]",
+		"SecretAccessKey": "ENC[AES256_GCM,data:PAVaW/JJ93Lp3Fx+SzN7ZWZ311NuXjK4E7Fc9kRPxXDmQ/oVScCNYQ==,iv:8BEyHh9q+quWTQGpFYcTaXQcu1G9dSH6xhxLded9ht8=,tag:ut13RYce6EJ8QzBXw5oz1w==,type:str]",
+		"UserName": "ENC[AES256_GCM,data:K5C/1DqLJQrmifgZdWirxa3ws5vmV4M=,iv:raMPP4vlfmLsiLkgeyzBmZjffNLnVKFrS2A9S5DfNLM=,tag:wCMDinJkFGAnvIIIvliF0w==,type:str]"
+	},
+	"sops": {
+		"kms": null,
+		"gcp_kms": [
+			{
+				"resource_id": "projects/two-eye-two-see/locations/global/keyRings/sops-keys/cryptoKeys/similar-hubs",
+				"created_at": "2023-12-12T16:24:08Z",
+				"enc": "CiUA4OM7eOXl1aeA440/OPv/B9ma2zG5unKWGkDFcWuyjc5s3zg6EkkAjTWv+nwxhn+vpS6YKnVaaH/RXfstRskGPc3dp2i0uPgew3epepkKiLgOgC83b49xR6hlUIWYaShh0sj756PXEXi9O4HFGAtx"
+			}
+		],
+		"azure_kv": null,
+		"hc_vault": null,
+		"age": null,
+		"lastmodified": "2023-12-12T16:24:12Z",
+		"mac": "ENC[AES256_GCM,data:6wtwcF2rP6HQzbhaPv3hjpMD9nQbgOeFcuUkFa8pUk1xJy64ISB5VBLop3PgMMXO5kODMsPSC+z3Jmlo8BGtWLb73IvEPQvDUWDxDD9cpN4GRY5et2SD7S9xj7sG1pAoW89xPojsaRBJjGxsNvNMZfE91wSgZMw5hP06SLczbTs=,iv:lxMzrM4Q703SrMfb7DgJxN+tqSU6X+qhBPlURibAOJg=,tag:WnpMif7eXtfO++fy4q3xyQ==,type:str]",
+		"pgp": null,
+		"unencrypted_suffix": "_unencrypted",
+		"version": "3.8.1"
+	}
+}

config/clusters/earthscope/enc-grafana-token.secret.yaml

@@ -0,0 +1,15 @@
+grafana_token: ENC[AES256_GCM,data:BMfj2nbCJsDHhMx2o2yEoTturNCXOc+jBwKa9Qc/ExO33kAimGZ+e4K43XFoCg==,iv:mhLg7G468X5xcBfqZrO3L5O5VR2sQ8kX87pPg6JU6UI=,tag:Ih/2tuZrBStr42ytEG6kXw==,type:str]
+sops:
+    kms: []
+    gcp_kms:
+        - resource_id: projects/two-eye-two-see/locations/global/keyRings/sops-keys/cryptoKeys/similar-hubs
+          created_at: "2023-12-14T14:57:49Z"
+          enc: CiUA4OM7eBNA5OdxpRPD7yDKzvTB3I+oOlGepXn6Kc6VVzU6kcvJEkkAjTWv+oydSi4AiBKtVMhOCs/7vIG7DGwzvOy+kmFq5jJt1FJrhJ4FA2L9x0XygvC+xFI0P7RKQjWEaolkkmJy7chYLVZ1OE42
+    azure_kv: []
+    hc_vault: []
+    age: []
+    lastmodified: "2023-12-14T14:57:50Z"
+    mac: ENC[AES256_GCM,data:Tmm/phrp21xChffGJl9Y5bONlhrC8cHEb8it/iQXv1BwN03vOT1vbeVmGCOm5QN9sEeISsMlKOO7F90xrndF1DX2B7S8i82DYlKoAN14KPj5pLZycu1aEWVYKjNXE2kq0g6P9CFm4YTE0np+wfFrXbjnGN3klqeh/XhmANmVbAI=,iv:JB2FZHMHIssG7PLzFz0Hdde7bMD/jTtGanJab7i7CsU=,tag:4l3bmrsw+QykwoB38ECMAQ==,type:str]
+    pgp: []
+    unencrypted_suffix: _unencrypted
+    version: 3.8.1

config/clusters/earthscope/enc-prod.secret.values.yaml

@@ -0,0 +1,21 @@
+basehub:
+    jupyterhub:
+        hub:
+            config:
+                CILogonOAuthenticator:
+                    client_id: ENC[AES256_GCM,data:1C0ercYZjjc63vTPPcVa7B0Y1bnuawg854Yf3Kl4UnJ0gYuqem+zuv1lQfOzU8zKXy5L,iv:2IZjb7WzomJg8I9uDDXINjULJPXUBfJCldMOxH+B8tA=,tag:Dv1xaVkDCpI7/GLuGv6GzA==,type:str]
+                    client_secret: ENC[AES256_GCM,data:2mGbTTnKcVZp57ZX2Tj2o+j2y0NfABPtTiV6sw3oWlR/t7w4fiFkSK9cyArnJwQfRjWc6M6NNB50A3zWZrKaoPLRj8Afiq8pFTjtRZnZGe5g4h2mXYg=,iv:xmJEHc2V0aG1KEh2eAPj80tZoNzFnBz42QdCSmzO2mc=,tag:+48SuYVdlJCizaYVMn9hrA==,type:str]
+sops:
+    kms: []
+    gcp_kms:
+        - resource_id: projects/two-eye-two-see/locations/global/keyRings/sops-keys/cryptoKeys/similar-hubs
+          created_at: "2023-12-14T18:24:33Z"
+          enc: CiUA4OM7eKr6IJS10QOinx3kZfdMUdO4HmyV6U+JIe+s7IqTAm1DEkkAjTWv+ifsDVRYeh6Gdg0+tLE53DfZfJP0xHGuo6yxdoREE2FGKpodr42/SaaTUt2k5zKlg6k9tOe4FgmRGCT5JjIltibg5hwU
+    azure_kv: []
+    hc_vault: []
+    age: []
+    lastmodified: "2023-12-14T18:24:34Z"
+    mac: ENC[AES256_GCM,data:0Kde6XE/A7k9CwhxQFsa3I61ohr9WN7AO2haWkFETpDG+jXtU5MYkrScbwnlayLa0vM6vk2OfUxR6LrB9jPcxTx8+n2Pqx6kPTzgr8a8ORhG4xc6Lqj0a1KyDMdnGi5beqoXSxolPyd1mnSTAFAVIGwle37Gg0fIr0VFii9lsfQ=,iv:gPVYPvyTEriA9sxbmtMRo611b5dB5idYa0J+DtEYcaY=,tag:RNOItHNKtUGZ/UgfT1Ea2Q==,type:str]
+    pgp: []
+    unencrypted_suffix: _unencrypted
+    version: 3.8.1

config/clusters/earthscope/enc-staging.secret.values.yaml

@@ -0,0 +1,21 @@
+basehub:
+    jupyterhub:
+        hub:
+            config:
+                CILogonOAuthenticator:
+                    client_id: ENC[AES256_GCM,data:Lv/25K0A8CZs6dK20mujkn536hpreimP/MUqGOJ4cpXLTFnJNRmGkN7mYPC2klalEKcn,iv:nj4b7Y75A9wgg+w2XBas17Cs8Az3AzDkeO9u1ZwI1Jo=,tag:gCMMoa3iQWVRQvTQkCIkAg==,type:str]
+                    client_secret: ENC[AES256_GCM,data:EAD3iQGXs7soD4VxRXol2YuuJBmOpDBbX5Cg+VyTk7xA7Jn715vZMNBeOKtal1a6kzyds3tuw+h+DWsF3Dod2MxHS7H4FARHLopP9xuAvS6Tw3mZZ28=,iv:F8CqwLYz7WR5qge0Yj91aU/w5pj6fiEaBvndVe4zvG4=,tag:60BekNlkRhf2a3Nkvo1kWg==,type:str]
+sops:
+    kms: []
+    gcp_kms:
+        - resource_id: projects/two-eye-two-see/locations/global/keyRings/sops-keys/cryptoKeys/similar-hubs
+          created_at: "2023-12-14T18:42:48Z"
+          enc: CiUA4OM7eGqP+F9UNcdWkWcEANT1YIeSiFyzogRgfD+PMhJISk+lEkkAjTWv+sk2C+z/gAjXwaTvoJEJKeuCyiegMLu8QTkJ1KCtcQEU52qv/gm6HvBAlQAnTUKxpQFejxzGOp/8+FNCZiAuaT2hHq1D
+    azure_kv: []
+    hc_vault: []
+    age: []
+    lastmodified: "2023-12-14T18:42:52Z"
+    mac: ENC[AES256_GCM,data:DWO/hv47PbcFx8NATfOJrLUMkOV3dTUzr53nUtpDge+NseEOSoMKeEWz1L7jWYhM+Iga05csm78BT9c3gI921dKlOXRJ6fn1e5guxqKPOAuZugbWUeEqGa8Z26sAwuSRXIZyWiWDJZJThsNk4+s0s7vZmXcrGHGjWA3eCEvTwxE=,iv:9QDeyrmE0euFgqcvZMCuubNA44YB8x2Sa1CqEGJjKjM=,tag:qj5ZnX1TS2Lt4QbXuJFB0Q==,type:str]
+    pgp: []
+    unencrypted_suffix: _unencrypted
+    version: 3.8.1

config/clusters/earthscope/enc-support.secret.values.yaml

@@ -0,0 +1,17 @@
+prometheusIngressAuthSecret:
+    username: ENC[AES256_GCM,data:IyOeJ2ltWgjhRzUrspPpfSpvkgFfAGwrmW2tlBxwoWYWK97hqSn6TAs5M+lT/kp7AICAMflPj1k3IazNiSdvXw==,iv:2vLBAANHd3T+gyYoBVs69ryC4cRQvnc7WufaF3ZliDs=,tag:+Tbpy2ezJTV4xuhA6WXWXg==,type:str]
+    password: ENC[AES256_GCM,data:+LSkp7SQSr6vW4b9rGPgj3I495DquL7uXGo6zbZhfqqqgCTsKgVKNpPFt5Y05TqbOw0fl5qnB6jezNzR05+SBQ==,iv:mtrFlfCbvb4CNjlopzSKL2Q9pLnNjXZw5kB238RFfXQ=,tag:fXhkUIVZ02P0fr1jyx08fg==,type:str]
+sops:
+    kms: []
+    gcp_kms:
+        - resource_id: projects/two-eye-two-see/locations/global/keyRings/sops-keys/cryptoKeys/similar-hubs
+          created_at: "2023-12-10T12:56:11Z"
+          enc: CiUA4OM7eKM2uAp93NZo20W2g59kkkg+f9lIdX3X3C5yMr18KqNWEkkAjTWv+u4Qk+RFc9++zwDFIdNUIJglHKdQjyrPgmKu7Y2GjH8cCYQZnmRy5hABlKEuILmRwdNPqjdfiqRsCwGUbPBZhpIbwUjO
+    azure_kv: []
+    hc_vault: []
+    age: []
+    lastmodified: "2023-12-10T12:56:13Z"
+    mac: ENC[AES256_GCM,data:c4cVSM6GfV9ysrVxx8SYtLHuub9qJc4i8S4XsRrbkJONZgledsuFm65Qe7/yfpypNQd1cJMgHK2kSjEbVfBgO+FMUPHc3dPgG0UjyzkFjTgmzQpDlcyZ0sPrSkCm2RAVLBB0y3qGA8/4jq4gRp+hVgNlhW5q8QKNDYMmR8IK+MU=,iv:+tqnpit70z5KQwFjYU8EdunSFBWx41seakSDDTLtpaI=,tag:HXboDkt6hPrzsX2xY4roiw==,type:str]
+    pgp: []
+    unencrypted_suffix: _unencrypted
+    version: 3.8.1

config/clusters/earthscope/prod.values.yaml

@@ -0,0 +1,16 @@
+basehub:
+  jupyterhub:
+    ingress:
+      hosts: [earthscope.2i2c.cloud]
+      tls:
+        - hosts: [earthscope.2i2c.cloud]
+          secretName: https-auto-tls
+    custom:
+      homepage:
+        templateVars:
+          org:
+            name: "EarthScope"
+    hub:
+      config:
+        CILogonOAuthenticator:
+          oauth_callback_url: https://earthscope.2i2c.cloud/hub/oauth_callback

config/clusters/earthscope/staging.values.yaml

@@ -0,0 +1,16 @@
+basehub:
+  jupyterhub:
+    ingress:
+      hosts: [staging.earthscope.2i2c.cloud]
+      tls:
+        - hosts: [staging.earthscope.2i2c.cloud]
+          secretName: https-auto-tls
+    custom:
+      homepage:
+        templateVars:
+          org:
+            name: "EarthScope staging"
+    hub:
+      config:
+        CILogonOAuthenticator:
+          oauth_callback_url: https://staging.earthscope.2i2c.cloud/hub/oauth_callback

config/clusters/earthscope/support.values.yaml

@@ -0,0 +1,34 @@
+prometheusIngressAuthSecret:
+  enabled: true
+
+prometheus:
+  server:
+    ingress:
+      enabled: true
+      hosts:
+        - prometheus.earthscope.2i2c.cloud
+      tls:
+        - secretName: prometheus-tls
+          hosts:
+            - prometheus.earthscope.2i2c.cloud
+
+grafana:
+  grafana.ini:
+    server:
+      root_url: https://grafana.earthscope.2i2c.cloud/
+  auth.github:
+    enabled: true
+    allowed_organizations: 2i2c-org
+  ingress:
+    hosts:
+      - grafana.earthscope.2i2c.cloud
+    tls:
+      - secretName: grafana-tls
+        hosts:
+          - grafana.earthscope.2i2c.cloud
+
+cluster-autoscaler:
+  enabled: true
+  autoDiscovery:
+    clusterName: earthscope
+  awsRegion: us-east-2

deployer/commands/deployer.py

@@ -97,7 +97,7 @@ def deploy(
         help="Name of hub to operate deploy. Omit to deploy all hubs on the cluster",
     ),
     dask_gateway_version: str = typer.Option(
-        "2023.9.0", help="Version of dask-gateway to install CRDs for"
+        "2024.1.0", help="Version of dask-gateway to install CRDs for"
     ),
     debug: bool = typer.Option(
         False,

docs/howto/upgrade-cluster/index.md

@@ -11,19 +11,26 @@ clusters on AWS.
 
 ## Upgrade policy
 
-We aim to ensure we use a k8s version for the control plane and node groups that is at least **five minor versions** behind the latest one available at any given time.
-
-Ideally, the following rules should also be respected:
-
-1. Every new cluster we deploy should be using the latest available kubernetes version.
-1. All of the clusters deployed in a cloud provider should be using the same version.
-1. Check if new upgrades are needed at least every 3 months.
-
-
-```{warning}
-As of now, we have not yet established practices on how to ensure these upgrades happen according to the policy above. Establishing this is tracked in [this GitHub
-issue](https://github.com/2i2c-org/infrastructure/issues/412).
-```
+1. To keep our k8s cluster's control plane and node pools upgraded to the latest
+   _three_ and _four_ [official minor k8s versions] respectively at all times.
+2. To await a level of maturity for minor k8s versions before we adopt them.
+
+   | Kubernetes distribution | Our maturity criteria                     |
+   | -                       | -                                         |
+   | GKE                     | Part of [GKE's regular release channel]   |
+   | EKS                     | [Supported by `eksctl`] and is GKE mature |
+   | AKS                     | Listed as [generally available on AKS]    |
+3. To upgrade k8s cluster's control plane and node pools at least _twice_ and
+   _once_ per year respectively.
+4. To not disrupt user nodes with running users, by instead rolling out new user
+   node pools if needed and cleaning up the old at a later time.
+5. To check if actions needs to be scheduled related to this in the beginning of
+   every quarter.
+
+[official minor k8s versions]: https://kubernetes.io/releases/
+[gke's regular release channel]: https://cloud.google.com/kubernetes-engine/docs/release-notes-regular
+[supported by `eksctl`]: https://eksctl.io/getting-started/#basic-cluster-creation
+[generally available on aks]: https://learn.microsoft.com/en-gb/azure/aks/supported-kubernetes-versions?tabs=azure-cli#aks-kubernetes-release-calendar
 
 ```{toctree}
 :maxdepth: 1