Merge pull request #3 from minrk/deployment

Deploy JupyterHub with tofu, helm

.binder/environment.yml

@@ -23,11 +23,11 @@ dependencies:
   - s3fs
   - git
   - jupyterlab-git
-  - 'nodejs>=16,<17'
+  - "nodejs>=16,<17"
   - jupyterlab-myst>=2.0.0
   - pip:
-    - wget
-    - sphinx-exercise
-    - jupytext
-    - nbgitpuller
-    - mystmd
+      - wget
+      - sphinx-exercise
+      - jupytext
+      - nbgitpuller
+      - mystmd

.gitattributes

@@ -0,0 +1 @@
+**/secrets/** filter=git-crypt diff=git-crypt

.gitignore

@@ -158,3 +158,8 @@ cython_debug/
 #  and can be added to the global gitignore or merged into this file.  For a more nuclear
 #  option (not recommended) you can uncomment the following to ignore the entire idea folder.
 #.idea/
+
+.DS_Store
+.terraform
+charts
+tmpcharts

.pre-commit-config.yaml

@@ -0,0 +1,42 @@
+# pre-commit is a tool to perform a predefined set of tasks manually and/or
+# automatically before git commits are made.
+#
+# Config reference: https://pre-commit.com/#pre-commit-configyaml---top-level
+#
+# Common tasks
+#
+# - Run on all files:   pre-commit run --all-files
+# - Register git hooks: pre-commit install --install-hooks
+#
+
+ci:
+  # pre-commit.ci will open PRs updating our hooks once a month
+  autoupdate_schedule: monthly
+
+exclude: "(.*/)?secrets/.*|code_of_conduct.md|rule_of_participation.md"
+
+repos:
+  # autoformat and lint Python code
+  - repo: https://github.com/astral-sh/ruff-pre-commit
+    rev: v0.1.14
+    hooks:
+      - id: ruff
+        types_or: [python, jupyter]
+        args: ["--fix", "--show-fixes"]
+      - id: ruff-format
+        types_or: [python, jupyter]
+
+  # Autoformat: markdown, yaml, javascript (see the file .prettierignore)
+  - repo: https://github.com/pre-commit/mirrors-prettier
+    rev: v4.0.0-alpha.8
+    hooks:
+      - id: prettier
+
+  # Autoformat and linting, misc. details
+  - repo: https://github.com/pre-commit/pre-commit-hooks
+    rev: v4.5.0
+    hooks:
+      - id: end-of-file-fixer
+      - id: requirements-txt-fixer
+      - id: check-case-conflict
+      - id: check-executables-have-shebangs

docs/_config.yml

@@ -1,15 +1,15 @@
 #######################################################################################
 # A default configuration that will be loaded for all jupyter books
-# See the documentation for help and more options: 
+# See the documentation for help and more options:
 # https://jupyterbook.org/customize/config.html
 
 #######################################################################################
 # Book settings
-title                       : Global Fish Tracking System (GFTS) - DESP use case  # The title of the book. Will be placed in the left navbar.
-author                      : Pangeo # The author of the book
-copyright                   : "2023"  # Copyright year to be placed in the footer
-logo                        : "./images/gfts.png"  # A path to the book logo
-only_build_toc_files        : true
+title: Global Fish Tracking System (GFTS) - DESP use case # The title of the book. Will be placed in the left navbar.
+author: Pangeo # The author of the book
+copyright: "2023" # Copyright year to be placed in the footer
+logo: "./images/gfts.png" # A path to the book logo
+only_build_toc_files: true
 
 # Force re-execution of notebooks on each build.
 # See https://jupyterbook.org/content/execute.html
@@ -20,8 +20,8 @@ execute:
 # Add a launch button on a specific binder instance
 launch_buttons:
   notebook_interface: "jupyterlab"
-  binderhub_url: "https://notebooks.gesis.org/binder/"  # The URL for your BinderHub (e.g., https://mybinder.org)
-  jupyterhub_url: "http://pangeo-eosc.vm.fedcloud.eu/jupyterhub/"  # The URL for your JupyterHub. (e.g., https://datahub.berkeley.edu)
+  binderhub_url: "https://notebooks.gesis.org/binder/" # The URL for your BinderHub (e.g., https://mybinder.org)
+  jupyterhub_url: "http://pangeo-eosc.vm.fedcloud.eu/jupyterhub/" # The URL for your JupyterHub. (e.g., https://datahub.berkeley.edu)
 
 # Define the name of the latex output file for PDF builds
 latex:
@@ -34,9 +34,9 @@ bibtex_bibfiles:
 
 # Information about where the book exists on the web
 repository:
-  url: https://github.com/destination-earth/DestinE_ESA_GFTS  # Online location of your book
-  path_to_book: docs  # Optional path to your book, relative to the repository root
-  branch: main  # Which branch of the repository should be used when creating links (optional)
+  url: https://github.com/destination-earth/DestinE_ESA_GFTS # Online location of your book
+  path_to_book: docs # Optional path to your book, relative to the repository root
+  branch: main # Which branch of the repository should be used when creating links (optional)
 
 # Add GitHub buttons to your book
 # See https://jupyterbook.org/customize/config.html#add-a-link-to-your-repository
@@ -47,6 +47,6 @@ html:
 sphinx:
   config:
     html_js_files:
-    - https://cdnjs.cloudflare.com/ajax/libs/require.js/2.3.4/require.min.js
+      - https://cdnjs.cloudflare.com/ajax/libs/require.js/2.3.4/require.min.js
   extra_extensions:
     - sphinx_exercise

docs/intro.md

@@ -1,15 +1,15 @@
 ---
 # File metadata may be provided as frontmatter YAML
-title: Global Fish Tracking System (GFTS) 
+title: Global Fish Tracking System (GFTS)
 subtitle: DESP Use case
-description: Making fishing more sustainable 
+description: Making fishing more sustainable
 date: 2023-12-12
 authors:
   - id: yellowcap
     name: Daniel Wiesmann
     orcid: 0000-0002-3190-4278
     corresponding: true
-    email: [email protected] 
+    email: [email protected]
     roles:
       - Lead
       - User Interface
@@ -71,7 +71,7 @@ authors:
       - Modelling
     affiliations:
       - ifremer
-  - id: emmanuelleautret 
+  - id: emmanuelleautret
     name: Emmanuelle Autret
     orcid: 0000-0002-0979-9192
     corresponding: false
@@ -103,7 +103,6 @@ tags:
 thumbnail: images/gfts.png
 ---
 
-
 +++ {"part":"abstract"}
 
 % The article should include an abstract block at the beginning. The block is delimited by `+++` before and after, and you must specify `"part": "abstract"` as JSON metadata on the block opener. This metadata is required for recognizing the content of this cell as the abstract.
@@ -113,15 +112,14 @@ This project entails the implementation of an advanced fish tracking system util
 
 +++
 
-<a href="https://w3id.org/ro-id/2edcfa66-0f59-42f4-aa29-1c5681466424">  <img alt="RoHub" src="https://img.shields.io/badge/RoHub-FAIR_Executable_Research_Object-2ea44f?logo=Open+Access&logoColor=blue"></a>
+<a href="https://w3id.org/ro-id/2edcfa66-0f59-42f4-aa29-1c5681466424"> <img alt="RoHub" src="https://img.shields.io/badge/RoHub-FAIR_Executable_Research_Object-2ea44f?logo=Open+Access&logoColor=blue"></a>
 
 # Overview of GFTS DESP use case
 
 The presentation below has been presented by Mathieu Woillez at the [Roadshow Webinar: DestinE in action – meet the first DESP use cases](https://destination-earth.eu/event/destine-in-action-meet-the-first-desp-use-cases/) (13 December 2023)
 
 [![DOI](https://zenodo.org/badge/DOI/10.5281/zenodo.10372387.svg)](https://doi.org/10.5281/zenodo.10372387)
 
-
 <style>
 .responsive-wrap iframe{ max-width: 100%;}
 </style>
@@ -130,5 +128,3 @@ The presentation below has been presented by Mathieu Woillez at the [Roadshow We
   <iframe src="https://docs.google.com/presentation/d/1DMa__GRQXhpkqx4VNWDgHdYr_Z2SsWkc_j3m2E01GUs/embed?start=false&loop=false&delayms=3000" frameborder="0" width="960" height="569" allowfullscreen="true" mozallowfullscreen="true" webkitallowfullscreen="true"></iframe>
 <!-- Google embed ends -->
 </div>
-
-

jupyterhub/Makefile

@@ -0,0 +1,6 @@
+images/user/conda-linux-64.lock: images/user/environment.yml images/user/virtual-packages.yaml
+	conda-lock lock -k explicit --mamba --channel conda-forge --platform linux-64 --virtual-package-spec images/user/virtual-packages.yaml --filename-template $@ -f $<
+
+cert-manager:
+	helm repo add jetstack https://charts.jetstack.io --force-update
+	helm install --upgrade --namespace cert-manager jetstack/cert-manager --create-namespace --version v1.13.3 --set installCRDs=true

jupyterhub/README.md

@@ -0,0 +1,95 @@
+# Deploying GFTS Hub
+
+This is a log and record of deploying JupyterHub for GFTS
+
+As much as possible, deployment uses automation via [OpenTofu][], [helm][], but there are always some manual steps.
+
+[OpenTofu]: https://opentofu.org
+[helm]: https://helm.sh
+
+Initial manual steps:
+
+1. create bucket for storing tofu state. Create user and store in `secrets/ovh-creds.sh`, and put bucket name in s3 backend configuration
+2. create API token for OVH API, store in `secrets/ovh-creds.sh`
+
+Next, run tofu, which will create the kubernetes cluster
+
+```bash
+tofu init
+tofu plan
+tofu apply
+```
+
+At this point, we have a kubernetes cluster. Export the kubernetes cluster config:
+
+```bash
+export KUBECONFIG=$PWD/../jupyterhub/secrets/kubeconfig.yaml
+tofu output -raw kubeconfig > $KUBECONFIG
+chmod 600 $KUBECONFIG
+kubectl config rename-context kubernetes-admin@gfts gfts
+kubectl config use-context gfts
+```
+
+And login to the private image registry:
+
+```bash
+echo $(tofu output -raw registry_builder_token) | docker login $(tofu output -raw registry_url) --username $(tofu output -raw registry_builder_name) --password-stdin
+```
+
+Now we move to the `jupyterhub` directory.
+
+Build the image with [chartpress](https://github.com/jupyterhub/chartpress):
+
+```
+chartpress --push
+```
+
+and deploy the chart with:
+
+```
+python deploy.py
+```
+
+Now jupyterhub should be running at https://gfts.minrk.net
+
+## Background
+
+`tofu` is used to deploy cloud resources.
+Its configuration is in the `terraform` directory.
+We only need to use `tofu`
+Once we have kubernetes running, we don't use `tofu` much anymore.
+`tofu` is not run on CI, because its actions can be quite destructive.
+
+`helm` is used to deploy things on kubernetes.
+This is the main mechanism by which we modify our jupyterhub deployment.
+This can be done on CI (but isn't yet).
+
+There are two configuration files:
+
+- gfts-hub/values.yaml is the main configuration file
+- secrets/config.yaml is the file containing
+
+`chartpress` is used to build our user image and update the helm chart
+
+Deploying updates is two steps:
+
+1. `chartpress` to ensure the image is up-to-date
+2. `helm upgrade` to apply the changes
+
+To deploy an update:
+
+```
+python3 deploy.py chartpress
+python3 deploy.py helm
+```
+
+and cleanup your local files:
+
+```
+chartpress --reset
+```
+
+## The user image
+
+The user image is defined in `images/user`.
+To change what's in the image, modify `images/user/environment.yml` and run `make images/user/conda-linux-64.lock`.

jupyterhub/chartpress.yaml

@@ -0,0 +1,6 @@
+charts:
+  - name: gfts-hub
+    imagePrefix: c63eqfuv.c1.gra9.container-registry.ovh.net/gfts/jupyterhub-
+    images:
+      user:
+        valuesPath: jupyterhub.singleuser.image