From a690a908c02e5c84cc8d70c6da071fb3420003cc Mon Sep 17 00:00:00 2001
From: Danny Rorabaugh <imnasnainaec@gmail.com>
Date: Mon, 21 Oct 2024 10:02:37 -0400
Subject: [PATCH 1/4] Update maintenance scripts

---
 docs/deploy/README.md                       | 18 +++----
 docs/deploy/kubernetes_design/README.md     |  4 +-
 maintenance/scripts/add_user_to_proj.py     |  2 +-
 maintenance/scripts/combine_backup.py       |  2 +-
 maintenance/scripts/combine_restore.py      |  2 +-
 maintenance/scripts/db_update_audio_type.py | 59 ---------------------
 maintenance/scripts/get_fonts.py            |  2 +-
 maintenance/scripts/monitor.py              |  2 +-
 maintenance/scripts/rm_project.py           |  2 +-
 maintenance/scripts/update_cert.py          |  2 +-
 10 files changed, 18 insertions(+), 77 deletions(-)
 delete mode 100755 maintenance/scripts/db_update_audio_type.py

diff --git a/docs/deploy/README.md b/docs/deploy/README.md
index ccbc089065..287bd99656 100644
--- a/docs/deploy/README.md
+++ b/docs/deploy/README.md
@@ -397,19 +397,13 @@ Notes:
 
 ### Maintenance Scripts for Kubernetes
 
-There are several maintenance scripts that can be run in the kubernetes cluster:
-
-- `combine-backup-job.sh` - performs a backup of _The Combine_ database and backend files, pushes the backup to AWS S3
-  storage and then removes old backups keeping the latest 3 backups.
-- `combine_backup.py` - just performs the backup and pushes the result to AWS S3 storage.
-- `combine-clean-aws.py` - removes the oldest backups, keeping up to `max_backups`. The default for `max_backups` is 3.
-- `combine_restore.py` - restores _The Combine_ database and backend files from one of the backups in AWS S3 storage.
+There are several maintenance scripts that can be run in the kubernetes cluster; they are listed in
+[./kubernetes_design/README.md#combine_maint-image](./kubernetes_design/README.md#combine_maint-image).
 
 The `combine-backup-job.sh` is currently being run daily on _The Combine_ QA and Production servers as a Kubernetes
 CronJob.
 
-In addition to the daily backup, any of the scripts can be run on-demand using the `kubectl` command. Using the
-`kubectl` command takes the form:
+In addition to the daily backup, any of the scripts can be run on-demand using the `kubectl` command as follows:
 
 ```bash
 kubectl [--kubeconfig=<path-to-kubernetes-file>] [-n thecombine] exec -it deployment/maintenance -- <maintenance script> <script options>
@@ -429,7 +423,7 @@ Notes:
   kubectl [--kubeconfig=<path-to-kubernetes-file>] [-n thecombine] exec -it deployment/maintenance -- <maintenance scripts> --help
   ```
 
-  The only exception is `combine-backup-job.sh` which does not have any script options.
+  The exception is `combine-backup-job.sh` which does not have any script options.
 
 - The `-n thecombine` option is not required if you set `thecombine` as the default namespace for your kubeconfig file
   by running:
@@ -438,6 +432,10 @@ Notes:
   kubectl config set-context --current --namespace=thecombine
   ```
 
+- The `maintenance/scripts/*.py` scripts begin with `#!/usr/bin/env python3` so that they can be run directly in the
+  `maintenance` deployment. If you need to execute one of them in a Python virtual environment `(venv)`, precede the
+  script name with `python`.
+
 ### Checking Certificate Expiration
 
 The `check_cert.py` will print the expiration timestamp for _The Combine's_ TLS certificate.
diff --git a/docs/deploy/kubernetes_design/README.md b/docs/deploy/kubernetes_design/README.md
index 90f8cb36e0..d38e1d9691 100644
--- a/docs/deploy/kubernetes_design/README.md
+++ b/docs/deploy/kubernetes_design/README.md
@@ -89,13 +89,15 @@ features:
   `sillsdev/aws-kubectl`)
 - _Python 3_ - the maintenance scripts included in `combine_maint` are written in _Python_
 - A collection of Maintenance Scripts:
+  - `add_user_to_proj.py` - add specified user to specified project (as harvester if role not specified)
   - `combine_backup.py` - create a compressed tarball of the backend files and database contents and push it to AWS S3
     storage
   - `combine_restore.py` - pull a backup from AWS S3 storage and replace the database and backend files with the
     contents of the backup
-  - `combine-clean-aws.sh` - a `bash` script to cleanup old backups from AWS S3 storage
   - `combine-backup-job.sh` - a `bash` script to run the backup and then cleanup the S3 storage
+  - `combine-clean-aws.sh` - a `bash` script to cleanup old backups from AWS S3 storage
   - `monitor.py` - monitor a set of TLS secrets for updates; when a secret is updated, it is pushed to AWS S3 storage
+  - `rm_project.py` - remove specified project(s) from database and all associated entries and files
   - `update_cert.py` - a script to be used by the cert proxy clients on the NUCs. `update_cert.py` will update a TLS
     certificate if the NUC is connected to the internet and if the certificate is ready for renewal. If these conditions
     are met, it will update the certificate from AWS S3 storage.
diff --git a/maintenance/scripts/add_user_to_proj.py b/maintenance/scripts/add_user_to_proj.py
index cebdb9e526..b9e5a62f44 100755
--- a/maintenance/scripts/add_user_to_proj.py
+++ b/maintenance/scripts/add_user_to_proj.py
@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
 """
 Add user to a project.
 
diff --git a/maintenance/scripts/combine_backup.py b/maintenance/scripts/combine_backup.py
index a73f176321..9841e2715b 100755
--- a/maintenance/scripts/combine_backup.py
+++ b/maintenance/scripts/combine_backup.py
@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
 """Create a backup of TheCombine and push the file to AWS S3 service."""
 
 import argparse
diff --git a/maintenance/scripts/combine_restore.py b/maintenance/scripts/combine_restore.py
index 868738f994..d98b9e55df 100755
--- a/maintenance/scripts/combine_restore.py
+++ b/maintenance/scripts/combine_restore.py
@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
 """
 Restore The Combine from a backup stored in the AWS S3 service.
 
diff --git a/maintenance/scripts/db_update_audio_type.py b/maintenance/scripts/db_update_audio_type.py
deleted file mode 100755
index 3c3cb0244e..0000000000
--- a/maintenance/scripts/db_update_audio_type.py
+++ /dev/null
@@ -1,59 +0,0 @@
-#!/usr/bin/env python
-
-import argparse
-import logging
-from typing import Any, Dict
-
-from bson.binary import UuidRepresentation
-from bson.codec_options import CodecOptions
-from bson.objectid import ObjectId
-from pymongo import MongoClient
-
-
-def parse_args() -> argparse.Namespace:
-    """Define command line arguments for parser."""
-    parser = argparse.ArgumentParser(
-        description="Convert all audio entries from string to a pronunciation object",
-        formatter_class=argparse.ArgumentDefaultsHelpFormatter,
-    )
-    parser.add_argument("--host", default="localhost", help="Database hostname")
-    parser.add_argument("--port", "-p", default=27017, help="Database connection port")
-    parser.add_argument(
-        "--verbose", "-v", action="store_true", help="Print info while running script."
-    )
-    return parser.parse_args()
-
-
-def main() -> None:
-    args = parse_args()
-
-    logging_level = logging.INFO if args.verbose else logging.WARNING
-    logging.basicConfig(format="%(levelname)s:%(message)s", level=logging_level)
-
-    client: MongoClient[Dict[str, Any]] = MongoClient(args.host, args.port)
-    db = client.CombineDatabase
-    codec_opts: CodecOptions[Dict[str, Any]] = CodecOptions(
-        uuid_representation=UuidRepresentation.PYTHON_LEGACY
-    )
-
-    query: Dict[str, Any] = {"audio": {"$ne": []}}
-    for collection_name in ["FrontierCollection", "WordsCollection"]:
-        updates: Dict[ObjectId, Any] = {}
-        curr_collection = db.get_collection(collection_name, codec_options=codec_opts)
-        total_docs = curr_collection.count_documents({})
-        for word in curr_collection.find(query):
-            new_audio = []
-            for aud in word["audio"]:
-                if isinstance(aud, str):
-                    new_audio.append({"speakerId": "", "protected": True, "fileName": aud})
-                else:
-                    new_audio.append(aud)
-            word["audio"] = new_audio
-            updates[ObjectId(word["_id"])] = word
-        logging.info(f"Updating {len(updates)}/{total_docs} documents in {collection_name}.")
-        for obj_id, update in updates.items():
-            curr_collection.update_one({"_id": obj_id}, {"$set": update})
-
-
-if __name__ == "__main__":
-    main()
diff --git a/maintenance/scripts/get_fonts.py b/maintenance/scripts/get_fonts.py
index d3904b0ecf..2186be0cd1 100755
--- a/maintenance/scripts/get_fonts.py
+++ b/maintenance/scripts/get_fonts.py
@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
 """
 Generates font support for all SIL fonts used in Mui-Language-Picker.
 
diff --git a/maintenance/scripts/monitor.py b/maintenance/scripts/monitor.py
index 191d7e9666..8e82c8396d 100755
--- a/maintenance/scripts/monitor.py
+++ b/maintenance/scripts/monitor.py
@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
 """
 Monitor TLS secrets for changes and push changes to AWS S3.
 
diff --git a/maintenance/scripts/rm_project.py b/maintenance/scripts/rm_project.py
index d35fd4dabf..ffeddf4345 100755
--- a/maintenance/scripts/rm_project.py
+++ b/maintenance/scripts/rm_project.py
@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
 """
 Remove a project and its associated data from TheCombine.
 
diff --git a/maintenance/scripts/update_cert.py b/maintenance/scripts/update_cert.py
index 0068be981b..ada2cdcf9b 100755
--- a/maintenance/scripts/update_cert.py
+++ b/maintenance/scripts/update_cert.py
@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
 """
 Check the expiration time of the TLS secret and update if needed.
 

From 6a48f0f02adbd1bd48ec9bc0f61724dae7df3963 Mon Sep 17 00:00:00 2001
From: Danny Rorabaugh <imnasnainaec@gmail.com>
Date: Mon, 21 Oct 2024 11:56:25 -0400
Subject: [PATCH 2/4] More documentation

---
 .gitignore            |  2 +-
 docs/deploy/README.md | 26 ++++++++++++++++++++++++--
 2 files changed, 25 insertions(+), 3 deletions(-)

diff --git a/.gitignore b/.gitignore
index 835a807bf8..6eeb9a3afc 100644
--- a/.gitignore
+++ b/.gitignore
@@ -33,7 +33,7 @@ scripts/*.js
 !scripts/createBackendLicenses.js
 !scripts/printVersion.js
 !scripts/setRelease.js
-
+setup_cluster.log
 npm-debug.log*
 yarn-debug.log*
 yarn-error.log*
diff --git a/docs/deploy/README.md b/docs/deploy/README.md
index 287bd99656..d21356c2a4 100644
--- a/docs/deploy/README.md
+++ b/docs/deploy/README.md
@@ -62,6 +62,7 @@ separate organization. The characteristics of these systems are:
   - the namespace `thecombine` is created
   - the TLS certificate for the server is installed in `thecombine` namespace as a `kubernetes.io/tls` secret with the
     name `thecombine-app-tls`
+  - PersistentVolumeClaims for `backend-data`, `database-data`, and `font-data`
 
 - The QA server has services to login to a private AWS Elastic Container Registry to run private images for _The
   Combine_. In contrast, the Production server only runs public images.
@@ -284,10 +285,25 @@ setup automatically by the Ansible playbook run in the previous section.
 
 For the Production or QA server,
 
-1. login to the Rancher Dashboard for the Production (or QA) server. You need to have an account on the server that was
+1. Login to the Rancher Dashboard for the Production (or QA) server. You need to have an account on the server that was
    created by the operations group.
 2. Copy your `kubectl` configuration to the clipboard and paste it into a file on your host machine, e.g.
    `${HOME}/.kube/prod/config` for the production server.
+3. Check that the PVCs are annotated and labeled:
+   - Get the full list of `<pvc>`s with `kubectl [--context <context>] -n thecombine get pvc`
+   - Check the content of a `<pvc>` with `kubectl [--context <context>] -n thecombine get pvc <pvc> -o yaml`
+   - For all of them, make sure that `metadata:` includes the following lines:
+     ```
+       annotations:
+         meta.helm.sh/release-name: thecombine
+         meta.helm.sh/release-namespace: thecombine
+     ```
+     and
+     ```
+       labels:
+         app.kubernetes.io/managed-by: Helm
+     ```
+   - You can edit a `<pvc>` with `kubectl [--context <context>] -n thecombine edit pvc <pvc>`
 
 ### Setup Environment
 
@@ -308,6 +324,7 @@ deployments (NUC):
 - COMBINE_CAPTCHA_SECRET_KEY
 - COMBINE_SMTP_USERNAME
 - COMBINE_SMTP_PASSWORD
+- HONEYCOMB_API_KEY
 
 You may also set the KUBECONFIG environment variable to the location of the `kubectl` configuration file. This is not
 necessary if the configuration file is at `${HOME}/.kube/config`.
@@ -343,7 +360,8 @@ If using the Docker image,
 
 ## Install Helm Charts Required by _The Combine_
 
-This step sets up the NGINX Ingress Controller and the Certificate Manager, [cert-manager.io](https://cert-manager.io/).
+This step sets up the NGINX Ingress Controller, the Certificate Manager ([cert-manager.io](https://cert-manager.io/)),
+and the OpenTelemetry analytics collector.
 
 If using the Docker image, [open the Docker image terminal](#open-docker-image-terminal) and run:
 
@@ -358,6 +376,10 @@ cd <COMBINE>/deploy/scripts
 ./setup_cluster.py
 ```
 
+Note: This script is not used for the QA/Production deployments. If you need to do a completely fresh install for either
+of those, you can see all the cluster setup steps by executing `setup_cluster.py` with
+`--type development --debug 2> setup_cluster.log`.
+
 ## Install _The Combine_
 
 This step installs _The Combine_ application itself.

From 9f44a702dba4a20558d4a6841f608dea5c160348 Mon Sep 17 00:00:00 2001
From: Danny Rorabaugh <imnasnainaec@gmail.com>
Date: Mon, 21 Oct 2024 14:10:16 -0400
Subject: [PATCH 3/4] Add comment

---
 .github/workflows/deploy_qa.yml | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/.github/workflows/deploy_qa.yml b/.github/workflows/deploy_qa.yml
index d49665da08..54376ead7d 100644
--- a/.github/workflows/deploy_qa.yml
+++ b/.github/workflows/deploy_qa.yml
@@ -86,6 +86,9 @@ jobs:
           aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
           aws-region: ${{ secrets.AWS_DEFAULT_REGION }}
       - name: Remove old AWS ECR images
+        # Remove all images for previous version numbers.
+        # Example: for tag beginning with 1.2.5-, remove all images with tag 1.2.4-*
+        # Example: for tag beginning with 2.4.0-, remove all images with tag 2.3.*
         run: |
           TAG=${{ needs.build.outputs.image_tag }}
           if [[ $TAG =~ ^v([0-9]+)\.([0-9]+)\.([0-9]+)-.* ]]; then

From 3fae27e1deee777232b59a605f2d4b2704bae371 Mon Sep 17 00:00:00 2001
From: Danny Rorabaugh <imnasnainaec@gmail.com>
Date: Mon, 21 Oct 2024 14:13:17 -0400
Subject: [PATCH 4/4] Fix comment

---
 .github/workflows/deploy_qa.yml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/deploy_qa.yml b/.github/workflows/deploy_qa.yml
index 54376ead7d..e1b7bcce2f 100644
--- a/.github/workflows/deploy_qa.yml
+++ b/.github/workflows/deploy_qa.yml
@@ -87,8 +87,8 @@ jobs:
           aws-region: ${{ secrets.AWS_DEFAULT_REGION }}
       - name: Remove old AWS ECR images
         # Remove all images for previous version numbers.
-        # Example: for tag beginning with 1.2.5-, remove all images with tag 1.2.4-*
-        # Example: for tag beginning with 2.4.0-, remove all images with tag 2.3.*
+        # Example: for tag beginning with v1.2.5-, remove all images with tag v1.2.4-*
+        # Example: for tag beginning with v2.4.0-, remove all images with tag v2.3.*
         run: |
           TAG=${{ needs.build.outputs.image_tag }}
           if [[ $TAG =~ ^v([0-9]+)\.([0-9]+)\.([0-9]+)-.* ]]; then