Add smtp port for mastodon, add valkey restore to restore_mastodon,…

… and update textual + rich (#327) * remove unused function from nextcloud * clean up the restore_mastodon function * document smtp_port for mastodon secret keys * update textual to 0.84, rich to 13.9 and smol-k8s-lab version to 5.18.0 * add restores for valkey primary and replica * valkey-primary and valkey-replica pvcs added to mastodon restore individually * make smtp port a string instead of an int for mastodon * change pvc storage underscore to a hyphen
small-hack · Oct 23, 2024 · 1f94f39 · 1f94f39
1 parent eb60c8c
commit 1f94f39
Show file tree

Hide file tree

Showing 7 changed files with 351 additions and 312 deletions.
diff --git a/docs/assets/images/screenshots/help_text.svg b/docs/assets/images/screenshots/help_text.svg
diff --git a/docs/k8s_apps/mastodon.md b/docs/k8s_apps/mastodon.md
@@ -71,6 +71,8 @@ apps:
         restic_snapshot_ids:
           seaweedfs_volume: latest
           seaweedfs_filer: latest
+          mastodon_valkey_primary: latest
+          mastodon_valkey_replica: latest
       values:
         # admin user
         admin_user: "tootadmin"
@@ -108,6 +110,9 @@ apps:
     argo:
       # secrets keys to make available to Argo CD ApplicationSets
       secret_keys:
+        # smtp port on your mail server
+        smtp_port: '25'
+        # admin user for your mastodon instance
         admin_user: tootadmin
         # hostname that users go to in the browser
         hostname: ""

diff --git a/poetry.lock b/poetry.lock
diff --git a/pyproject.toml b/pyproject.toml
@@ -1,6 +1,6 @@
 [tool.poetry]
 name          = "smol_k8s_lab"
-version       = "5.17.4"
+version       = "5.18.0"
 description   = "CLI and TUI to quickly install slimmer Kubernetes distros and then manage apps declaratively using Argo CD"
 authors       = ["Jesse Hitch <[email protected]>",
                  "Max Roby <[email protected]>"]
@@ -40,10 +40,10 @@ pyjwt              = "^2.9"
 python             = ">=3.11,<3.13"
 pyyaml             = "^6.0"
 requests           = "^2.32"
-rich               = "^13.8"
+rich               = "^13.9"
 ruamel-yaml        = "^0.18"
 ruamel-yaml-string = "^0.1"
-textual            = "^0.83.0"
+textual            = "^0.84"
 xdg-base-dirs      = "^6.0"
 pygame             = "^2.5"
 python-ulid        = "^3.0"

diff --git a/smol_k8s_lab/config/default_config.yaml b/smol_k8s_lab/config/default_config.yaml
@@ -803,6 +803,8 @@ apps:
           # seaweedfs_master: latest
           seaweedfs_volume: latest
           seaweedfs_filer: latest
+          mastodon_valkey_primary: latest
+          mastodon_valkey_replica: latest
       values:
         # admin user
         admin_user: "tootadmin"
@@ -840,6 +842,9 @@ apps:
     argo:
       # secrets keys to make available to Argo CD ApplicationSets
       secret_keys:
+        # smtp port on your mail server
+        smtp_port: '25'
+        # admin user for your mastodon instance
         admin_user: tootadmin
         # hostname that users go to in the browser
         hostname: ""

diff --git a/smol_k8s_lab/k8s_apps/social/mastodon.py b/smol_k8s_lab/k8s_apps/social/mastodon.py
@@ -3,7 +3,9 @@
 from smol_k8s_lab.k8s_apps.operators.minio import create_minio_alias
 from smol_k8s_lab.k8s_apps.social.mastodon_secrets import generate_mastodon_secrets
 from smol_k8s_lab.k8s_tools.argocd_util import ArgoCD
-from smol_k8s_lab.k8s_tools.restores import restore_seaweedfs, restore_cnpg_cluster
+from smol_k8s_lab.k8s_tools.restores import (restore_seaweedfs,
+                                             k8up_restore_pvc,
+                                             restore_cnpg_cluster)
 from smol_k8s_lab.utils.passwords import create_password
 from smol_k8s_lab.utils.rich_cli.console_logging import sub_header, header
 from smol_k8s_lab.utils.run.subproc import subproc
@@ -453,14 +455,15 @@ def setup_bitwarden_items(argocd: ArgoCD,
                 f"Recieved: {e}"
                 )
 
+
 def restore_mastodon(argocd: ArgoCD,
                      mastodon_hostname: str,
                      mastodon_namespace: str,
                      argo_dict: dict,
                      secrets: dict,
                      restore_dict: dict,
                      backup_dict: dict,
-                     pvc_storage_class: str,
+                     global_pvc_storage_class: str,
                      pgsql_cluster_name: str,
                      bitwarden: BwCLI) -> None:
     """
@@ -475,20 +478,18 @@ def restore_mastodon(argocd: ArgoCD,
     restic_repo_password = backup_dict['restic_repo_pass']
     cnpg_backup_schedule = backup_dict['postgres_schedule']
 
-
-    # apply the external secrets so we can immediately use them for restores
-    # ⚠️ ⚠️ ⚠️ ⚠️ ⚠️ ⚠️ ⚠️ ⚠️ ⚠️ ⚠️ ⚠️ ⚠️ ⚠️ ⚠️ ⚠️ ⚠️ ⚠️ ⚠️ ⚠️ ⚠️ ⚠️ ⚠️ ⚠️ ⚠️ ⚠️ ⚠️
-    # WARNING: change this back to main when done testing
-    # ⚠️ ⚠️ ⚠️ ⚠️ ⚠️ ⚠️ ⚠️ ⚠️ ⚠️ ⚠️ ⚠️ ⚠️ ⚠️ ⚠️ ⚠️ ⚠️ ⚠️ ⚠️ ⚠️ ⚠️ ⚠️ ⚠️ ⚠️ ⚠️ ⚠️ ⚠️
-    revision = argo_dict.get("revision", "add-pvc-helm-chart-for-mastodon")
-    argo_path = argo_dict["path"]
+    # get argo git repo info
+    revision = argo_dict['revision']
+    argo_path = argo_dict['path']
 
     # first we grab existing bitwarden items if they exist
     if bitwarden:
         refresh_bweso(argocd, mastodon_hostname, bitwarden)
+
+        # apply the external secrets so we can immediately use them for restores
         external_secrets_yaml = (
-                "https://raw.githubusercontent.com/small-hack/argocd-apps/"
-                f"{revision}/{argo_path}/external_secrets_argocd_appset.yaml"
+                f"https://raw.githubusercontent.com/small-hack/argocd-apps/{revision}/"
+                f"{argo_path}external_secrets_argocd_appset.yaml"
                 )
         argocd.k8s.apply_manifests(external_secrets_yaml, argocd.namespace)
 
@@ -505,8 +506,10 @@ def restore_mastodon(argocd: ArgoCD,
 
     # then we create all the seaweedfs pvcs we lost and restore them
     snapshot_ids = restore_dict['restic_snapshot_ids']
+    s3_pvc_storage_class = secrets.get("s3_pvc_storage_class", global_pvc_storage_class)
+
     restore_seaweedfs(
-            argocd.k8s,
+            argocd,
             'mastodon',
             mastodon_namespace,
             revision,
@@ -517,35 +520,51 @@ def restore_mastodon(argocd: ArgoCD,
             secret_access_key,
             restic_repo_password,
             s3_pvc_capacity,
-            pvc_storage_class,
+            s3_pvc_storage_class,
             "ReadWriteOnce",
             snapshot_ids['seaweedfs_volume'],
-            snapshot_ids['seaweedfs_filer']
-            )
+            snapshot_ids['seaweedfs_filer'])
 
     # then we finally can restore the postgres database :D
     if restore_dict.get("cnpg_restore", False):
         psql_version = restore_dict.get("postgresql_version", 16)
         s3_endpoint = secrets.get('s3_endpoint', "")
-        restore_cnpg_cluster('mastodon',
+        restore_cnpg_cluster(argocd.k8s,
+                             'mastodon',
                              mastodon_namespace,
                              pgsql_cluster_name,
                              psql_version,
                              s3_endpoint,
                              pg_access_key_id,
                              pg_secret_access_key,
-                             seaweedf_s3_endpoint,
                              pgsql_cluster_name,
                              cnpg_backup_schedule)
 
+    podconfig_yaml = (
+            f"https://raw.githubusercontent.com/small-hack/argocd-apps/{revision}/"
+            f"{argo_path}pvc_argocd_appset.yaml"
+            )
+    argocd.k8s.apply_manifests(podconfig_yaml, argocd.namespace)
+
+    # then we begin the restic restore of all the mastodon PVCs we lost
+    for pvc in ['valkey_primary', 'valkey_replica']:
+        pvc_enabled = secrets.get('valkey_pvc_enabled', 'false')
+        if pvc_enabled and pvc_enabled.lower() != 'false':
+            # restores the mastodon pvc
+            k8up_restore_pvc(
+                    k8s_obj=argocd.k8s,
+                    app='mastodon',
+                    pvc=f'mastodon-{pvc.replace("_","-")}',
+                    namespace='mastodon',
+                    s3_endpoint=s3_backup_endpoint,
+                    s3_bucket=s3_backup_bucket,
+                    access_key_id=access_key_id,
+                    secret_access_key=secret_access_key,
+                    restic_repo_password=restic_repo_password,
+                    snapshot_id=snapshot_ids[f'mastodon_{pvc}'],
+                    pod_config="file-backups-podconfig"
+                    )
+
     # todo: from here on out, this could be async to start on other tasks
-    # install mastodon as usual
-    argocd.install_app('mastodon', argo_dict)
-
-    # verify mastodon rolled out
-    rollout = (f"kubectl rollout status -n {mastodon_namespace} "
-               "deployment/mastodon-web-app --watch --timeout 10m")
-    while True:
-        rolled_out = subproc([rollout])
-        if "NotFound" not in rolled_out:
-            break
+    # install mastodon as usual, but wait on it this time
+    argocd.install_app('mastodon', argo_dict, True)
diff --git a/smol_k8s_lab/k8s_apps/social/nextcloud.py b/smol_k8s_lab/k8s_apps/social/nextcloud.py
@@ -5,7 +5,6 @@
 from smol_k8s_lab.k8s_apps.social.nextcloud_occ_commands import Nextcloud
 from smol_k8s_lab.k8s_tools.argocd_util import ArgoCD
 from smol_k8s_lab.k8s_tools.restores import (restore_seaweedfs,
-                                             recreate_pvc,
                                              k8up_restore_pvc,
                                              restore_cnpg_cluster)
 from smol_k8s_lab.utils.passwords import create_password