From 947b16163d532c4890ad36f8f4481e0546998e14 Mon Sep 17 00:00:00 2001 From: Jesse Hitch Date: Mon, 19 Aug 2024 10:24:57 +0200 Subject: [PATCH] Fix restores by fixing up podconfig for k8up and removing duplicate slashes in URLs (#305) * remove dup slash and try to fix podconfigs * simplify how we do restores by using existing argocd manifests for nextcloud * update help text --- docs/assets/images/screenshots/help_text.svg | 120 +++++++++---------- pyproject.toml | 2 +- smol_k8s_lab/k8s_apps/social/nextcloud.py | 23 ++-- smol_k8s_lab/k8s_tools/restores.py | 21 ++-- 4 files changed, 77 insertions(+), 89 deletions(-) diff --git a/docs/assets/images/screenshots/help_text.svg b/docs/assets/images/screenshots/help_text.svg index acf3cc05a..0da48cd27 100644 --- a/docs/assets/images/screenshots/help_text.svg +++ b/docs/assets/images/screenshots/help_text.svg @@ -19,131 +19,131 @@ font-weight: 700; } - .terminal-1065325241-matrix { + .terminal-1327010490-matrix { font-family: Fira Code, monospace; font-size: 20px; line-height: 24.4px; font-variant-east-asian: full-width; } - .terminal-1065325241-title { + .terminal-1327010490-title { font-size: 18px; font-weight: bold; font-family: arial; } - .terminal-1065325241-r1 { fill: #c5c8c6 } -.terminal-1065325241-r2 { fill: #5f87ff } -.terminal-1065325241-r3 { fill: #5f87af;font-style: italic; } -.terminal-1065325241-r4 { fill: #5f87af } -.terminal-1065325241-r5 { fill: #8787ff } -.terminal-1065325241-r6 { fill: #afafff } -.terminal-1065325241-r7 { fill: #87afff } -.terminal-1065325241-r8 { fill: #afafff;font-weight: bold } -.terminal-1065325241-r9 { fill: #868887 } -.terminal-1065325241-r10 { fill: #6179a9 } -.terminal-1065325241-r11 { fill: #6161a9 } -.terminal-1065325241-r12 { fill: #7979a9;font-weight: bold } -.terminal-1065325241-r13 { fill: #4961a9 } + .terminal-1327010490-r1 { fill: #c5c8c6 } +.terminal-1327010490-r2 { fill: #5f87ff } +.terminal-1327010490-r3 { fill: #5f87af;font-style: italic; } +.terminal-1327010490-r4 { fill: #5f87af } +.terminal-1327010490-r5 { fill: #8787ff } +.terminal-1327010490-r6 { fill: #afafff } +.terminal-1327010490-r7 { fill: #87afff } +.terminal-1327010490-r8 { fill: #afafff;font-weight: bold } +.terminal-1327010490-r9 { fill: #868887 } +.terminal-1327010490-r10 { fill: #6179a9 } +.terminal-1327010490-r11 { fill: #6161a9 } +.terminal-1327010490-r12 { fill: #7979a9;font-weight: bold } +.terminal-1327010490-r13 { fill: #4961a9 } - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - term + term - + - -                           ๐Ÿงธsmol k8s lab - -Install slim Kubernetes distros + plus all your apps via Argo CD. - -Usage:smol-k8s-lab[OPTIONS] - -โ•ญโ”€ ส• แต”แดฅแต”ส” Options โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ•ฎ -โ”‚โ”‚ -โ”‚-c--config CONFIG_FILEFull path and name of the YAML config file to parse.            โ”‚ -โ”‚Defaults to $XDG_CONFIG_HOME/smol-k8s-lab/config.yamlโ”‚ -โ”‚โ”‚ -โ”‚-D--delete CLUSTER_NAMEDelete an existing cluster by name.                             โ”‚ -โ”‚โ”‚ -โ”‚-i--interactiveโš™๏ธ Interactively configures smol-k8s-labโ”‚ -โ”‚โ”‚ -โ”‚-v--versionPrint the version of smol-k8s-lab (v5.14.0)                     โ”‚ -โ”‚โ”‚ -โ”‚-f--final_cmdRun command immediately after smol-k8s-lab before main cli phaseโ”‚ -โ”‚โ”‚ -โ”‚-h--helpShow this message and exit.                                     โ”‚ -โ•ฐโ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€ โ™ฅ docs: https://small-hack.github.io/smol-k8s-labโ”€โ•ฏ + +                           ๐Ÿงธsmol k8s lab + +Install slim Kubernetes distros + plus all your apps via Argo CD. + +Usage:smol-k8s-lab[OPTIONS] + +โ•ญโ”€ ส• แต”แดฅแต”ส” Options โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ•ฎ +โ”‚โ”‚ +โ”‚-c--config CONFIG_FILEFull path and name of the YAML config file to parse.            โ”‚ +โ”‚Defaults to $XDG_CONFIG_HOME/smol-k8s-lab/config.yamlโ”‚ +โ”‚โ”‚ +โ”‚-D--delete CLUSTER_NAMEDelete an existing cluster by name.                             โ”‚ +โ”‚โ”‚ +โ”‚-i--interactiveโš™๏ธ Interactively configures smol-k8s-labโ”‚ +โ”‚โ”‚ +โ”‚-v--versionPrint the version of smol-k8s-lab (v5.14.1)                     โ”‚ +โ”‚โ”‚ +โ”‚-f--final_cmdRun command immediately after smol-k8s-lab before main cli phaseโ”‚ +โ”‚โ”‚ +โ”‚-h--helpShow this message and exit.                                     โ”‚ +โ•ฐโ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€ โ™ฅ docs: https://small-hack.github.io/smol-k8s-labโ”€โ•ฏ diff --git a/pyproject.toml b/pyproject.toml index e3d6249b0..98b9f456f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "smol_k8s_lab" -version = "5.14.0" +version = "5.14.1" description = "CLI and TUI to quickly install slimmer Kubernetes distros and then manage apps declaratively using Argo CD" authors = ["Jesse Hitch ", "Max Roby "] diff --git a/smol_k8s_lab/k8s_apps/social/nextcloud.py b/smol_k8s_lab/k8s_apps/social/nextcloud.py index 77ed7c516..2b9ba1c09 100644 --- a/smol_k8s_lab/k8s_apps/social/nextcloud.py +++ b/smol_k8s_lab/k8s_apps/social/nextcloud.py @@ -242,7 +242,7 @@ def restore_nextcloud(argocd: ArgoCD, # apply the external secrets so we can immediately use them for restores external_secrets_yaml = ( f"https://raw.githubusercontent.com/small-hack/argocd-apps/{revision}/" - f"{argo_path}/external_secrets_argocd_appset.yaml" + f"{argo_path}external_secrets_argocd_appset.yaml" ) argocd.k8s.apply_manifests(external_secrets_yaml, argocd.namespace) @@ -293,22 +293,16 @@ def restore_nextcloud(argocd: ArgoCD, pgsql_cluster_name, cnpg_backup_schedule) + podconfig_yaml = ( + f"https://raw.githubusercontent.com/small-hack/argocd-apps/{revision}/" + f"{argo_path}pvc_argocd_appset.yaml" + ) + argocd.k8s.apply_manifests(podconfig_yaml, argocd.namespace) + # then we begin the restic restore of all the nextcloud PVCs we lost for pvc in ['files', 'config']: pvc_enabled = secrets.get(f'{pvc}_pvc_enabled', 'false') if pvc_enabled and pvc_enabled.lower() != 'false': - storage_class = secrets.get(f"{pvc}_storage_class", global_pvc_storage_class) - # creates the nexcloud pvc - recreate_pvc(argocd.k8s, - 'nextcloud', - f'nextcloud-{pvc}', - nextcloud_namespace, - secrets[f'{pvc}_storage'], - storage_class, - secrets[f'{pvc}_access_mode'], - "nextcloud-pvc" - ) - # restores the nextcloud pvc k8up_restore_pvc(argocd.k8s, 'nextcloud', @@ -319,7 +313,8 @@ def restore_nextcloud(argocd: ArgoCD, access_key_id, secret_access_key, restic_repo_password, - snapshot_ids[f'nextcloud_{pvc}'] + snapshot_ids[f'nextcloud_{pvc}'], + "file-backups-podconfig" ) # todo: from here on out, this could be async to start on other tasks diff --git a/smol_k8s_lab/k8s_tools/restores.py b/smol_k8s_lab/k8s_tools/restores.py index 3e7cf1e2a..b22c431e3 100644 --- a/smol_k8s_lab/k8s_tools/restores.py +++ b/smol_k8s_lab/k8s_tools/restores.py @@ -47,7 +47,7 @@ def restore_seaweedfs(argocd: ArgoCD, # recreate the seaweedfs PVCs appset pvc_appset = ( f"https://raw.githubusercontent.com/small-hack/argocd-apps/{revision}/" - f"{argocd_path}/s3_pvc_appset.yaml") + f"{argocd_path}s3_pvc_appset.yaml") argocd.k8s.apply_manifests(pvc_appset, argocd.namespace) for swfs_pvc, snapshot_id in snapshots.items(): @@ -61,12 +61,13 @@ def restore_seaweedfs(argocd: ArgoCD, access_key_id, secret_access_key, restic_repo_password, - snapshot_id) + snapshot_id, + "s3-backups-podconfig") # deploy the seaweedfs appset, which will use the restored PVCs above seaweedfs_appset = ( f"https://raw.githubusercontent.com/small-hack/argocd-apps/{revision}/" - f"{argocd_path}/s3_provider_argocd_appset.yaml") + f"{argocd_path}s3_provider_argocd_appset.yaml") argocd.k8s.apply_manifests(seaweedfs_appset, argocd.namespace) # and finally wait for the seaweedfs helm chart app to be ready @@ -75,15 +76,6 @@ def restore_seaweedfs(argocd: ArgoCD, # but then wait again on the pods, just in case... argocd.k8s.wait(namespace, instance=f"{app}-seaweedfs") - # finally, make sure future scheduled backups are working - seaweedfs_pvc_appset = ( - "https://raw.githubusercontent.com/small-hack/argocd-apps/main/" - f"{app}/app_of_apps/s3_pvc_appset.yaml" - ) - argocd.k8s.apply_manifests(seaweedfs_pvc_appset, argocd.namespace) - argocd.wait_for_app(f"{app}-s3-pvc", retry=True) - argocd.sync_app(f"{app}-s3-pvc") - def k8up_restore_pvc(k8s_obj: K8s, app: str, @@ -94,7 +86,8 @@ def k8up_restore_pvc(k8s_obj: K8s, access_key_id: str, secret_access_key: str, restic_repo_password: str, - snapshot_id: str = "latest"): + snapshot_id: str = "latest", + pod_config: str = "backups-podconfig"): """ builds a k8up restore manifest and applies it """ @@ -110,7 +103,7 @@ def k8up_restore_pvc(k8s_obj: K8s, 'failedJobsHistoryLimit': 5, 'successfulJobsHistoryLimit': 1, 'podConfigRef': { - 'name': "backups-podconfig" + 'name': pod_config }, 'restoreMethod': { 'folder': {