Database backup Cron jobs

ewc/cron-jobs/Dockerfile

@@ -0,0 +1,47 @@
+FROM alpine:3.20
+
+# Set arguments for product and version
+ARG PRODUCT=vault
+ARG VAULT_VERSION=1.18.0
+ARG ETCD_VERSION=v3.5.17
+
+# Install dependencies and AWS CLI & PostgreSQL client
+# aws-cli is around ~100MB
+RUN apk add --update --virtual .deps --no-cache gnupg wget unzip && \
+    apk add --no-cache aws-cli postgresql-client bash && \
+    cd /tmp && \
+    # Install Vault
+    # Vault installation snippet from https://www.hashicorp.com/blog/installing-hashicorp-tools-in-alpine-linux-containers
+    # Vault binary is enormous ~436MB, might need to think some other option.. 
+    # Someone else also not happy about it https://github.com/hashicorp/vault/issues/22893
+    wget https://releases.hashicorp.com/${PRODUCT}/${VAULT_VERSION}/${PRODUCT}_${VAULT_VERSION}_linux_amd64.zip && \
+    wget https://releases.hashicorp.com/${PRODUCT}/${VAULT_VERSION}/${PRODUCT}_${VAULT_VERSION}_SHA256SUMS && \
+    wget https://releases.hashicorp.com/${PRODUCT}/${VAULT_VERSION}/${PRODUCT}_${VAULT_VERSION}_SHA256SUMS.sig && \
+    wget -qO- https://www.hashicorp.com/.well-known/pgp-key.txt | gpg --import && \
+    gpg --verify ${PRODUCT}_${VAULT_VERSION}_SHA256SUMS.sig ${PRODUCT}_${VAULT_VERSION}_SHA256SUMS && \
+    grep ${PRODUCT}_${VAULT_VERSION}_linux_amd64.zip ${PRODUCT}_${VAULT_VERSION}_SHA256SUMS | sha256sum -c && \
+    unzip /tmp/${PRODUCT}_${VAULT_VERSION}_linux_amd64.zip -d /tmp && \
+    mv /tmp/${PRODUCT} /usr/local/bin/${PRODUCT} && \
+    # Install etcd
+    wget https://github.com/etcd-io/etcd/releases/download/${ETCD_VERSION}/etcd-${ETCD_VERSION}-linux-amd64.tar.gz -O /tmp/etcd-${ETCD_VERSION}-linux-amd64.tar.gz && \
+    tar xzvf /tmp/etcd-${ETCD_VERSION}-linux-amd64.tar.gz && \
+    mv /tmp/etcd-${ETCD_VERSION}-linux-amd64/etcdctl /usr/local/bin/etcdctl && \
+    # Remove temporary files & build dependencies
+    rm -rf /tmp/* && \
+    rm -f ${PRODUCT}_${VAULT_VERSION}_SHA256SUMS ${PRODUCT}_${VAULT_VERSION}_SHA256SUMS.sig && \
+    apk del .deps
+
+# Verify installations
+RUN etcdctl version && vault version && aws --version && pg_dump --version
+
+# Add scripts and make them executable
+COPY common-functions.sh /usr/local/bin/common-functions.sh
+COPY vault-snapshot.sh /usr/local/bin/vault-snapshot.sh
+COPY apisix-snapshot.sh /usr/local/bin/apisix-snapshot.sh
+COPY keycloak-snapshot.sh /usr/local/bin/keycloak-snapshot.sh
+RUN chmod +x /usr/local/bin/common-functions.sh \
+             /usr/local/bin/vault-snapshot.sh \
+             /usr/local/bin/apisix-snapshot.sh \
+             /usr/local/bin/keycloak-snapshot.sh
+
+CMD ["/usr/local/bin/vault-snapshot.sh"]

ewc/cron-jobs/apisix-snapshot.sh

@@ -0,0 +1,36 @@
+#!/bin/bash
+
+# Source common functions
+source /usr/local/bin/common-functions.sh
+
+# Variables
+ETCD_ENDPOINT=${ETCD_ENDPOINT}
+S3_BUCKET_BASE_PATH=${S3_BUCKET_BASE_PATH}
+AWS_ACCESS_KEY_ID=${AWS_ACCESS_KEY_ID}
+AWS_SECRET_ACCESS_KEY=${AWS_SECRET_ACCESS_KEY}
+AWS_REGION=${AWS_REGION:-"eu-north-1"}
+
+# Check required variables
+check_var "ETCD_ENDPOINT" "$ETCD_ENDPOINT"
+check_var "S3_BUCKET_BASE_PATH" "$S3_BUCKET_BASE_PATH"
+check_var "AWS_ACCESS_KEY_ID" "$AWS_ACCESS_KEY_ID"
+check_var "AWS_SECRET_ACCESS_KEY" "$AWS_SECRET_ACCESS_KEY"
+
+# Generate ISO 8601 compliant timestamp
+TIMESTAMP_ISO_8601=$(generate_iso_8601_timestamp)
+
+SNAPSHOT_NAME="snapshot-$TIMESTAMP_ISO_8601.db"
+
+# Take the etcd snapshot
+ETCDCTL_API=3 etcdctl --endpoints=${ETCD_ENDPOINT} snapshot save /tmp/$SNAPSHOT_NAME
+
+# Upload to S3
+aws s3 cp /tmp/$SNAPSHOT_NAME s3://${S3_BUCKET_BASE_PATH}${SNAPSHOT_NAME} --region "${AWS_REGION}"
+
+if [ $? -ne 0 ]; then
+  echo "Error: Failed to upload snapshot to S3"
+  exit 1
+fi
+
+# Clean up
+rm /tmp/$SNAPSHOT_NAME

ewc/cron-jobs/common-functions.sh

@@ -0,0 +1,24 @@
+#!/bin/bash
+
+# Function to check if a variable is set
+check_var() {
+  local var_name=$1
+  local var_value=$2
+  if [ -z "$var_value" ]; then
+    echo "Error: $var_name is not set."
+    exit 1
+  fi
+}
+
+# Function to generate ISO 8601 compliant timestamp
+generate_iso_8601_timestamp() {
+  local timezone_offset=$(date +%z)
+  local timestamp=$(date +%Y-%m-%dT%H:%M:%S)
+
+  if [ "$timezone_offset" == "+0000" ]; then
+    echo "${timestamp}Z"
+  else
+    # (need to use sed as couldn't make it work with '%:z' in date command)
+    echo "${timestamp}$(echo $timezone_offset | sed 's/\(..\)$/:\1/')"
+  fi
+}

ewc/cron-jobs/keycloak-snapshot.sh

@@ -0,0 +1,42 @@
+#!/bin/bash
+
+# Source common functions
+source /usr/local/bin/common-functions.sh
+
+# Variables
+POSTGRES_HOST=${POSTGRES_HOST}
+POSTGRES_DB=${POSTGRES_DB}
+POSTGRES_USER=${POSTGRES_USER}
+POSTGRES_PASSWORD=${POSTGRES_PASSWORD}
+S3_BUCKET_BASE_PATH=${S3_BUCKET_BASE_PATH}
+AWS_ACCESS_KEY_ID=${AWS_ACCESS_KEY_ID}
+AWS_SECRET_ACCESS_KEY=${AWS_SECRET_ACCESS_KEY}
+AWS_REGION=${AWS_REGION:-"eu-north-1"}
+
+# Check required variables
+check_var "POSTGRES_HOST" "$POSTGRES_HOST"
+check_var "POSTGRES_DB" "$POSTGRES_DB"
+check_var "POSTGRES_USER" "$POSTGRES_USER"
+check_var "POSTGRES_PASSWORD" "$POSTGRES_PASSWORD"
+check_var "S3_BUCKET_BASE_PATH" "$S3_BUCKET_BASE_PATH"
+check_var "AWS_ACCESS_KEY_ID" "$AWS_ACCESS_KEY_ID"
+check_var "AWS_SECRET_ACCESS_KEY" "$AWS_SECRET_ACCESS_KEY"
+
+# Generate ISO 8601 compliant timestamp
+TIMESTAMP_ISO_8601=$(generate_iso_8601_timestamp)
+
+SNAPSHOT_NAME="snapshot-$TIMESTAMP_ISO_8601.sql"
+
+# Take the db snapshot
+PGPASSWORD=$POSTGRES_PASSWORD pg_dump -h $POSTGRES_HOST -U $POSTGRES_USER -d $POSTGRES_DB -F c -b -v -f /tmp/$SNAPSHOT_NAME
+
+# Upload to S3
+aws s3 cp /tmp/$SNAPSHOT_NAME s3://${S3_BUCKET_BASE_PATH}${SNAPSHOT_NAME} --region "${AWS_REGION}"
+
+if [ $? -ne 0 ]; then
+  echo "Error: Failed to upload snapshot to S3"
+  exit 1
+fi
+
+# Clean up
+rm /tmp/$SNAPSHOT_NAME

ewc/cron-jobs/vault-snapshot.sh

@@ -0,0 +1,45 @@
+#!/bin/bash
+
+# Source common functions
+source /usr/local/bin/common-functions.sh
+
+# Variables
+VAULT_ADDR=${VAULT_ADDR}
+S3_BUCKET_BASE_PATH=${S3_BUCKET_BASE_PATH}
+AWS_ACCESS_KEY_ID=${AWS_ACCESS_KEY_ID}
+AWS_SECRET_ACCESS_KEY=${AWS_SECRET_ACCESS_KEY}
+AWS_REGION=${AWS_REGION:-"eu-north-1"}
+
+# Check required variables
+check_var "VAULT_ADDR" "$VAULT_ADDR"
+check_var "S3_BUCKET_BASE_PATH" "$S3_BUCKET_BASE_PATH"
+check_var "AWS_ACCESS_KEY_ID" "$AWS_ACCESS_KEY_ID"
+check_var "AWS_SECRET_ACCESS_KEY" "$AWS_SECRET_ACCESS_KEY"
+
+# Retrieve the provided service account token
+SA_TOKEN=$(cat /var/run/secrets/kubernetes.io/serviceaccount/token)
+
+# Authenticate with Vault using the Kubernetes auth method to obtain a Vault token
+export VAULT_TOKEN=$(vault write -field=token auth/kubernetes/login \
+  role=backup-cron-job \
+  jwt=$SA_TOKEN)
+
+# Generate ISO 8601 compliant timestamp
+TIMESTAMP_ISO_8601=$(generate_iso_8601_timestamp)
+
+SNAPSHOT_NAME="snapshot-$TIMESTAMP_ISO_8601.snap"
+
+# Take the snapshot
+# https://developer.hashicorp.com/vault/tutorials/standard-procedures/sop-backup
+vault operator raft snapshot save /tmp/$SNAPSHOT_NAME
+
+# Upload to S3
+aws s3 cp /tmp/$SNAPSHOT_NAME s3://${S3_BUCKET_BASE_PATH}${SNAPSHOT_NAME} --region "${AWS_REGION}"
+
+if [ $? -ne 0 ]; then
+  echo "Error: Failed to upload snapshot to S3"
+  exit 1
+fi
+
+# Clean up
+rm /tmp/$SNAPSHOT_NAME

ewc/cron_jobs.tf

@@ -0,0 +1,192 @@
+################################################################################
+
+# Backups
+
+################################################################################
+
+# TODO consider using a service account for assuming AWS role - no need to use access key and secret key
+# E.g. using service account with self-hosted k8s cluster requires own OIDC provider, keycloak, s3, dex etc.
+
+################################################################################
+
+# Vault
+################################################################################
+resource "kubernetes_service_account" "vault_backup_cron_job_service_account" {
+  metadata {
+    name      = "vault-backup-cron-job-sa"
+    namespace = module.ewc-vault-init.vault_namespace_name
+  }
+
+  automount_service_account_token = true
+
+  depends_on = [module.ewc-vault-init]
+
+}
+
+resource "kubernetes_secret" "vault_backup_cron_job_secrets" {
+  metadata {
+    name      = "vault-backup-cron-jobs"
+    namespace = module.ewc-vault-init.vault_namespace_name
+  }
+
+  data = {
+    AWS_ACCESS_KEY_ID     = var.s3_bucket_access_key
+    AWS_SECRET_ACCESS_KEY = var.s3_bucket_secret_key
+  }
+
+  type = "Opaque"
+}
+
+resource "kubernetes_cron_job_v1" "vault_backup" {
+  metadata {
+    name      = "vault-backup"
+    namespace = module.ewc-vault-init.vault_namespace_name
+  }
+
+  spec {
+    concurrency_policy            = "Replace"
+    failed_jobs_history_limit     = 3 # Keep the latest 3 failed jobs
+    schedule                      = "1 0 * * *"
+    timezone                      = "Etc/UTC"
+    starting_deadline_seconds     = 43200 # 12 hours
+    successful_jobs_history_limit = 1     # Keep the latest
+
+    job_template {
+      metadata {}
+      spec {
+        backoff_limit = 6 # This the default value
+        template {
+          metadata {}
+          spec {
+            restart_policy       = "OnFailure"
+            service_account_name = kubernetes_service_account.vault_backup_cron_job_service_account.metadata.0.name
+            container {
+              name              = "vault-backup"
+              image             = "ghcr.io/eurodeo/femdi-gateway-iac/cron-jobs:latest"
+              image_pull_policy = "Always" # TODO change to IfNotPresent once tested out to be working
+              command           = ["/bin/sh", "-c", "/usr/local/bin/vault-snapshot.sh"]
+
+              env {
+                name  = "VAULT_ADDR"
+                value = local.vault_host
+              }
+
+              env {
+                name  = "S3_BUCKET_BASE_PATH"
+                value = var.vault_backup_bucket_base_path
+              }
+
+              env {
+                name = "AWS_ACCESS_KEY_ID"
+                value_from {
+                  secret_key_ref {
+                    name = kubernetes_secret.vault_backup_cron_job_secrets.metadata.0.name
+                    key  = "AWS_ACCESS_KEY_ID"
+                  }
+                }
+              }
+
+              env {
+                name = "AWS_SECRET_ACCESS_KEY"
+                value_from {
+                  secret_key_ref {
+                    name = kubernetes_secret.vault_backup_cron_job_secrets.metadata.0.name
+                    key  = "AWS_SECRET_ACCESS_KEY"
+                  }
+                }
+              }
+            }
+          }
+        }
+      }
+    }
+  }
+
+  depends_on = [module.ewc-vault-init]
+
+}
+
+
+################################################################################
+
+# APISIX backup
+################################################################################
+resource "kubernetes_secret" "apisix_backup_cron_job_secrets" {
+  metadata {
+    name      = "apisix-backup-cron-jobs"
+    namespace = kubernetes_namespace.apisix.metadata.0.name
+  }
+
+  data = {
+    AWS_ACCESS_KEY_ID     = var.s3_bucket_access_key
+    AWS_SECRET_ACCESS_KEY = var.s3_bucket_secret_key
+  }
+
+  type = "Opaque"
+}
+
+resource "kubernetes_cron_job_v1" "apisix_backup" {
+  metadata {
+    name      = "apisix-backup"
+    namespace = kubernetes_namespace.apisix.metadata.0.name
+  }
+
+  spec {
+    concurrency_policy            = "Replace"
+    failed_jobs_history_limit     = 3 # Keep the latest 3 failed jobs
+    schedule                      = "1 0 * * *"
+    timezone                      = "Etc/UTC"
+    starting_deadline_seconds     = 43200 # 12 hours
+    successful_jobs_history_limit = 1     # Keep the latest
+
+    job_template {
+      metadata {}
+      spec {
+        backoff_limit = 6 # This the default value
+        template {
+          metadata {}
+          spec {
+            restart_policy = "OnFailure"
+            container {
+              name              = "apisix-backup"
+              image             = "ghcr.io/eurodeo/femdi-gateway-iac/cron-jobs:latest"
+              image_pull_policy = "Always" # TODO change to IfNotPresent once tested out to be working
+              command           = ["/bin/sh", "-c", "/usr/local/bin/apisix-snapshot.sh"]
+
+              env {
+                name  = "ETCD_ENDPOINT"
+                value = local.etcd_host
+              }
+
+              env {
+                name  = "S3_BUCKET_BASE_PATH"
+                value = var.apisix_backup_bucket_base_path
+              }
+
+              env {
+                name = "AWS_ACCESS_KEY_ID"
+                value_from {
+                  secret_key_ref {
+                    name = kubernetes_secret.apisix_backup_cron_job_secrets.metadata.0.name
+                    key  = "AWS_ACCESS_KEY_ID"
+                  }
+                }
+              }
+
+              env {
+                name = "AWS_SECRET_ACCESS_KEY"
+                value_from {
+                  secret_key_ref {
+                    name = kubernetes_secret.apisix_backup_cron_job_secrets.metadata.0.name
+                    key  = "AWS_SECRET_ACCESS_KEY"
+                  }
+                }
+              }
+            }
+          }
+        }
+      }
+    }
+  }
+
+}