From 7d35921f9a117d0f03e8b9c47806f375d58886bb Mon Sep 17 00:00:00 2001 From: B&R Date: Wed, 1 Nov 2023 20:42:11 +0100 Subject: [PATCH] feat: Support for access tokens injected by the `Backup Maker Operator` just before the build starts / https://github.com/riotkit-org/backup-repository/issues/299 --- pkg/generate/chart/cronjob.yaml | 26 ++++++++++++++ pkg/generate/chart/job.yaml | 9 +++++ pkg/generate/chart/pod.yaml | 9 +++++ pkg/generate/e2e_test.go | 5 ++- pkg/generate/helm.go | 4 +++ pkg/generate/templates/backup/kubectl.tmpl | 6 ++++ pkg/generate/templates/backup/mysql-dump.tmpl | 6 ++++ pkg/generate/templates/backup/postgres.tmpl | 5 +++ pkg/generate/templates/backup/tar.tmpl | 5 +++ pkg/generate/templating_test.go | 6 ++-- .../postgres-with-operator-generated-jwt.yaml | 34 +++++++++++++++++++ test_generator.mk | 23 ++++++++----- 12 files changed, 126 insertions(+), 12 deletions(-) create mode 100644 pkg/generate/test_data/examples/postgres-with-operator-generated-jwt.yaml diff --git a/pkg/generate/chart/cronjob.yaml b/pkg/generate/chart/cronjob.yaml index 60748c4..b6aae1a 100644 --- a/pkg/generate/chart/cronjob.yaml +++ b/pkg/generate/chart/cronjob.yaml @@ -7,6 +7,10 @@ metadata: labels: riotkit.org/backup-maker: "true" riotkit.org/jobName: "{{ .Values.name }}" + {{ if .Values.operator.scheduledBackupName }} + # Name of the kind: ScheduledBackup that defined this execution + riotkit.org/scheduledBackupName: "{{ .Values.operator.scheduledBackupName }}" + {{ end }}> annotations: riotkit.org/action: "{{ .Values.operationType }}" spec: @@ -26,6 +30,28 @@ spec: # Pod template template: + metadata: + labels: + riotkit.org/backup-maker: "true" + riotkit.org/jobName: "{{ .Values.name }}" + + {{ if .Values.operator }} + {{ if eq .Values.operator.dynamicToken "true" }} + # ---------------------------- + # Backup Maker Operator labels + # ---------------------------- + # Notice: This label should stay only on a kind that is created during execution + # for example on Pod, Job, but not on CronJob. This label means operator will + # update the kind: Secret just before the object is created + riotkit.org/usesDynamicJWT: "true" + {{ end }} + {{ if .Values.operator.scheduledBackupName }} + # Name of the kind: ScheduledBackup that defined this execution + riotkit.org/scheduledBackupName: "{{ .Values.operator.scheduledBackupName }}" + {{ end }} + {{ end }} + annotations: + riotkit.org/action: "{{ .Values.operationType }}" spec: {{ include "podSpecTemplate" . | nindent 20 }} {{- end }} diff --git a/pkg/generate/chart/job.yaml b/pkg/generate/chart/job.yaml index 74c2922..97a760b 100644 --- a/pkg/generate/chart/job.yaml +++ b/pkg/generate/chart/job.yaml @@ -7,6 +7,15 @@ metadata: labels: riotkit.org/backup-maker: "true" riotkit.org/jobName: "{{ .Values.name }}" + + {{ if .Values.operator }} + {{ if eq .Values.operator.dynamicToken "true" }} + riotkit.org/usesDynamicJWT: "true" + {{ end }} + {{ if .Values.operator.scheduledBackupName }} + riotkit.org/scheduledBackupName: "{{ .Values.operator.scheduledBackupName }}" + {{ end }} + {{ end }} annotations: riotkit.org/action: "{{ .Values.operationType }}" spec: diff --git a/pkg/generate/chart/pod.yaml b/pkg/generate/chart/pod.yaml index d8b8701..322cc89 100644 --- a/pkg/generate/chart/pod.yaml +++ b/pkg/generate/chart/pod.yaml @@ -7,6 +7,15 @@ metadata: labels: riotkit.org/backup-maker: "true" riotkit.org/jobName: "{{ .Values.name }}" + + {{ if .Values.operator }} + {{ if eq .Values.operator.dynamicToken "true" }} + riotkit.org/usesDynamicJWT: "true" + {{ end }} + {{ if .Values.operator.scheduledBackupName }} + riotkit.org/scheduledBackupName: "{{ .Values.operator.scheduledBackupName }}" + {{ end }} + {{ end }} annotations: riotkit.org/action: "{{ .Values.operationType }}" spec: diff --git a/pkg/generate/e2e_test.go b/pkg/generate/e2e_test.go index ac87046..335dbaf 100644 --- a/pkg/generate/e2e_test.go +++ b/pkg/generate/e2e_test.go @@ -2,6 +2,7 @@ package generate_test import ( "fmt" + "github.com/docker/docker/api/types/container" "github.com/pkg/errors" "github.com/riotkit-org/br-backup-maker/utils/testingutils" "github.com/testcontainers/testcontainers-go" @@ -153,7 +154,9 @@ Repository: } cr := testcontainers.ContainerRequest{} - cr.NetworkMode = "host" + cr.HostConfigModifier = func(c *container.HostConfig) { + c.NetworkMode = "host" + } cr.Mounts = []testcontainers.ContainerMount{ { Source: testcontainers.DockerBindMountSource{ diff --git a/pkg/generate/helm.go b/pkg/generate/helm.go index 659ed68..3d413a1 100644 --- a/pkg/generate/helm.go +++ b/pkg/generate/helm.go @@ -53,6 +53,10 @@ func (t *Templating) RenderChart(script string, gpgKeyContent string, schedule s "operationType": operation, "isGPGSealedSecret": isSealedSecret, "backupMakerImage": version.GetSelfContainerImage(), + "operator": map[string]interface{}{ + "scheduledBackupName": "", + "dynamicToken": "false", + }, "serviceAccount": map[string]interface{}{ "name": "default", "create": false, diff --git a/pkg/generate/templates/backup/kubectl.tmpl b/pkg/generate/templates/backup/kubectl.tmpl index 54b5ebe..157cef4 100644 --- a/pkg/generate/templates/backup/kubectl.tmpl +++ b/pkg/generate/templates/backup/kubectl.tmpl @@ -123,7 +123,13 @@ for ns in limit_namespaces: print(" >> Packing and uploading") env = os.environ +{{ if .Repository.token }} env["BM_AUTH_TOKEN"] = "{{ .Repository.token }}" +{{ else }} +env["BM_AUTH_LOGIN"] = "{{ .Repository.login }}" +env["BM_AUTH_PASSWORD"] = "{{ .Repository.password }}" +{{ end }} + env["BM_COLLECTION_ID"] = "{{ .Repository.collectionId }}" env["BM_PASSPHRASE"] = "{{ with .Repository.passphrase }}{{ . }}{{ end }}" diff --git a/pkg/generate/templates/backup/mysql-dump.tmpl b/pkg/generate/templates/backup/mysql-dump.tmpl index bbe3485..879b770 100644 --- a/pkg/generate/templates/backup/mysql-dump.tmpl +++ b/pkg/generate/templates/backup/mysql-dump.tmpl @@ -37,7 +37,13 @@ COMMAND="tar -zcvf - ${FILES_TO_PACK} /mnt/workspace/backup-db.sql.gz" {{ end }} +{{ if .Repository.token }} export BM_AUTH_TOKEN="{{ .Repository.token }}"; +{{ else }} +export BM_AUTH_LOGIN="{{ .Repository.login }}"; +export BM_AUTH_PASSWORD="{{ .Repository.password }}"; +{{ end }} + export BM_COLLECTION_ID="{{ .Repository.collectionId }}"; export BM_PASSPHRASE="{{ with .Repository.passphrase }}{{ . }}{{ end }}"; diff --git a/pkg/generate/templates/backup/postgres.tmpl b/pkg/generate/templates/backup/postgres.tmpl index 25336d7..d615892 100644 --- a/pkg/generate/templates/backup/postgres.tmpl +++ b/pkg/generate/templates/backup/postgres.tmpl @@ -48,7 +48,12 @@ COMMAND="tar -zcvf - ${FILES_TO_PACK} /mnt/workspace/backup-db.sql.gz" {{ end }} +{{ if .Repository.token }} export BM_AUTH_TOKEN="{{ .Repository.token }}"; +{{ else }} +export BM_AUTH_LOGIN="{{ .Repository.login }}"; +export BM_AUTH_PASSWORD="{{ .Repository.password }}"; +{{ end }} export BM_COLLECTION_ID="{{ .Repository.collectionId }}"; export BM_PASSPHRASE="{{ with .Repository.passphrase }}{{ . }}{{ end }}"; diff --git a/pkg/generate/templates/backup/tar.tmpl b/pkg/generate/templates/backup/tar.tmpl index 62e9628..16e30b1 100644 --- a/pkg/generate/templates/backup/tar.tmpl +++ b/pkg/generate/templates/backup/tar.tmpl @@ -13,7 +13,12 @@ fi COMMAND="(tar --exclude='../' -zcvf - ${tarArgs}) | cat -"; +{{ if .Repository.token }} export BM_AUTH_TOKEN="{{ .Repository.token }}"; +{{ else }} +export BM_AUTH_LOGIN="{{ .Repository.login }}"; +export BM_AUTH_PASSWORD="{{ .Repository.password }}"; +{{ end }} export BM_COLLECTION_ID="{{ .Repository.collectionId }}"; export BM_PASSPHRASE="{{ with .Repository.passphrase }}{{ . }}{{ end }}"; diff --git a/pkg/generate/templating_test.go b/pkg/generate/templating_test.go index 7c7bbd0..004cf7d 100644 --- a/pkg/generate/templating_test.go +++ b/pkg/generate/templating_test.go @@ -1,7 +1,7 @@ package generate_test import ( - generate2 "github.com/riotkit-org/br-backup-maker/pkg/generate" + generate "github.com/riotkit-org/br-backup-maker/pkg/generate" "github.com/stretchr/testify/assert" "os" "testing" @@ -118,9 +118,9 @@ HelmValues: claimName: passbolt-pvc `)) - tpl := generate2.Templating{} + tpl := generate.Templating{} println(file.Name()) - cfg, err := tpl.LoadConfiguration("passbolt-pg15", file.Name(), generate2.Config{}) + cfg, err := tpl.LoadConfiguration("passbolt-pg15", file.Name(), generate.Config{}) assert.Nil(t, err) diff --git a/pkg/generate/test_data/examples/postgres-with-operator-generated-jwt.yaml b/pkg/generate/test_data/examples/postgres-with-operator-generated-jwt.yaml new file mode 100644 index 0000000..1d5c4be --- /dev/null +++ b/pkg/generate/test_data/examples/postgres-with-operator-generated-jwt.yaml @@ -0,0 +1,34 @@ +# System-specific variables, in this case specific to PostgreSQL +# ${...} and $(...) syntax will be evaluated in target environment e.g. Kubernetes POD +Params: + hostname: postgres.db.svc.cluster.local + port: 5432 + db: rkc-test + user: riotkit + password: "${DB_PASSWORD}" # injects a shell-syntax, put your password in a `kind: Secret` and mount as environment variable. You can also use $(cat /mnt/secret) syntax, be aware of newlines! + +# Generic repository access details. Everything here will land AS IS into the bash script. +# This means that any ${...} and $(...) will be executed in target environment e.g. inside Kubernetes POD +Repository: + url: "https://example.org" + token: "${BR_TOKEN}" + encryptionKeyPath: "/var/lib/backup-repository/encryption.key" + passphrase: "${GPG_PASSPHRASE}" + recipient: "your-gpg@email.org" + collectionId: "111-222-333-444" + +# Generic values for Helm used to generate jobs/pods. Those values will overwrite others. +# Notice: Environment variables with '${...}' and '$(...)' will be evaluated in LOCAL SHELL DURING BUILD +HelmValues: + name: "hello-world" + operator: + dynamicToken: "true" + scheduledBackupName: "db-1" + env: + # if specified, then will be added to `kind: Secret` and injected into POD as environment + # the value from ${GPG_PASSPHRASE} will be retrieved from the SHELL DURING THE BUILD + GPG_PASSPHRASE: "${GPG_PASSPHRASE}" + + # most secure way for Kubernetes is to not provide secrets there, but define them as environment variables + # inside SealedSecrets - all encryptedData keys will be accessible as environment variables inside container + diff --git a/test_generator.mk b/test_generator.mk index 03dafd0..98c0ff9 100644 --- a/test_generator.mk +++ b/test_generator.mk @@ -1,18 +1,25 @@ bmg_test_postgres_backup: ${BM_BIN_PATH} procedure backup \ - --definition=generate/test_data/examples/postgres.yaml \ - --template postgres + --definition=pkg/generate/test_data/examples/postgres.yaml \ + --template pg15 bmg_test_postgres_backup_k8s: ${BM_BIN_PATH} procedure backup \ - --definition=generate/test_data/examples/postgres.yaml \ - --template postgres \ + --definition=pkg/generate/test_data/examples/postgres.yaml \ + --template pg15 \ --kubernetes \ - --gpg-key-path generate/test_data/examples/gpg.key + --gpg-key-path pkg/generate/test_data/examples/gpg.key + +bmg_test_postgres_backup_dynamic_jwt_k8s: + ${BM_BIN_PATH} procedure backup \ + --definition=pkg/generate/test_data/examples/postgres-with-operator-generated-jwt.yaml \ + --template pg15 \ + --kubernetes \ + --gpg-key-path pkg/generate/test_data/examples/gpg.key bmg_test_postgres_backup_k8s_sealed_secret: ${BM_BIN_PATH} procedure backup \ - --definition=generate/test_data/examples/postgres.yaml \ - --template postgres \ + --definition=pkg/generate/test_data/examples/postgres.yaml \ + --template pg15 \ --kubernetes \ - --gpg-key-path generate/test_data/examples/valid-sealed-secret.yaml + --gpg-key-path pkg/generate/test_data/examples/valid-sealed-secret.yaml