From 31adb35fda463a00e1c6176c3e8804ff8574709c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ege=20G=C3=BCne=C5=9F?= Date: Mon, 11 Nov 2024 12:45:56 +0300 Subject: [PATCH] K8SPSMDB-1109: Add filesystem storage support for backups/restores (#1682) * K8SPSMDB-1109: Add filesystem storage support for backups/restores * run test * add nfs volume example to cr.yaml * delete-backup finalizer for fs backups * Implement backupSource * remove some workarounds for PBM --------- Co-authored-by: Viacheslav Sarzhan --- Makefile | 3 + ...rcona.com_perconaservermongodbbackups.yaml | 7 + ...cona.com_perconaservermongodbrestores.yaml | 7 + ...mdb.percona.com_perconaservermongodbs.yaml | 29 ++ deploy/bundle.yaml | 43 +++ deploy/cr.yaml | 11 + deploy/crd.yaml | 43 +++ deploy/cw-bundle.yaml | 43 +++ .../demand-backup-fs/compare/find-2nd.json | 4 + .../demand-backup-fs/compare/find-3rd.json | 4 + .../demand-backup-fs/compare/find-4th.json | 5 + .../demand-backup-fs/compare/find-5th.json | 5 + e2e-tests/demand-backup-fs/compare/find.json | 3 + .../compare/statefulset_some-name-rs0.yml | 277 ++++++++++++++++++ .../demand-backup-fs/conf/backup-nfs.yml | 9 + .../demand-backup-fs/conf/nfs-server.yaml | 62 ++++ e2e-tests/demand-backup-fs/conf/pitr.yml | 11 + e2e-tests/demand-backup-fs/conf/restore.yml | 7 + .../demand-backup-fs/conf/some-name.yaml | 102 +++++++ e2e-tests/demand-backup-fs/run | 162 ++++++++++ e2e-tests/functions | 7 +- e2e-tests/run-pr.csv | 1 + e2e-tests/run-release.csv | 1 + e2e-tests/version-service/conf/crd.yaml | 43 +++ .../v1/perconaservermongodbbackup_types.go | 23 +- .../v1/perconaservermongodbrestore_types.go | 4 +- pkg/apis/psmdb/v1/psmdb_types.go | 12 +- pkg/apis/psmdb/v1/zz_generated.deepcopy.go | 28 ++ pkg/controller/perconaservermongodb/backup.go | 2 +- pkg/controller/perconaservermongodb/users.go | 2 +- .../perconaservermongodbbackup/backup.go | 3 + .../perconaservermongodbbackup_controller.go | 48 ++- .../perconaservermongodbrestore/logical.go | 8 +- .../perconaservermongodbrestore_controller.go | 23 +- .../perconaservermongodbrestore/physical.go | 11 +- .../perconaservermongodbrestore/validate.go | 14 - pkg/naming/naming.go | 4 + pkg/psmdb/backup/pbm.go | 37 +-- pkg/psmdb/statefulset.go | 8 +- 39 files changed, 1044 insertions(+), 72 deletions(-) create mode 100644 e2e-tests/demand-backup-fs/compare/find-2nd.json create mode 100644 e2e-tests/demand-backup-fs/compare/find-3rd.json create mode 100644 e2e-tests/demand-backup-fs/compare/find-4th.json create mode 100644 e2e-tests/demand-backup-fs/compare/find-5th.json create mode 100644 e2e-tests/demand-backup-fs/compare/find.json create mode 100644 e2e-tests/demand-backup-fs/compare/statefulset_some-name-rs0.yml create mode 100644 e2e-tests/demand-backup-fs/conf/backup-nfs.yml create mode 100644 e2e-tests/demand-backup-fs/conf/nfs-server.yaml create mode 100644 e2e-tests/demand-backup-fs/conf/pitr.yml create mode 100644 e2e-tests/demand-backup-fs/conf/restore.yml create mode 100644 e2e-tests/demand-backup-fs/conf/some-name.yaml create mode 100755 e2e-tests/demand-backup-fs/run diff --git a/Makefile b/Makefile index 56b8fcdb43..9b609e2f57 100644 --- a/Makefile +++ b/Makefile @@ -37,6 +37,9 @@ $(DEPLOYDIR)/cw-bundle.yaml: $(DEPLOYDIR)/crd.yaml $(DEPLOYDIR)/cw-rbac.yaml $(D manifests: $(DEPLOYDIR)/crd.yaml $(DEPLOYDIR)/bundle.yaml $(DEPLOYDIR)/cw-bundle.yaml ## Put generated manifests to deploy directory +e2e-test: + IMAGE=$(IMAGE) ./e2e-tests/$(TEST)/run + ##@ Build .PHONY: build diff --git a/config/crd/bases/psmdb.percona.com_perconaservermongodbbackups.yaml b/config/crd/bases/psmdb.percona.com_perconaservermongodbbackups.yaml index b6df2dbbbe..b81937a3c4 100644 --- a/config/crd/bases/psmdb.percona.com_perconaservermongodbbackups.yaml +++ b/config/crd/bases/psmdb.percona.com_perconaservermongodbbackups.yaml @@ -95,6 +95,13 @@ spec: type: string error: type: string + filesystem: + properties: + path: + type: string + required: + - path + type: object lastTransition: format: date-time type: string diff --git a/config/crd/bases/psmdb.percona.com_perconaservermongodbrestores.yaml b/config/crd/bases/psmdb.percona.com_perconaservermongodbrestores.yaml index 31f117f72d..6261d0a41b 100644 --- a/config/crd/bases/psmdb.percona.com_perconaservermongodbrestores.yaml +++ b/config/crd/bases/psmdb.percona.com_perconaservermongodbrestores.yaml @@ -65,6 +65,13 @@ spec: type: string error: type: string + filesystem: + properties: + path: + type: string + required: + - path + type: object lastTransition: format: date-time type: string diff --git a/config/crd/bases/psmdb.percona.com_perconaservermongodbs.yaml b/config/crd/bases/psmdb.percona.com_perconaservermongodbs.yaml index d25f623393..782a146401 100644 --- a/config/crd/bases/psmdb.percona.com_perconaservermongodbs.yaml +++ b/config/crd/bases/psmdb.percona.com_perconaservermongodbs.yaml @@ -308,6 +308,13 @@ spec: required: - credentialsSecret type: object + filesystem: + properties: + path: + type: string + required: + - path + type: object s3: properties: bucket: @@ -388,6 +395,28 @@ spec: - name type: object type: array + volumeMounts: + items: + properties: + mountPath: + type: string + mountPropagation: + type: string + name: + type: string + readOnly: + type: boolean + recursiveReadOnly: + type: string + subPath: + type: string + subPathExpr: + type: string + required: + - mountPath + - name + type: object + type: array required: - enabled - image diff --git a/deploy/bundle.yaml b/deploy/bundle.yaml index 7ca6323545..bcbeb7865e 100644 --- a/deploy/bundle.yaml +++ b/deploy/bundle.yaml @@ -94,6 +94,13 @@ spec: type: string error: type: string + filesystem: + properties: + path: + type: string + required: + - path + type: object lastTransition: format: date-time type: string @@ -241,6 +248,13 @@ spec: type: string error: type: string + filesystem: + properties: + path: + type: string + required: + - path + type: object lastTransition: format: date-time type: string @@ -990,6 +1004,13 @@ spec: required: - credentialsSecret type: object + filesystem: + properties: + path: + type: string + required: + - path + type: object s3: properties: bucket: @@ -1070,6 +1091,28 @@ spec: - name type: object type: array + volumeMounts: + items: + properties: + mountPath: + type: string + mountPropagation: + type: string + name: + type: string + readOnly: + type: boolean + recursiveReadOnly: + type: string + subPath: + type: string + subPathExpr: + type: string + required: + - mountPath + - name + type: object + type: array required: - enabled - image diff --git a/deploy/cr.yaml b/deploy/cr.yaml index 3e94c89407..9d8c87b489 100644 --- a/deploy/cr.yaml +++ b/deploy/cr.yaml @@ -192,6 +192,10 @@ spec: # - name: sidecar-config # configMap: # name: myconfigmap +# - name: backup-nfs +# nfs: +# server: "nfs-service.storage.svc.cluster.local" +# path: "/psmdb-some-name-rs0" # sidecarPVCs: # - apiVersion: v1 # kind: PersistentVolumeClaim @@ -660,6 +664,13 @@ spec: # prefix: PREFIX-NAME # endpointUrl: https://accountName.blob.core.windows.net # credentialsSecret: SECRET-NAME +# backup-nfs: +# type: filesystem +# filesystem: +# path: /mnt/nfs/ +# volumeMounts: +# - mountPath: /mnt/nfs/ +# name: backup-nfs pitr: enabled: false oplogOnly: false diff --git a/deploy/crd.yaml b/deploy/crd.yaml index ce839e1be5..cc552748de 100644 --- a/deploy/crd.yaml +++ b/deploy/crd.yaml @@ -94,6 +94,13 @@ spec: type: string error: type: string + filesystem: + properties: + path: + type: string + required: + - path + type: object lastTransition: format: date-time type: string @@ -241,6 +248,13 @@ spec: type: string error: type: string + filesystem: + properties: + path: + type: string + required: + - path + type: object lastTransition: format: date-time type: string @@ -990,6 +1004,13 @@ spec: required: - credentialsSecret type: object + filesystem: + properties: + path: + type: string + required: + - path + type: object s3: properties: bucket: @@ -1070,6 +1091,28 @@ spec: - name type: object type: array + volumeMounts: + items: + properties: + mountPath: + type: string + mountPropagation: + type: string + name: + type: string + readOnly: + type: boolean + recursiveReadOnly: + type: string + subPath: + type: string + subPathExpr: + type: string + required: + - mountPath + - name + type: object + type: array required: - enabled - image diff --git a/deploy/cw-bundle.yaml b/deploy/cw-bundle.yaml index 5ab99145f9..2e863b6097 100644 --- a/deploy/cw-bundle.yaml +++ b/deploy/cw-bundle.yaml @@ -94,6 +94,13 @@ spec: type: string error: type: string + filesystem: + properties: + path: + type: string + required: + - path + type: object lastTransition: format: date-time type: string @@ -241,6 +248,13 @@ spec: type: string error: type: string + filesystem: + properties: + path: + type: string + required: + - path + type: object lastTransition: format: date-time type: string @@ -990,6 +1004,13 @@ spec: required: - credentialsSecret type: object + filesystem: + properties: + path: + type: string + required: + - path + type: object s3: properties: bucket: @@ -1070,6 +1091,28 @@ spec: - name type: object type: array + volumeMounts: + items: + properties: + mountPath: + type: string + mountPropagation: + type: string + name: + type: string + readOnly: + type: boolean + recursiveReadOnly: + type: string + subPath: + type: string + subPathExpr: + type: string + required: + - mountPath + - name + type: object + type: array required: - enabled - image diff --git a/e2e-tests/demand-backup-fs/compare/find-2nd.json b/e2e-tests/demand-backup-fs/compare/find-2nd.json new file mode 100644 index 0000000000..d0a0868e32 --- /dev/null +++ b/e2e-tests/demand-backup-fs/compare/find-2nd.json @@ -0,0 +1,4 @@ +switched to db myApp +{ "_id" : , "x" : 100500 } +{ "_id" : , "x" : 100501 } +bye diff --git a/e2e-tests/demand-backup-fs/compare/find-3rd.json b/e2e-tests/demand-backup-fs/compare/find-3rd.json new file mode 100644 index 0000000000..a5e665128c --- /dev/null +++ b/e2e-tests/demand-backup-fs/compare/find-3rd.json @@ -0,0 +1,4 @@ +switched to db myApp +{ "_id" : , "x" : 100500 } +{ "_id" : , "x" : 100502 } +bye diff --git a/e2e-tests/demand-backup-fs/compare/find-4th.json b/e2e-tests/demand-backup-fs/compare/find-4th.json new file mode 100644 index 0000000000..b83a80e5db --- /dev/null +++ b/e2e-tests/demand-backup-fs/compare/find-4th.json @@ -0,0 +1,5 @@ +switched to db myApp +{ "_id" : , "x" : 100500 } +{ "_id" : , "x" : 100502 } +{ "_id" : , "x" : 100501 } +bye diff --git a/e2e-tests/demand-backup-fs/compare/find-5th.json b/e2e-tests/demand-backup-fs/compare/find-5th.json new file mode 100644 index 0000000000..e41b9a216d --- /dev/null +++ b/e2e-tests/demand-backup-fs/compare/find-5th.json @@ -0,0 +1,5 @@ +switched to db myApp +{ "_id" : , "x" : 100500 } +{ "_id" : , "x" : 100502 } +{ "_id" : , "x" : 100503 } +bye diff --git a/e2e-tests/demand-backup-fs/compare/find.json b/e2e-tests/demand-backup-fs/compare/find.json new file mode 100644 index 0000000000..74495091bf --- /dev/null +++ b/e2e-tests/demand-backup-fs/compare/find.json @@ -0,0 +1,3 @@ +switched to db myApp +{ "_id" : , "x" : 100500 } +bye diff --git a/e2e-tests/demand-backup-fs/compare/statefulset_some-name-rs0.yml b/e2e-tests/demand-backup-fs/compare/statefulset_some-name-rs0.yml new file mode 100644 index 0000000000..15b556f506 --- /dev/null +++ b/e2e-tests/demand-backup-fs/compare/statefulset_some-name-rs0.yml @@ -0,0 +1,277 @@ +apiVersion: apps/v1 +kind: StatefulSet +metadata: + annotations: {} + generation: 1 + labels: + app.kubernetes.io/component: mongod + app.kubernetes.io/instance: some-name + app.kubernetes.io/managed-by: percona-server-mongodb-operator + app.kubernetes.io/name: percona-server-mongodb + app.kubernetes.io/part-of: percona-server-mongodb + app.kubernetes.io/replset: rs0 + name: some-name-rs0 + ownerReferences: + - controller: true + kind: PerconaServerMongoDB + name: some-name +spec: + podManagementPolicy: OrderedReady + replicas: 3 + revisionHistoryLimit: 10 + selector: + matchLabels: + app.kubernetes.io/component: mongod + app.kubernetes.io/instance: some-name + app.kubernetes.io/managed-by: percona-server-mongodb-operator + app.kubernetes.io/name: percona-server-mongodb + app.kubernetes.io/part-of: percona-server-mongodb + app.kubernetes.io/replset: rs0 + serviceName: some-name-rs0 + template: + metadata: + annotations: {} + labels: + app.kubernetes.io/component: mongod + app.kubernetes.io/instance: some-name + app.kubernetes.io/managed-by: percona-server-mongodb-operator + app.kubernetes.io/name: percona-server-mongodb + app.kubernetes.io/part-of: percona-server-mongodb + app.kubernetes.io/replset: rs0 + spec: + affinity: + podAntiAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + - labelSelector: + matchLabels: + app.kubernetes.io/component: mongod + app.kubernetes.io/instance: some-name + app.kubernetes.io/managed-by: percona-server-mongodb-operator + app.kubernetes.io/name: percona-server-mongodb + app.kubernetes.io/part-of: percona-server-mongodb + app.kubernetes.io/replset: rs0 + topologyKey: kubernetes.io/hostname + containers: + - args: + - --bind_ip_all + - --auth + - --dbpath=/data/db + - --port=27017 + - --replSet=rs0 + - --storageEngine=wiredTiger + - --relaxPermChecks + - --sslAllowInvalidCertificates + - --clusterAuthMode=x509 + - --tlsMode=preferTLS + - --enableEncryption + - --encryptionKeyFile=/etc/mongodb-encryption/encryption-key + - --wiredTigerCacheSizeGB=0.25 + - --wiredTigerIndexPrefixCompression=true + - --quiet + command: + - /opt/percona/ps-entry.sh + env: + - name: SERVICE_NAME + value: some-name + - name: MONGODB_PORT + value: "27017" + - name: MONGODB_REPLSET + value: rs0 + envFrom: + - secretRef: + name: internal-some-name-users + optional: false + imagePullPolicy: Always + livenessProbe: + exec: + command: + - /opt/percona/mongodb-healthcheck + - k8s + - liveness + - --ssl + - --sslInsecure + - --sslCAFile + - /etc/mongodb-ssl/ca.crt + - --sslPEMKeyFile + - /tmp/tls.pem + - --startupDelaySeconds + - "7200" + failureThreshold: 4 + initialDelaySeconds: 60 + periodSeconds: 30 + successThreshold: 1 + timeoutSeconds: 10 + name: mongod + ports: + - containerPort: 27017 + name: mongodb + protocol: TCP + readinessProbe: + exec: + command: + - /opt/percona/mongodb-healthcheck + - k8s + - readiness + - --component + - mongod + failureThreshold: 8 + initialDelaySeconds: 10 + periodSeconds: 3 + successThreshold: 1 + timeoutSeconds: 2 + resources: + limits: + cpu: 300m + memory: 500M + requests: + cpu: 300m + memory: 500M + securityContext: + runAsNonRoot: true + runAsUser: 1001 + terminationMessagePath: /dev/termination-log + terminationMessagePolicy: File + volumeMounts: + - mountPath: /data/db + name: mongod-data + - mountPath: /etc/mongodb-secrets + name: some-name-mongodb-keyfile + readOnly: true + - mountPath: /etc/mongodb-ssl + name: ssl + readOnly: true + - mountPath: /etc/mongodb-ssl-internal + name: ssl-internal + readOnly: true + - mountPath: /opt/percona + name: bin + - mountPath: /etc/mongodb-encryption + name: some-name-mongodb-encryption-key + readOnly: true + - mountPath: /etc/users-secret + name: users-secret-file + workingDir: /data/db + - args: + - pbm-agent-entrypoint + command: + - /opt/percona/pbm-entry.sh + env: + - name: PBM_AGENT_MONGODB_USERNAME + valueFrom: + secretKeyRef: + key: MONGODB_BACKUP_USER + name: internal-some-name-users + optional: false + - name: PBM_AGENT_MONGODB_PASSWORD + valueFrom: + secretKeyRef: + key: MONGODB_BACKUP_PASSWORD + name: internal-some-name-users + optional: false + - name: PBM_MONGODB_REPLSET + value: rs0 + - name: PBM_MONGODB_PORT + value: "27017" + - name: PBM_AGENT_SIDECAR + value: "true" + - name: PBM_AGENT_SIDECAR_SLEEP + value: "5" + - name: POD_NAME + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: metadata.name + - name: PBM_MONGODB_URI + value: mongodb://$(PBM_AGENT_MONGODB_USERNAME):$(PBM_AGENT_MONGODB_PASSWORD)@$(POD_NAME) + - name: PBM_AGENT_TLS_ENABLED + value: "true" + imagePullPolicy: Always + name: backup-agent + resources: {} + securityContext: + runAsNonRoot: true + runAsUser: 1001 + terminationMessagePath: /dev/termination-log + terminationMessagePolicy: File + volumeMounts: + - mountPath: /etc/mongodb-ssl + name: ssl + readOnly: true + - mountPath: /opt/percona + name: bin + readOnly: true + - mountPath: /data/db + name: mongod-data + - mountPath: /mnt/nfs/ + name: backup-nfs + dnsPolicy: ClusterFirst + initContainers: + - command: + - /init-entrypoint.sh + imagePullPolicy: Always + name: mongo-init + resources: + limits: + cpu: 300m + memory: 500M + requests: + cpu: 300m + memory: 500M + terminationMessagePath: /dev/termination-log + terminationMessagePolicy: File + volumeMounts: + - mountPath: /data/db + name: mongod-data + - mountPath: /opt/percona + name: bin + restartPolicy: Always + schedulerName: default-scheduler + securityContext: + fsGroup: 1001 + serviceAccount: default + serviceAccountName: default + terminationGracePeriodSeconds: 60 + volumes: + - name: some-name-mongodb-keyfile + secret: + defaultMode: 288 + optional: false + secretName: some-name-mongodb-keyfile + - emptyDir: {} + name: bin + - name: some-name-mongodb-encryption-key + secret: + defaultMode: 288 + optional: false + secretName: some-name-mongodb-encryption-key + - name: ssl + secret: + defaultMode: 288 + optional: false + secretName: some-name-ssl + - name: ssl-internal + secret: + defaultMode: 288 + optional: true + secretName: some-name-ssl-internal + - name: users-secret-file + secret: + defaultMode: 420 + secretName: internal-some-name-users + - name: backup-nfs + nfs: + path: /psmdb-some-name-rs0 + server: nfs-service.storage.svc.cluster.local + updateStrategy: + type: OnDelete + volumeClaimTemplates: + - metadata: + name: mongod-data + spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 3Gi + status: + phase: Pending diff --git a/e2e-tests/demand-backup-fs/conf/backup-nfs.yml b/e2e-tests/demand-backup-fs/conf/backup-nfs.yml new file mode 100644 index 0000000000..ea48b32c9e --- /dev/null +++ b/e2e-tests/demand-backup-fs/conf/backup-nfs.yml @@ -0,0 +1,9 @@ +apiVersion: psmdb.percona.com/v1 +kind: PerconaServerMongoDBBackup +metadata: + finalizers: + - percona.com/delete-backup + name: backup +spec: + clusterName: some-name + storageName: nfs diff --git a/e2e-tests/demand-backup-fs/conf/nfs-server.yaml b/e2e-tests/demand-backup-fs/conf/nfs-server.yaml new file mode 100644 index 0000000000..7e963f0fa9 --- /dev/null +++ b/e2e-tests/demand-backup-fs/conf/nfs-server.yaml @@ -0,0 +1,62 @@ +kind: PersistentVolumeClaim +apiVersion: v1 +metadata: + name: nfs-pvc + namespace: storage +spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 50Gi +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: nfs-server + namespace: storage +spec: + selector: + matchLabels: + app: nfs-server + template: + metadata: + labels: + app: nfs-server + spec: + containers: + - name: nfs-server + image: k8s.gcr.io/volume-nfs:0.8 + ports: + - name: nfs + containerPort: 2049 + - name: mountd + containerPort: 20048 + - name: rpcbind + containerPort: 111 + securityContext: + privileged: true + volumeMounts: + - name: storage + mountPath: /exports + volumes: + - name: storage + persistentVolumeClaim: + claimName: nfs-pvc + readOnly: false +--- +apiVersion: v1 +kind: Service +metadata: + name: nfs-service + namespace: storage +spec: + ports: + - name: nfs + port: 2049 + - name: mountd + port: 20048 + - name: rpcbind + port: 111 + selector: + app: nfs-server diff --git a/e2e-tests/demand-backup-fs/conf/pitr.yml b/e2e-tests/demand-backup-fs/conf/pitr.yml new file mode 100644 index 0000000000..78c4fff8ee --- /dev/null +++ b/e2e-tests/demand-backup-fs/conf/pitr.yml @@ -0,0 +1,11 @@ +apiVersion: psmdb.percona.com/v1 +kind: PerconaServerMongoDBRestore +metadata: + name: +spec: + clusterName: some-name + backupName: + storageName: nfs + pitr: + type: date + date: diff --git a/e2e-tests/demand-backup-fs/conf/restore.yml b/e2e-tests/demand-backup-fs/conf/restore.yml new file mode 100644 index 0000000000..32ab3c4b9a --- /dev/null +++ b/e2e-tests/demand-backup-fs/conf/restore.yml @@ -0,0 +1,7 @@ +apiVersion: psmdb.percona.com/v1 +kind: PerconaServerMongoDBRestore +metadata: + name: +spec: + clusterName: some-name + backupName: diff --git a/e2e-tests/demand-backup-fs/conf/some-name.yaml b/e2e-tests/demand-backup-fs/conf/some-name.yaml new file mode 100644 index 0000000000..a418cec330 --- /dev/null +++ b/e2e-tests/demand-backup-fs/conf/some-name.yaml @@ -0,0 +1,102 @@ +apiVersion: psmdb.percona.com/v1 +kind: PerconaServerMongoDB +metadata: + finalizers: + - percona.com/delete-psmdb-pods-in-order + name: some-name +spec: + backup: + enabled: true + image: perconalab/percona-server-mongodb-operator:main-backup + pitr: + compressionLevel: 6 + compressionType: gzip + enabled: true + oplogOnly: false + oplogSpanMin: 1 + storages: + nfs: + filesystem: + path: /mnt/nfs/ + type: filesystem + volumeMounts: + - mountPath: /mnt/nfs/ + name: backup-nfs + crVersion: 1.18.0 + image: perconalab/percona-server-mongodb-operator:main-mongod7.0 + imagePullPolicy: Always + pmm: + enabled: false + image: perconalab/pmm-client:dev-latest + serverHost: monitoring-service + replsets: + - name: rs0 + expose: + enabled: false + type: ClusterIP + podDisruptionBudget: + maxUnavailable: 1 + resources: + limits: + cpu: 300m + memory: 0.5G + requests: + cpu: 300m + memory: 0.5G + sidecarVolumes: + - name: backup-nfs + nfs: + server: "nfs-service.storage.svc.cluster.local" + path: "/psmdb-some-name-rs0" + size: 3 + volumeSpec: + persistentVolumeClaim: + resources: + requests: + storage: 3Gi + secrets: + users: some-users + sharding: + configsvrReplSet: + affinity: + antiAffinityTopologyKey: none + expose: + enabled: false + type: ClusterIP + podDisruptionBudget: + maxUnavailable: 1 + resources: + limits: + cpu: 300m + memory: 0.5G + requests: + cpu: 300m + memory: 0.5G + size: 3 + volumeSpec: + persistentVolumeClaim: + resources: + requests: + storage: 3Gi + enabled: false + mongos: + affinity: + antiAffinityTopologyKey: none + expose: + type: ClusterIP + podDisruptionBudget: + maxUnavailable: 1 + resources: + limits: + cpu: 300m + memory: 0.5G + requests: + cpu: 300m + memory: 0.5G + size: 3 + updateStrategy: SmartUpdate + upgradeOptions: + apply: disabled + schedule: 0 2 * * * + setFCV: false + versionServiceEndpoint: https://check-dev.percona.com diff --git a/e2e-tests/demand-backup-fs/run b/e2e-tests/demand-backup-fs/run new file mode 100755 index 0000000000..044837f026 --- /dev/null +++ b/e2e-tests/demand-backup-fs/run @@ -0,0 +1,162 @@ +#!/bin/bash + +set -o errexit + +test_dir=$(realpath $(dirname $0)) +. ${test_dir}/../functions +set_debug + +deploy_nfs_server() { + kubectl_bin create namespace storage + kubectl_bin apply -n storage -f ${test_dir}/conf/nfs-server.yaml + sleep 5 # wait for NFS server pod to be created + local nfsPod=$(kubectl_bin get pod -n storage -l app=nfs-server -o jsonpath={.items[].metadata.name}) + + until [[ "$(kubectl_bin get pod ${nfsPod} -n storage -o jsonpath={.status.phase})" == "Running" ]]; do + log "Waiting for ${nfsPod} to start Running" + sleep 1 + done + + kubectl_bin exec -n storage ${nfsPod} -- mkdir /exports/psmdb-${cluster}-rs0 + kubectl_bin exec -n storage ${nfsPod} -- chown 1001:1001 /exports/psmdb-${cluster}-rs0 +} + +run_recovery_check() { + local backup=$1 + local cluster=$2 + local find_prefix_before=$3 + local find_prefix_after=$4 + + write_data 100501 "${find_prefix_before}" + run_restore "${backup}" + wait_restore "${backup}" "${cluster}" + compare_mongo_cmd "find" "myApp:myPass@${cluster}-rs0.${namespace}" "${find_prefix_after}" ".svc.cluster.local" "myApp" "test" +} + +get_latest_oplog_chunk_ts() { + local cluster=$1 + echo $(kubectl_bin exec ${cluster}-rs0-0 -c backup-agent -- pbm status -o json | jq '.backups.pitrChunks.pitrChunks | last | .range.end') +} + +format_date() { + local timestamp=$1 + echo $(TZ=UTC $date -d@${timestamp} '+%Y-%m-%d %H:%M:%S') +} + +wait_for_oplogs() { + local cluster1=$1 + + local backup_last_write=$(kubectl_bin exec ${cluster}-rs0-0 -c backup-agent -- pbm status -o json | jq .backups.snapshot[0].restoreTo) + + local retries=0 + local last_chunk=$(get_latest_oplog_chunk_ts ${cluster}) + until [[ ${last_chunk} -gt ${backup_last_write} ]]; do + if [[ $retries -gt 30 ]]; then + log "Last oplog chunk ($(format_date ${last_chunk})) is not greater than last write ($(format_date ${backup_last_write}))" + exit 1 + fi + last_chunk=$(get_latest_oplog_chunk_ts ${cluster}) + retries=$((retries + 1)) + log "Waiting for last oplog chunk ($(format_date ${last_chunk})) to be greater than last write ($(format_date ${backup_last_write}))" + sleep 10 + done +} + +run_pitr_check() { + local backup=$1 + local cluster=$2 + local find_prefix=$3 + + wait_for_oplogs "${cluster}" + local target_time=$(format_date $(get_latest_oplog_chunk_ts ${cluster})) + + log "dropping test collection" + run_mongo 'use myApp\n db.test.drop()' "myApp:myPass@${cluster}-rs0.${namespace}" + + log "checking pitr... backup: ${backup} target: ${target_time}" + cat $test_dir/conf/pitr.yml \ + | yq eval ".metadata.name = \"restore-${backup}\"" \ + | yq eval ".spec.backupName = \"${backup}\"" \ + | yq eval ".spec.pitr.date = \"${target_time}\"" \ + | kubectl_bin apply -f - + + wait_restore "${backup}" "${cluster}" + compare_mongo_cmd "find" "myApp:myPass@${cluster}-rs0.${namespace}" "${find_prefix}" ".svc.cluster.local" "myApp" "test" +} + +write_data() { + local x=$1 + local find_prefix=$2 + + run_mongo \ + "use myApp\n db.test.insert({ x: ${x} })" \ + "myApp:myPass@${cluster}-rs0.${namespace}" + compare_mongo_cmd "find" "myApp:myPass@${cluster}-rs0.${namespace}" "${find_prefix}" ".svc.cluster.local" "myApp" "test" +} + + +cluster="some-name" + +create_infra ${namespace} + +kubectl_bin delete ns storage || : + +log "deploying NFS server" +deploy_nfs_server + +log 'creating secrets and start client' +kubectl_bin apply \ + -f "${conf_dir}/secrets.yml" \ + -f "${conf_dir}/client.yml" + +log "creating PSMDB cluster ${cluster}" +apply_cluster ${test_dir}/conf/${cluster}.yaml + +log 'wait for all 3 pods to start' +wait_for_running ${cluster}-rs0 3 + +log 'checking if statefulset created with expected config' +compare_kubectl statefulset/${cluster}-rs0 + +log 'creating user' +run_mongo \ + 'db.createUser({user:"myApp",pwd:"myPass",roles:[{db:"myApp",role:"readWrite"}]})' \ + "userAdmin:userAdmin123456@${cluster}-rs0.${namespace}" +sleep 2 + +log 'write initial data' +write_data 100500 "" + +wait_backup_agent ${cluster}-rs0-0 +wait_backup_agent ${cluster}-rs0-1 +wait_backup_agent ${cluster}-rs0-2 + +desc "CASE 1: Logical backup and restore" +backup_name="backup-nfs-logical" +run_backup "nfs" "${backup_name}" "logical" +wait_backup "${backup_name}" +run_recovery_check "${backup_name}" "${cluster}" "-2nd" "" + +desc "CASE 2: Logical backup and PiTR" +backup_name="backup-nfs-logical-pitr" +run_backup "nfs" "${backup_name}" "logical" +wait_backup "${backup_name}" +write_data 100502 "-3rd" +run_pitr_check "${backup_name}" "${cluster}" "-3rd" + +desc "CASE 3: Physical backup and restore" +backup_name="backup-nfs-physical" +run_backup "nfs" "${backup_name}" "physical" +wait_backup "${backup_name}" +run_recovery_check "${backup_name}" "${cluster}" "-4th" "-3rd" + +desc "CASE 4: Physical backup and PiTR" +backup_name="backup-nfs-physical-pitr" +run_backup "nfs" "${backup_name}" "physical" +wait_backup "${backup_name}" +write_data 100503 "-5th" +run_pitr_check "${backup_name}" "${cluster}" "-5th" + +destroy $namespace + +desc 'test passed' diff --git a/e2e-tests/functions b/e2e-tests/functions index d8dde927b5..90921a8444 100755 --- a/e2e-tests/functions +++ b/e2e-tests/functions @@ -68,6 +68,10 @@ else fi KUBE_VERSION=$(kubectl version -o json | jq -r '.serverVersion.major + "." + .serverVersion.minor' | $sed -r 's/[^0-9.]+//g') +log() { + echo "[$(date +%Y-%m-%dT%H:%M:%S%z)]" $* +} + set_debug() { if [[ ${DEBUG_TESTS} == 1 ]]; then set -o xtrace @@ -75,6 +79,7 @@ set_debug() { set +o xtrace fi } + version_gt() { # return true if kubernetes version equal or greater than desired if [ $(echo "${KUBE_VERSION} >= $1" | bc -l) -eq 1 ]; then @@ -819,7 +824,7 @@ compare_mongo_cmd() { | egrep -v 'I NETWORK|W NETWORK|F NETWORK|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:' \ | $sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/' \ >$tmp_dir/${command}${postfix} - diff ${test_dir}/compare/${command}${postfix}.json $tmp_dir/${command}${postfix} + diff -u ${test_dir}/compare/${command}${postfix}.json $tmp_dir/${command}${postfix} } compare_mongos_cmd() { diff --git a/e2e-tests/run-pr.csv b/e2e-tests/run-pr.csv index ee1bda43d5..3eb32cd924 100644 --- a/e2e-tests/run-pr.csv +++ b/e2e-tests/run-pr.csv @@ -8,6 +8,7 @@ cross-site-sharded data-at-rest-encryption data-sharded demand-backup +demand-backup-fs demand-backup-eks-credentials demand-backup-physical demand-backup-physical-sharded diff --git a/e2e-tests/run-release.csv b/e2e-tests/run-release.csv index f94e50aedd..c49fb773cd 100644 --- a/e2e-tests/run-release.csv +++ b/e2e-tests/run-release.csv @@ -9,6 +9,7 @@ data-at-rest-encryption data-sharded default-cr demand-backup +demand-backup-fs demand-backup-eks-credentials demand-backup-physical demand-backup-physical-sharded diff --git a/e2e-tests/version-service/conf/crd.yaml b/e2e-tests/version-service/conf/crd.yaml index ce839e1be5..cc552748de 100644 --- a/e2e-tests/version-service/conf/crd.yaml +++ b/e2e-tests/version-service/conf/crd.yaml @@ -94,6 +94,13 @@ spec: type: string error: type: string + filesystem: + properties: + path: + type: string + required: + - path + type: object lastTransition: format: date-time type: string @@ -241,6 +248,13 @@ spec: type: string error: type: string + filesystem: + properties: + path: + type: string + required: + - path + type: object lastTransition: format: date-time type: string @@ -990,6 +1004,13 @@ spec: required: - credentialsSecret type: object + filesystem: + properties: + path: + type: string + required: + - path + type: object s3: properties: bucket: @@ -1070,6 +1091,28 @@ spec: - name type: object type: array + volumeMounts: + items: + properties: + mountPath: + type: string + mountPropagation: + type: string + name: + type: string + readOnly: + type: boolean + recursiveReadOnly: + type: string + subPath: + type: string + subPathExpr: + type: string + required: + - mountPath + - name + type: object + type: array required: - enabled - image diff --git a/pkg/apis/psmdb/v1/perconaservermongodbbackup_types.go b/pkg/apis/psmdb/v1/perconaservermongodbbackup_types.go index 50cdbc6c16..2244916855 100644 --- a/pkg/apis/psmdb/v1/perconaservermongodbbackup_types.go +++ b/pkg/apis/psmdb/v1/perconaservermongodbbackup_types.go @@ -35,17 +35,18 @@ const ( // PerconaServerMongoDBBackupStatus defines the observed state of PerconaServerMongoDBBackup type PerconaServerMongoDBBackupStatus struct { - Type defs.BackupType `json:"type,omitempty"` - State BackupState `json:"state,omitempty"` - StartAt *metav1.Time `json:"start,omitempty"` - CompletedAt *metav1.Time `json:"completed,omitempty"` - LastTransition *metav1.Time `json:"lastTransition,omitempty"` - Destination string `json:"destination,omitempty"` - StorageName string `json:"storageName,omitempty"` - S3 *BackupStorageS3Spec `json:"s3,omitempty"` - Azure *BackupStorageAzureSpec `json:"azure,omitempty"` - ReplsetNames []string `json:"replsetNames,omitempty"` - PBMname string `json:"pbmName,omitempty"` + Type defs.BackupType `json:"type,omitempty"` + State BackupState `json:"state,omitempty"` + StartAt *metav1.Time `json:"start,omitempty"` + CompletedAt *metav1.Time `json:"completed,omitempty"` + LastTransition *metav1.Time `json:"lastTransition,omitempty"` + Destination string `json:"destination,omitempty"` + StorageName string `json:"storageName,omitempty"` + S3 *BackupStorageS3Spec `json:"s3,omitempty"` + Azure *BackupStorageAzureSpec `json:"azure,omitempty"` + Filesystem *BackupStorageFilesystemSpec `json:"filesystem,omitempty"` + ReplsetNames []string `json:"replsetNames,omitempty"` + PBMname string `json:"pbmName,omitempty"` // Deprecated: Use PBMPods instead PBMPod string `json:"pbmPod,omitempty"` diff --git a/pkg/apis/psmdb/v1/perconaservermongodbrestore_types.go b/pkg/apis/psmdb/v1/perconaservermongodbrestore_types.go index cede9c407f..bf528c8b8b 100644 --- a/pkg/apis/psmdb/v1/perconaservermongodbrestore_types.go +++ b/pkg/apis/psmdb/v1/perconaservermongodbrestore_types.go @@ -119,8 +119,8 @@ func (r *PerconaServerMongoDBRestore) CheckFields() error { return errors.New("backupSource destination should use s3 protocol format") } - if len(r.Spec.StorageName) == 0 && r.Spec.BackupSource.S3 == nil && r.Spec.BackupSource.Azure == nil { - return errors.New("one of storageName, backupSource.s3 or backupSource.azure is required") + if len(r.Spec.StorageName) == 0 && r.Spec.BackupSource.S3 == nil && r.Spec.BackupSource.Azure == nil && r.Spec.BackupSource.Filesystem == nil { + return errors.New("one of storageName, backupSource.s3, backupSource.azure or backupSource.filesystem is required") } } diff --git a/pkg/apis/psmdb/v1/psmdb_types.go b/pkg/apis/psmdb/v1/psmdb_types.go index ae63671cde..dec9e0ed6a 100644 --- a/pkg/apis/psmdb/v1/psmdb_types.go +++ b/pkg/apis/psmdb/v1/psmdb_types.go @@ -921,6 +921,10 @@ type BackupStorageAzureSpec struct { EndpointURL string `json:"endpointUrl,omitempty"` } +type BackupStorageFilesystemSpec struct { + Path string `json:"path"` +} + type BackupStorageType string const ( @@ -930,9 +934,10 @@ const ( ) type BackupStorageSpec struct { - Type BackupStorageType `json:"type"` - S3 BackupStorageS3Spec `json:"s3,omitempty"` - Azure BackupStorageAzureSpec `json:"azure,omitempty"` + Type BackupStorageType `json:"type"` + S3 BackupStorageS3Spec `json:"s3,omitempty"` + Azure BackupStorageAzureSpec `json:"azure,omitempty"` + Filesystem BackupStorageFilesystemSpec `json:"filesystem,omitempty"` } type PITRSpec struct { @@ -987,6 +992,7 @@ type BackupSpec struct { RuntimeClassName *string `json:"runtimeClassName,omitempty"` PITR PITRSpec `json:"pitr,omitempty"` Configuration BackupConfig `json:"configuration,omitempty"` + VolumeMounts []corev1.VolumeMount `json:"volumeMounts,omitempty"` } func (b BackupSpec) IsEnabledPITR() bool { diff --git a/pkg/apis/psmdb/v1/zz_generated.deepcopy.go b/pkg/apis/psmdb/v1/zz_generated.deepcopy.go index 3aae59cae7..267cbd6a4b 100644 --- a/pkg/apis/psmdb/v1/zz_generated.deepcopy.go +++ b/pkg/apis/psmdb/v1/zz_generated.deepcopy.go @@ -129,6 +129,13 @@ func (in *BackupSpec) DeepCopyInto(out *BackupSpec) { } in.PITR.DeepCopyInto(&out.PITR) in.Configuration.DeepCopyInto(&out.Configuration) + if in.VolumeMounts != nil { + in, out := &in.VolumeMounts, &out.VolumeMounts + *out = make([]corev1.VolumeMount, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new BackupSpec. @@ -156,6 +163,21 @@ func (in *BackupStorageAzureSpec) DeepCopy() *BackupStorageAzureSpec { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *BackupStorageFilesystemSpec) DeepCopyInto(out *BackupStorageFilesystemSpec) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new BackupStorageFilesystemSpec. +func (in *BackupStorageFilesystemSpec) DeepCopy() *BackupStorageFilesystemSpec { + if in == nil { + return nil + } + out := new(BackupStorageFilesystemSpec) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *BackupStorageS3Spec) DeepCopyInto(out *BackupStorageS3Spec) { *out = *in @@ -187,6 +209,7 @@ func (in *BackupStorageSpec) DeepCopyInto(out *BackupStorageSpec) { *out = *in in.S3.DeepCopyInto(&out.S3) out.Azure = in.Azure + out.Filesystem = in.Filesystem } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new BackupStorageSpec. @@ -1073,6 +1096,11 @@ func (in *PerconaServerMongoDBBackupStatus) DeepCopyInto(out *PerconaServerMongo *out = new(BackupStorageAzureSpec) **out = **in } + if in.Filesystem != nil { + in, out := &in.Filesystem, &out.Filesystem + *out = new(BackupStorageFilesystemSpec) + **out = **in + } if in.ReplsetNames != nil { in, out := &in.ReplsetNames, &out.ReplsetNames *out = make([]string, len(*in)) diff --git a/pkg/controller/perconaservermongodb/backup.go b/pkg/controller/perconaservermongodb/backup.go index 4aadd0681c..46608e175d 100644 --- a/pkg/controller/perconaservermongodb/backup.go +++ b/pkg/controller/perconaservermongodb/backup.go @@ -675,7 +675,7 @@ func (r *ReconcilePerconaServerMongoDB) resyncPBMIfNeeded(ctx context.Context, c command := []string{"pbm", "config", "--force-resync"} log.Info("Starting PBM resync", "command", command) - err = r.clientcmd.Exec(ctx, pod, "backup-agent", command, nil, &stdoutBuffer, &stderrBuffer, false) + err = r.clientcmd.Exec(ctx, pod, naming.ContainerBackupAgent, command, nil, &stdoutBuffer, &stderrBuffer, false) if err != nil { return errors.Wrapf(err, "start PBM resync: run %v", command) } diff --git a/pkg/controller/perconaservermongodb/users.go b/pkg/controller/perconaservermongodb/users.go index dab70e63f6..f15f14ef20 100644 --- a/pkg/controller/perconaservermongodb/users.go +++ b/pkg/controller/perconaservermongodb/users.go @@ -286,7 +286,7 @@ func (r *ReconcilePerconaServerMongoDB) updateSysUsers(ctx context.Context, cr * if changed { switch u.nameKey { case api.EnvMongoDBBackupUser: - containers = append(containers, "backup-agent") + containers = append(containers, naming.ContainerBackupAgent) case api.EnvPMMServerUser, api.EnvPMMServerAPIKey: containers = append(containers, "pmm-client") } diff --git a/pkg/controller/perconaservermongodbbackup/backup.go b/pkg/controller/perconaservermongodbbackup/backup.go index db7764bbed..77b5047057 100644 --- a/pkg/controller/perconaservermongodbbackup/backup.go +++ b/pkg/controller/perconaservermongodbbackup/backup.go @@ -124,6 +124,9 @@ func (b *Backup) Start(ctx context.Context, k8sclient client.Client, cluster *ap status.Destination = "azure://" + status.Destination } } + case api.BackupStorageFilesystem: + status.Filesystem = &stg.Filesystem + status.Destination = strings.TrimSuffix(stg.Filesystem.Path, "/") } status.Destination += "/" + status.PBMname diff --git a/pkg/controller/perconaservermongodbbackup/perconaservermongodbbackup_controller.go b/pkg/controller/perconaservermongodbbackup/perconaservermongodbbackup_controller.go index 4b66d3b35f..fe2fe90651 100644 --- a/pkg/controller/perconaservermongodbbackup/perconaservermongodbbackup_controller.go +++ b/pkg/controller/perconaservermongodbbackup/perconaservermongodbbackup_controller.go @@ -1,8 +1,10 @@ package perconaservermongodbbackup import ( + "bytes" "context" "fmt" + "strings" "time" "github.com/pkg/errors" @@ -29,6 +31,7 @@ import ( "github.com/percona/percona-server-mongodb-operator/clientcmd" psmdbv1 "github.com/percona/percona-server-mongodb-operator/pkg/apis/psmdb/v1" "github.com/percona/percona-server-mongodb-operator/pkg/naming" + "github.com/percona/percona-server-mongodb-operator/pkg/psmdb" "github.com/percona/percona-server-mongodb-operator/pkg/psmdb/backup" "github.com/percona/percona-server-mongodb-operator/version" ) @@ -379,7 +382,7 @@ func (r *ReconcilePerconaServerMongoDBBackup) checkFinalizers(ctx context.Contex for _, f := range cr.GetFinalizers() { switch f { case "delete-backup": - log.Info("The value delete-backup is deprecated and will be deleted in 1.20.0. Use percona.com/delete-backup instead") + log.Info("delete-backup finalizer is deprecated and will be deleted in 1.20.0. Use percona.com/delete-backup instead") fallthrough case naming.FinalizerDeleteBackup: if err := r.deleteBackupFinalizer(ctx, cr, cluster, b); err != nil { @@ -401,6 +404,8 @@ func (r *ReconcilePerconaServerMongoDBBackup) deleteBackupFinalizer(ctx context. return nil } + log := logf.FromContext(ctx).WithName("deleteBackup").WithValues("backup", cr.Name, "pbmName", cr.Status.PBMname) + var meta *backup.BackupMeta var err error @@ -436,22 +441,63 @@ func (r *ReconcilePerconaServerMongoDBBackup) deleteBackupFinalizer(ctx context. case cr.Status.Azure != nil: storage.Type = psmdbv1.BackupStorageAzure storage.Azure = *cr.Status.Azure + case cr.Status.Filesystem != nil: + err := r.deleteFilesystemBackup(ctx, cluster, cr) + if err != nil { + return errors.Wrap(err, "delete filesystem backup") + } + return nil } err = b.pbm.GetNSetConfig(ctx, r.client, cluster, storage) if err != nil { return errors.Wrapf(err, "set backup config with storage %s", cr.Spec.StorageName) } + // We should delete PITR oplog chunks until `LastWriteTS` of the backup, // as it's not possible to delete backup if it is a base for the PITR timeline err = b.pbm.DeletePITRChunks(ctx, meta.LastWriteTS) if err != nil { return errors.Wrap(err, "failed to delete PITR") } + log.Info("PiTR chunks deleted", "until", meta.LastWriteTS) + err = b.pbm.DeleteBackup(ctx, cr.Status.PBMname) if err != nil { return errors.Wrap(err, "failed to delete backup") } + log.Info("Backup deleted") + + return nil +} + +func (r *ReconcilePerconaServerMongoDBBackup) deleteFilesystemBackup(ctx context.Context, cluster *psmdbv1.PerconaServerMongoDB, bcp *psmdbv1.PerconaServerMongoDBBackup) error { + log := logf.FromContext(ctx) + + rsName := bcp.Status.ReplsetNames[0] + rsPods, err := psmdb.GetRSPods(ctx, r.client, cluster, rsName) + if err != nil { + return errors.Wrapf(err, "get %s pods", rsName) + } + + pod := rsPods.Items[0] + + cmd := []string{ + "pbm", + "delete-backup", + bcp.Status.PBMname, + "--yes", + } + + log.V(1).Info("Deleting filesystem backup", "pod", pod.Name, "cmd", strings.Join(cmd, " ")) + + outB := bytes.Buffer{} + errB := bytes.Buffer{} + err = r.clientcmd.Exec(ctx, &pod, naming.ContainerBackupAgent, cmd, nil, &outB, &errB, false) + if err != nil { + return errors.Wrap(err, "exec delete-backup") + } + return nil } diff --git a/pkg/controller/perconaservermongodbrestore/logical.go b/pkg/controller/perconaservermongodbrestore/logical.go index fcfeaefb42..3769bc28f6 100644 --- a/pkg/controller/perconaservermongodbrestore/logical.go +++ b/pkg/controller/perconaservermongodbrestore/logical.go @@ -14,6 +14,7 @@ import ( "github.com/percona/percona-backup-mongodb/pbm/ctrl" "github.com/percona/percona-backup-mongodb/pbm/defs" pbmErrors "github.com/percona/percona-backup-mongodb/pbm/errors" + "github.com/percona/percona-backup-mongodb/pbm/storage" psmdbv1 "github.com/percona/percona-server-mongodb-operator/pkg/apis/psmdb/v1" "github.com/percona/percona-server-mongodb-operator/pkg/psmdb/backup" @@ -134,10 +135,13 @@ func runRestore(ctx context.Context, backup string, pbmc backup.PBM, pitr *psmdb cfg, err := pbmc.GetConfig(ctx) if err != nil { + return "", errors.Wrap(err, "get PBM config") } - if err := pbmc.ResyncStorage(ctx, &cfg.Storage); err != nil { - return "", errors.Wrap(err, "resync storage") + if cfg.Storage.Type != storage.Filesystem { + if err := pbmc.ResyncStorage(ctx, &cfg.Storage); err != nil { + return "", errors.Wrap(err, "resync storage") + } } var ( diff --git a/pkg/controller/perconaservermongodbrestore/perconaservermongodbrestore_controller.go b/pkg/controller/perconaservermongodbrestore/perconaservermongodbrestore_controller.go index 2d95248c62..dcebd62106 100644 --- a/pkg/controller/perconaservermongodbrestore/perconaservermongodbrestore_controller.go +++ b/pkg/controller/perconaservermongodbrestore/perconaservermongodbrestore_controller.go @@ -220,18 +220,26 @@ func (r *ReconcilePerconaServerMongoDBRestore) getStorage(cr *psmdbv1.PerconaSer } var azure psmdbv1.BackupStorageAzureSpec var s3 psmdbv1.BackupStorageS3Spec - storageType := psmdbv1.BackupStorageS3 + var fs psmdbv1.BackupStorageFilesystemSpec + var storageType psmdbv1.BackupStorageType - if cr.Spec.BackupSource.Azure != nil { - storageType = psmdbv1.BackupStorageAzure + switch { + case cr.Spec.BackupSource.Azure != nil: azure = *cr.Spec.BackupSource.Azure - } else if cr.Spec.BackupSource.S3 != nil { + storageType = psmdbv1.BackupStorageAzure + case cr.Spec.BackupSource.S3 != nil: s3 = *cr.Spec.BackupSource.S3 + storageType = psmdbv1.BackupStorageS3 + case cr.Spec.BackupSource.Filesystem != nil: + fs = *cr.Spec.BackupSource.Filesystem + storageType = psmdbv1.BackupStorageFilesystem } + return psmdbv1.BackupStorageSpec{ - Type: storageType, - S3: s3, - Azure: azure, + Type: storageType, + S3: s3, + Azure: azure, + Filesystem: fs, }, nil } @@ -256,6 +264,7 @@ func (r *ReconcilePerconaServerMongoDBRestore) getBackup(ctx context.Context, cr StorageName: cr.Spec.StorageName, S3: cr.Spec.BackupSource.S3, Azure: cr.Spec.BackupSource.Azure, + Filesystem: cr.Spec.BackupSource.Filesystem, PBMname: backupName, }, }, nil diff --git a/pkg/controller/perconaservermongodbrestore/physical.go b/pkg/controller/perconaservermongodbrestore/physical.go index 4d5b24b9e2..ac8ef3d443 100644 --- a/pkg/controller/perconaservermongodbrestore/physical.go +++ b/pkg/controller/perconaservermongodbrestore/physical.go @@ -373,7 +373,7 @@ func (r *ReconcilePerconaServerMongoDBRestore) updateStatefulSetForPhysicalResto // remove backup-agent container pbmIdx := -1 for idx, c := range sts.Spec.Template.Spec.Containers { - if c.Name == "backup-agent" { + if c.Name == naming.ContainerBackupAgent { pbmIdx = idx break } @@ -396,6 +396,7 @@ func (r *ReconcilePerconaServerMongoDBRestore) updateStatefulSetForPhysicalResto MountPath: "/etc/pbm/", ReadOnly: true, }) + sts.Spec.Template.Spec.Containers[0].VolumeMounts = append(sts.Spec.Template.Spec.Containers[0].VolumeMounts, cluster.Spec.Backup.VolumeMounts...) sts.Spec.Template.Spec.Containers[0].Command = []string{"/opt/percona/physical-restore-ps-entry.sh"} sts.Spec.Template.Spec.Containers[0].Env = append(sts.Spec.Template.Spec.Containers[0].Env, []corev1.EnvVar{ { @@ -805,7 +806,7 @@ func (r *ReconcilePerconaServerMongoDBRestore) checkIfStatefulSetsAreReadyForPhy } for _, c := range pod.Spec.Containers { - if c.Name == "backup-agent" { + if c.Name == naming.ContainerBackupAgent { return false, nil } } @@ -822,7 +823,7 @@ func (r *ReconcilePerconaServerMongoDBRestore) getLatestChunkTS(ctx context.Cont container := "mongod" pbmBinary := "/opt/percona/pbm" for _, c := range pod.Spec.Containers { - if c.Name == "backup-agent" { + if c.Name == naming.ContainerBackupAgent { container = c.Name pbmBinary = "pbm" } @@ -863,7 +864,7 @@ func (r *ReconcilePerconaServerMongoDBRestore) disablePITR(ctx context.Context, container := "mongod" pbmBinary := "/opt/percona/pbm" for _, c := range pod.Spec.Containers { - if c.Name == "backup-agent" { + if c.Name == naming.ContainerBackupAgent { container = c.Name pbmBinary = "pbm" } @@ -893,7 +894,7 @@ func (r *ReconcilePerconaServerMongoDBRestore) waitForPBMOperationsToFinish(ctx container := "mongod" pbmBinary := "/opt/percona/pbm" for _, c := range pod.Spec.Containers { - if c.Name == "backup-agent" { + if c.Name == naming.ContainerBackupAgent { container = c.Name pbmBinary = "pbm" } diff --git a/pkg/controller/perconaservermongodbrestore/validate.go b/pkg/controller/perconaservermongodbrestore/validate.go index 7d5fc3e715..00566b5964 100644 --- a/pkg/controller/perconaservermongodbrestore/validate.go +++ b/pkg/controller/perconaservermongodbrestore/validate.go @@ -28,20 +28,6 @@ func (r *ReconcilePerconaServerMongoDBRestore) validate(ctx context.Context, cr return errors.Wrap(err, "get backup") } - // TODO: remove this if statement after https://perconadev.atlassian.net/browse/PBM-1360 is fixed - if bcp.Status.Type != defs.PhysicalBackup { - cjobs, err := backup.HasActiveJobs(ctx, r.newPBMFunc, r.client, cluster, backup.NewRestoreJob(cr), backup.NotPITRLock) - if err != nil { - return errors.Wrap(err, "check for concurrent jobs") - } - if cjobs { - if cr.Status.State != psmdbv1.RestoreStateWaiting { - log.Info("waiting to finish another backup/restore.") - } - return errWaitingRestore - } - } - if bcp.Status.Type != defs.LogicalBackup && cr.Spec.Selective != nil { return errors.New("`.spec.selective` field is supported only for logical backups") } diff --git a/pkg/naming/naming.go b/pkg/naming/naming.go index 9cf665d95c..3325a91d0c 100644 --- a/pkg/naming/naming.go +++ b/pkg/naming/naming.go @@ -8,3 +8,7 @@ const ( FinalizerDeletePVC = perconaPrefix + "delete-psmdb-pvc" FinalizerDeletePSMDBPodsInOrder = perconaPrefix + "delete-psmdb-pods-in-order" ) + +const ( + ContainerBackupAgent = "backup-agent" +) diff --git a/pkg/psmdb/backup/pbm.go b/pkg/psmdb/backup/pbm.go index 84887634a7..88d01c80a2 100644 --- a/pkg/psmdb/backup/pbm.go +++ b/pkg/psmdb/backup/pbm.go @@ -5,7 +5,6 @@ import ( "fmt" "net/url" "os" - "path" "strings" "time" @@ -23,7 +22,6 @@ import ( "github.com/percona/percona-backup-mongodb/pbm/config" "github.com/percona/percona-backup-mongodb/pbm/connect" "github.com/percona/percona-backup-mongodb/pbm/ctrl" - "github.com/percona/percona-backup-mongodb/pbm/defs" "github.com/percona/percona-backup-mongodb/pbm/lock" pbmLog "github.com/percona/percona-backup-mongodb/pbm/log" "github.com/percona/percona-backup-mongodb/pbm/oplog" @@ -31,6 +29,7 @@ import ( "github.com/percona/percona-backup-mongodb/pbm/resync" "github.com/percona/percona-backup-mongodb/pbm/storage" "github.com/percona/percona-backup-mongodb/pbm/storage/azure" + "github.com/percona/percona-backup-mongodb/pbm/storage/fs" "github.com/percona/percona-backup-mongodb/pbm/storage/s3" "github.com/percona/percona-backup-mongodb/pbm/topo" "github.com/percona/percona-backup-mongodb/pbm/util" @@ -41,7 +40,6 @@ import ( ) const ( - agentContainerName = "backup-agent" AWSAccessKeySecretKey = "AWS_ACCESS_KEY_ID" AWSSecretAccessKeySecretKey = "AWS_SECRET_ACCESS_KEY" AzureStorageAccountNameSecretKey = "AZURE_STORAGE_ACCOUNT_NAME" @@ -380,7 +378,12 @@ func GetPBMConfig(ctx context.Context, k8sclient client.Client, cluster *api.Per }, } case api.BackupStorageFilesystem: - return conf, errors.New("filesystem backup storage not supported yet, skipping storage name") + conf.Storage = config.StorageConf{ + Type: storage.Filesystem, + Filesystem: &fs.Config{ + Path: stg.Filesystem.Path, + }, + } default: return conf, errors.New("unsupported backup storage type") } @@ -389,32 +392,24 @@ func GetPBMConfig(ctx context.Context, k8sclient client.Client, cluster *api.Per } func (b *pbmC) ValidateBackup(ctx context.Context, bcp *psmdbv1.PerconaServerMongoDBBackup, cfg config.Config) error { + if cfg.Storage.Type == storage.Filesystem { + return nil + } + e := b.Logger().NewEvent(string(ctrl.CmdRestore), "", "", primitive.Timestamp{}) - backupName := bcp.Status.PBMname stg, err := util.StorageFromConfig(&cfg.Storage, e) if err != nil { return errors.Wrap(err, "storage from config") } + + backupName := bcp.Status.PBMname m, err := restore.GetMetaFromStore(stg, backupName) if err != nil { return errors.Wrap(err, "get backup metadata from storage") } - switch bcp.Status.Type { - case "", defs.LogicalBackup: - if err := backup.CheckBackupFiles(ctx, stg, m.Name); err != nil { - return errors.Wrap(err, "check backup files") - } - case defs.PhysicalBackup: - for _, rs := range m.Replsets { - f := path.Join(m.Name, rs.Name) - files, err := stg.List(f, "") - if err != nil { - return errors.Wrapf(err, "failed to list backup files at %s", f) - } - if len(files) == 0 { - return errors.Wrap(err, "no physical backup files") - } - } + + if err := backup.CheckBackupFiles(ctx, stg, m.Name); err != nil { + return errors.Wrap(err, "check backup files") } return nil diff --git a/pkg/psmdb/statefulset.go b/pkg/psmdb/statefulset.go index 22ce549ebf..2c9df7c502 100644 --- a/pkg/psmdb/statefulset.go +++ b/pkg/psmdb/statefulset.go @@ -322,15 +322,13 @@ func StatefulSpec(ctx context.Context, cr *api.PerconaServerMongoDB, replset *ap }, nil } -const agentContainerName = "backup-agent" - // backupAgentContainer creates the container object for a backup agent func backupAgentContainer(cr *api.PerconaServerMongoDB, replsetName string, tlsEnabled bool) corev1.Container { fvar := false usersSecretName := api.UserSecretName(cr) c := corev1.Container{ - Name: agentContainerName, + Name: naming.ContainerBackupAgent, Image: cr.Spec.Backup.Image, ImagePullPolicy: cr.Spec.ImagePullPolicy, Env: []corev1.EnvVar{ @@ -437,6 +435,10 @@ func backupAgentContainer(cr *api.PerconaServerMongoDB, replsetName string, tlsE }) } + if len(cr.Spec.Backup.VolumeMounts) > 0 { + c.VolumeMounts = append(c.VolumeMounts, cr.Spec.Backup.VolumeMounts...) + } + return c }