Merge pull request #121 from endrec/aws-cli

Use AWS cli when available

Dockerfile

@@ -20,7 +20,8 @@ ARG VERSION
 
 ENV MONGODB_TOOLS_VERSION 4.2.1-r1
 ENV GOOGLE_CLOUD_SDK_VERSION 276.0.0
-ENV AZURE_CLI_VERSION 2.5.1
+ENV AZURE_CLI_VERSION 2.13.0
+ENV AWS_CLI_VERSION 1.18.159
 ENV PATH /root/google-cloud-sdk/bin:$PATH
 
 LABEL org.label-schema.build-date=$BUILD_DATE \
@@ -55,9 +56,9 @@ RUN apk --no-cache add \
         libc6-compat \
         openssh-client \
         git \
-    && pip3 install --upgrade pip && \
-    pip install wheel && \
-    pip install crcmod && \
+    && pip3 --no-cache-dir install --upgrade pip && \
+    pip --no-cache-dir install wheel && \
+    pip --no-cache-dir install crcmod && \
     curl -O https://dl.google.com/dl/cloudsdk/channels/rapid/downloads/google-cloud-sdk-${GOOGLE_CLOUD_SDK_VERSION}-linux-x86_64.tar.gz && \
     tar xzf google-cloud-sdk-${GOOGLE_CLOUD_SDK_VERSION}-linux-x86_64.tar.gz && \
     rm google-cloud-sdk-${GOOGLE_CLOUD_SDK_VERSION}-linux-x86_64.tar.gz && \
@@ -67,10 +68,11 @@ RUN apk --no-cache add \
     gcloud config set metrics/environment github_docker_image && \
     gcloud --version
 
-# install azure-cli
-RUN apk add --virtual=build gcc libffi-dev musl-dev openssl-dev python3-dev make && \
-  pip install cffi && \
-  pip install azure-cli==${AZURE_CLI_VERSION} && \
+# install azure-cli and aws-cli
+RUN apk --no-cache add --virtual=build gcc libffi-dev musl-dev openssl-dev python3-dev make && \
+  pip --no-cache-dir install cffi && \
+  pip --no-cache-dir install azure-cli==${AZURE_CLI_VERSION} && \
+  pip --no-cache-dir install awscli==${AWS_CLI_VERSION} && \
   apk del --purge build
 
 COPY --from=0 /go/src/github.com/stefanprodan/mgob/mgob .

README.md

@@ -83,8 +83,18 @@ target:
 s3:
   url: "https://play.minio.io:9000"
   bucket: "backup"
+  # accessKey and secretKey are optional for AWS, if your Docker image has awscli
   accessKey: "Q3AM3UQ867SPQQA43P2F"
   secretKey: "zuf+tfteSlswRu7BJ86wekitnifILbZam1KYY3TG"
+  # Optional, only used for AWS (when awscli is present)
+  # The customer-managed AWS Key Management  Service (KMS) key ID that should be used to 
+  # server-side encrypt the backup in S3
+  #kmsKeyId:
+  # Optional, only used for AWS (when awscli is present)
+  # Valid choices are: STANDARD | REDUCED_REDUNDANCY | STANDARD_IA  |  ONE-
+  #     ZONE_IA  |  INTELLIGENT_TIERING  |  GLACIER | DEEP_ARCHIVE. 
+  # Defaults to 'STANDARD'
+  #storageClass: STANDARD
   # For Minio and AWS use S3v4 for GCP use S3v2
   api: "S3v4"
 # GCloud upload (optional)

cmd/mgob/mgob.go

@@ -18,7 +18,7 @@ import (
 
 var (
 	appConfig = &config.AppConfig{}
-	version   = "v1.1.0-dev"
+	version   = "v1.2.0-dev"
 )
 
 func beforeApp(c *cli.Context) error {
@@ -95,6 +95,7 @@ func start(c *cli.Context) error {
 	appConfig.TmpPath = c.String("TmpPath")
 	appConfig.DataPath = c.String("DataPath")
 	appConfig.Version = version
+	appConfig.UseAwsCli = true
 
 	log.Infof("starting with config: %+v", appConfig)
 
@@ -110,6 +111,13 @@ func start(c *cli.Context) error {
 	}
 	log.Info(info)
 
+	info, err = backup.CheckAWSClient()
+	if err != nil {
+		log.Warn(err)
+		appConfig.UseAwsCli = false
+	}
+	log.Info(info)
+
 	info, err = backup.CheckGCloudClient()
 	if err != nil {
 		log.Fatal(err)

pkg/api/backup.go

@@ -37,7 +37,7 @@ func postBackup(w http.ResponseWriter, r *http.Request) {
 
 	log.WithField("plan", planID).Info("On demand backup started")
 
-	res, err := backup.Run(plan, cfg.TmpPath, cfg.StoragePath)
+	res, err := backup.Run(plan, &cfg)
 	if err != nil {
 		log.WithField("plan", planID).Errorf("On demand backup failed %v", err)
 		if err := notifier.SendNotification(fmt.Sprintf("%v on demand backup failed", planID),

pkg/backup/backup.go

@@ -13,7 +13,9 @@ import (
 	"github.com/stefanprodan/mgob/pkg/config"
 )
 
-func Run(plan config.Plan, tmpPath string, storagePath string) (Result, error) {
+func Run(plan config.Plan, conf *config.AppConfig) (Result, error) {
+	tmpPath := conf.TmpPath
+	storagePath := conf.StoragePath
 	t1 := time.Now()
 	planDir := fmt.Sprintf("%v/%v", storagePath, plan.Name)
 
@@ -81,7 +83,7 @@ func Run(plan config.Plan, tmpPath string, storagePath string) (Result, error) {
 	}
 
 	if plan.S3 != nil {
-		s3Output, err := s3Upload(file, plan)
+		s3Output, err := s3Upload(file, plan, conf.UseAwsCli)
 		if err != nil {
 			return res, err
 		} else {

pkg/backup/checks.go

@@ -33,6 +33,19 @@ func CheckMinioClient() (string, error) {
 	return strings.Replace(string(output), "\n", " ", -1), nil
 }
 
+func CheckAWSClient() (string, error) {
+	output, err := sh.Command("/bin/sh", "-c", "aws --version").CombinedOutput()
+	if err != nil {
+		ex := ""
+		if len(output) > 0 {
+			ex = strings.Replace(string(output), "\n", " ", -1)
+		}
+		return "", errors.Wrapf(err, "aws failed %v", ex)
+	}
+
+	return strings.Replace(string(output), "\n", " ", -1), nil
+}
+
 func CheckGCloudClient() (string, error) {
 	output, err := sh.Command("/bin/sh", "-c", "gcloud --version").CombinedOutput()
 	if err != nil {

pkg/backup/s3.go

@@ -5,14 +5,77 @@ import (
 	"path/filepath"
 	"strings"
 	"time"
+	"net/url"
 
 	"github.com/codeskyblue/go-sh"
 	"github.com/pkg/errors"
 
 	"github.com/stefanprodan/mgob/pkg/config"
 )
 
-func s3Upload(file string, plan config.Plan) (string, error) {
+func s3Upload(file string, plan config.Plan, useAwsCli bool) (string, error) {
+
+	s3Url, err := url.Parse(plan.S3.URL)
+
+	if err != nil {
+		return "", errors.Wrapf(err, "invalid S3 url for plan %v: %s", plan.Name, plan.S3.URL)
+	}
+
+	if useAwsCli && strings.HasSuffix(s3Url.Hostname(), "amazonaws.com") {
+		return awsUpload(file, plan)
+	}
+
+	return minioUpload(file, plan)
+}
+
+func awsUpload(file string, plan config.Plan) (string, error) {
+
+	output := ""
+	if len(plan.S3.AccessKey) > 0 && len(plan.S3.SecretKey) > 0 {
+		// Let's use credentials given
+		configure := fmt.Sprintf("aws configure set aws_access_key_id %v && aws configure set aws_secret_access_key %v",
+			plan.S3.AccessKey, plan.S3.SecretKey)
+
+		result, err := sh.Command("/bin/sh", "-c", configure).CombinedOutput()
+		if len(result) > 0 {
+			output += strings.Replace(string(result), "\n", " ", -1)
+		}
+		if err != nil {
+			return "", errors.Wrapf(err, "aws configure for plan %v failed %s", plan.Name, output)
+		}
+	}
+
+	fileName := filepath.Base(file)
+
+	encrypt := ""
+	if len(plan.S3.KmsKeyId) > 0 {
+		encrypt = fmt.Sprintf(" --sse aws:kms --sse-kms-key-id %v", plan.S3.KmsKeyId)
+	}
+
+	storage := ""
+	if len(plan.S3.StorageClass) > 0 {
+		storage = fmt.Sprintf(" --storage-class %v", plan.S3.StorageClass)
+	}
+
+	upload := fmt.Sprintf("aws --quiet s3 cp %v s3://%v/%v%v%v",
+		file, plan.S3.Bucket, fileName, encrypt, storage)
+
+	result, err := sh.Command("/bin/sh", "-c", upload).SetTimeout(time.Duration(plan.Scheduler.Timeout) * time.Minute).CombinedOutput()
+	if len(result) > 0 {
+		output += strings.Replace(string(result), "\n", " ", -1)
+	}
+	if err != nil {
+		return "", errors.Wrapf(err, "S3 uploading %v to %v/%v failed %v", file, plan.Name, plan.S3.Bucket, output)
+	}
+
+	if strings.Contains(output, "<ERROR>") {
+		return "", errors.Errorf("S3 upload failed %v", output)
+	}
+
+	return strings.Replace(output, "\n", " ", -1), nil
+}
+
+func minioUpload(file string, plan config.Plan) (string, error) {
 
 	register := fmt.Sprintf("mc config host add %v %v %v %v --api %v",
 		plan.Name, plan.S3.URL, plan.S3.AccessKey, plan.S3.SecretKey, plan.S3.API)

pkg/config/app.go

@@ -9,4 +9,5 @@ type AppConfig struct {
 	TmpPath     string `json:"tmp_path"`
 	DataPath    string `json:"data_path"`
 	Version     string `json:"version"`
+	UseAwsCli   bool   `json:"use_aws_cli"`
 }

pkg/config/plan.go

@@ -40,11 +40,13 @@ type Scheduler struct {
 }
 
 type S3 struct {
-	Bucket    string `yaml:"bucket"`
-	AccessKey string `yaml:"accessKey"`
-	API       string `yaml:"api"`
-	SecretKey string `yaml:"secretKey"`
-	URL       string `yaml:"url"`
+	Bucket       string `yaml:"bucket"`
+	AccessKey    string `yaml:"accessKey"`
+	API          string `yaml:"api"`
+	SecretKey    string `yaml:"secretKey"`
+	URL          string `yaml:"url"`
+	KmsKeyId     string `yaml:"kmsKeyId"`
+	StorageClass string `yaml:"storageClass" validate:"omitempty,oneof=STANDARD REDUCED_REDUNDANCY STANDARD_IA ONE-ZONE_IA INTELLIGENT_TIERING GLACIER DEEP_ARCHIVE`
 }
 
 type GCloud struct {

pkg/scheduler/scheduler.go

@@ -86,7 +86,7 @@ func (b backupJob) Run() {
 	var backupLog string
 	t1 := time.Now()
 
-	res, err := backup.Run(b.plan, b.conf.TmpPath, b.conf.StoragePath)
+	res, err := backup.Run(b.plan, b.conf)
 	if err != nil {
 		status = "500"
 		backupLog = fmt.Sprintf("Backup failed %v", err)