aws s3 backend full support, issues/197

.vscode/settings.json

@@ -6,43 +6,30 @@
     "code-block-style": false
   },
   "cSpell.words": [
-    "ATIAAJSXDBUVOQ",
-    "AWSPASS",
-    "AWSUSER",
-    "Autogenerated",
-    "CERTDIR",
-    "COLORTERM",
-    "CRDs",
-    "FUNCNAME",
-    "GREEDYDATA",
-    "Gitlab",
-    "HTTPDATE",
-    "KUBECONFIG",
-    "LOGFILE",
-    "Maksym",
-    "Maxym",
-    "NS'es",
-    "OPTARG",
-    "OPTIND",
-    "PHHF",
-    "Vlasov",
-    "Vtez",
     "addgroup",
     "adduser",
     "agithub",
     "alpha",
     "antonbabenko",
     "argocd",
     "argoproj",
+    "ATIAAJSXDBUVOQ",
+    "Autogenerated",
     "autoscaling",
+    "awsbase",
     "awscli",
+    "AWSPASS",
+    "AWSUSER",
     "cdev",
+    "CERTDIR",
     "chown",
     "cidr",
     "clusterdev",
     "clustername",
+    "COLORTERM",
     "config",
     "configurables",
+    "CRDs",
     "creds",
     "datacenters",
     "digitalocean",
@@ -53,14 +40,18 @@
     "eval",
     "filebeat",
     "fstring",
+    "FUNCNAME",
     "geoip",
     "getopts",
     "gitlab",
+    "Gitlab",
     "gitorg",
     "gitpython",
     "gnupg",
     "grafana",
+    "GREEDYDATA",
     "gsub",
+    "HTTPDATE",
     "id",
     "inittf",
     "io",
@@ -70,6 +61,7 @@
     "keystore",
     "kibana",
     "kubeconfig",
+    "KUBECONFIG",
     "kubectl",
     "kubelet",
     "kubernetes",
@@ -78,7 +70,10 @@
     "libffi",
     "lifecycle",
     "linebreak",
+    "LOGFILE",
     "maintf",
+    "Maksym",
+    "Maxym",
     "minikube",
     "mkdir",
     "mmcc",
@@ -91,13 +86,17 @@
     "nloh",
     "noarch",
     "noqa",
+    "NS'es",
     "oidc",
     "only",
     "openssh",
+    "OPTARG",
+    "OPTIND",
     "orderedlist",
     "outputstf",
     "pageview",
     "permalinks",
+    "PHHF",
     "posix",
     "printf",
     "publically",
@@ -133,13 +132,26 @@
     "v",
     "varstf",
     "vcpu",
+    "Vlasov",
     "vname",
     "voatsap",
+    "Vtez",
     "wemake",
     "wordpress",
     "workdir",
     "xlarge",
     "yaml's"
   ],
-  "workbench.list.automaticKeyboardNavigation": false
+  "workbench.list.automaticKeyboardNavigation": false,
+  "editor.formatOnSave": true,
+  "editor.formatOnPaste": false,
+  "go.formatTool": "gofmt",
+  "[go]": {
+    "editor.formatOnSave": true,
+    "editor.defaultFormatter": "golang.go"
+  },
+  "editor.formatOnSaveMode": "file",
+  "notebook.formatOnSave.enabled": true,
+  "editor.defaultFormatter": "golang.go",
+  "files.autoSave": "onFocusChange"
 }

cmd/cdev/includes.go

@@ -2,7 +2,6 @@ package main
 
 import (
 	_ "github.com/shalb/cluster.dev/pkg/backend/azurerm"
-	_ "github.com/shalb/cluster.dev/pkg/backend/do"
 	_ "github.com/shalb/cluster.dev/pkg/backend/gcs"
 	_ "github.com/shalb/cluster.dev/pkg/backend/local"
 	_ "github.com/shalb/cluster.dev/pkg/backend/s3"

docs/structure-backend.md

@@ -16,21 +16,21 @@ Example configuration:
 name: my-fs
 kind: backend
 provider: local
-spec: 
+spec:
   path: /home/cluster.dev/states/
 ```
 
 A path should be absolute or relative to the directory where `cdev` is running. An absolute path must begin with `/`, and a relative with `./` or `../`. 
 
 ## Remote backend
 
-Remote backend uses remote cloud services to store the cluster state, making it accessible for team work. 
+Remote backend uses remote cloud services to store the cluster state, making it accessible for team work.
 
 Currently you can use only S3 bucket as a remote backend. In the future we plan to add other remote backend options that are listed below. 
 
-### `s3` 
+### `s3`
 
-Stores the cluster state in AWS S3 bucket. 
+Stores the cluster state in AWS S3 bucket.
 
 ```yaml
 name: aws-backend
@@ -41,7 +41,7 @@ spec:
   region: {{ .project.variables.region }}
 ```
 
-### `azurerm` 
+### `azurerm`
 
 Stores the cluster state in Microsoft Azure cloud. You can also use any options of [Terraform azurerm](https://www.terraform.io/language/settings/backends/azurerm) backend.
 
@@ -55,7 +55,7 @@ spec:
   container_name: "cdev-states"
 ```
 
-### `gcs` 
+### `gcs`
 
 Stores the cluster state in Google Cloud service. You can also use any options of [Terraform gcs](https://www.terraform.io/language/settings/backends/gcs) backend. 
 
@@ -68,17 +68,45 @@ spec:
   prefix: pref
 ```
 
-### `do` 
+### Digital Ocean Spaces and minio.
+
+To use DO spaces or minio object storage as a backend, use `s3` backend provider with additional options. See details: 
 
-Stores the cluster state in DigitalOcean spaces. 
+- [DO Spaces](https://anichakraborty.medium.com/terraform-remote-state-backup-with-digital-ocean-spaces-697e35128a6a)
+- [minio](https://ruben-rodriguez.github.io/posts/minio-s3-terraform-backend/)
+
+DO Spaces example:
 
 ```yaml
 name: do-backend
-kind: backend
-provider: do
+kind: Backend
+provider: s3
 spec:
-  bucket: cdev-states
-  region: {{ .project.variables.region }}
-  access_key: {{ env "SPACES_ACCESS_KEY_ID" }}
-  secret_key: {{ env "SPACES_SECRET_ACCESS_KEY" }}
-```
+  bucket: cdev-state
+  region: main
+  access_key: "<SPACES_SECRET_KEY>" # Optional, it's better to use environment variable 'export SPACES_SECRET_KEY="key"'
+  secret_key: "<SPACES_ACCESS_TOKEN>" # Optional, it's better to use environment variable 'export SPACES_ACCESS_TOKEN="token"'
+  endpoint: "sgp1.digitaloceanspaces.com"
+  skip_credentials_validation: true
+  skip_region_validation: true
+  skip_metadata_api_check: true
+```
+
+Minio example:
+
+```yaml
+name: minio-backend
+kind: Backend
+provider: s3
+spec:
+  bucket: cdev-state
+  region: main
+  access_key: "minioadmin"
+  secret_key: "minioadmin"
+  endpoint: http://127.0.0.1:9000
+  skip_credentials_validation: true
+  skip_region_validation: true
+  skip_metadata_api_check: true
+  force_path_style: true
+```
+

go.mod

@@ -8,8 +8,12 @@ require (
 	github.com/apex/log v1.9.0
 	github.com/apparentlymart/go-cidr v1.1.0
 	github.com/aws/aws-sdk-go v1.44.216
+	github.com/aws/aws-sdk-go-v2/service/s3 v1.38.5
 	github.com/gookit/color v1.5.2
-	github.com/hashicorp/hcl/v2 v2.16.1
+	github.com/hashicorp/aws-sdk-go-base/v2 v2.0.0-beta.34
+	github.com/hashicorp/hcl/v2 v2.16.2
+	github.com/hashicorp/terraform v1.5.6
+	github.com/kr/pretty v0.3.1
 	github.com/kylelemons/godebug v1.1.0
 	github.com/olekukonko/tablewriter v0.0.5
 	github.com/paulrademacher/climenu v0.0.0-20151110221007-a1afbb4e378b
@@ -22,8 +26,11 @@ require (
 )
 
 require (
+	cloud.google.com/go v0.110.0 // indirect
 	cloud.google.com/go/compute v1.18.0 // indirect
 	cloud.google.com/go/compute/metadata v0.2.3 // indirect
+	cloud.google.com/go/iam v0.12.0 // indirect
+	cloud.google.com/go/storage v1.28.1 // indirect
 	filippo.io/age v1.1.1 // indirect
 	github.com/Azure/azure-sdk-for-go v68.0.0+incompatible // indirect
 	github.com/Azure/go-autorest v14.2.0+incompatible // indirect
@@ -37,15 +44,43 @@ require (
 	github.com/Azure/go-autorest/logger v0.2.1 // indirect
 	github.com/Azure/go-autorest/tracing v0.6.0 // indirect
 	github.com/Masterminds/goutils v1.1.1 // indirect
+	github.com/Masterminds/semver/v3 v3.1.1 // indirect
+	github.com/Masterminds/sprig/v3 v3.2.2 // indirect
 	github.com/ProtonMail/go-crypto v0.0.0-20230217124315-7d5c6f04bbb8 // indirect
 	github.com/agext/levenshtein v1.2.3 // indirect
 	github.com/apparentlymart/go-textseg/v13 v13.0.0 // indirect
+	github.com/apparentlymart/go-versions v1.0.1 // indirect
+	github.com/armon/go-radix v1.0.0 // indirect
+	github.com/aws/aws-sdk-go-v2 v1.21.0 // indirect
+	github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.4.13 // indirect
+	github.com/aws/aws-sdk-go-v2/config v1.18.33 // indirect
+	github.com/aws/aws-sdk-go-v2/credentials v1.13.32 // indirect
+	github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.13.8 // indirect
+	github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.41 // indirect
+	github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.35 // indirect
+	github.com/aws/aws-sdk-go-v2/internal/ini v1.3.39 // indirect
+	github.com/aws/aws-sdk-go-v2/internal/v4a v1.1.4 // indirect
+	github.com/aws/aws-sdk-go-v2/service/iam v1.22.2 // indirect
+	github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.9.14 // indirect
+	github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.1.36 // indirect
+	github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.35 // indirect
+	github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.15.4 // indirect
+	github.com/aws/aws-sdk-go-v2/service/sso v1.13.2 // indirect
+	github.com/aws/aws-sdk-go-v2/service/ssooidc v1.15.2 // indirect
+	github.com/aws/aws-sdk-go-v2/service/sts v1.21.2 // indirect
+	github.com/aws/smithy-go v1.14.2 // indirect
+	github.com/bgentry/go-netrc v0.0.0-20140422174119-9fd32a8b3d3d // indirect
+	github.com/bgentry/speakeasy v0.1.0 // indirect
 	github.com/blang/semver v3.5.1+incompatible // indirect
+	github.com/bmatcuk/doublestar v1.1.5 // indirect
 	github.com/buger/goterm v1.0.4 // indirect
 	github.com/cenkalti/backoff/v3 v3.2.2 // indirect
 	github.com/cloudflare/circl v1.3.2 // indirect
+	github.com/coreos/go-systemd v0.0.0-20181012123002-c6f51f82210d // indirect
+	github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f // indirect
+	github.com/davecgh/go-spew v1.1.1 // indirect
 	github.com/dimchansky/utfbom v1.1.1 // indirect
-	github.com/fatih/color v1.14.1 // indirect
+	github.com/fatih/color v1.15.0 // indirect
 	github.com/golang-jwt/jwt/v4 v4.5.0 // indirect
 	github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect
 	github.com/golang/protobuf v1.5.2 // indirect
@@ -54,46 +89,74 @@ require (
 	github.com/googleapis/enterprise-certificate-proxy v0.2.3 // indirect
 	github.com/googleapis/gax-go/v2 v2.7.0 // indirect
 	github.com/goware/prefixer v0.0.0-20160118172347-395022866408 // indirect
+	github.com/hashicorp/aws-sdk-go-base v0.7.1 // indirect
 	github.com/hashicorp/errwrap v1.1.0 // indirect
 	github.com/hashicorp/go-cleanhttp v0.5.2 // indirect
+	github.com/hashicorp/go-getter v1.7.0 // indirect
+	github.com/hashicorp/go-hclog v1.5.0 // indirect
 	github.com/hashicorp/go-multierror v1.1.1 // indirect
 	github.com/hashicorp/go-retryablehttp v0.7.2 // indirect
 	github.com/hashicorp/go-rootcerts v1.0.2 // indirect
+	github.com/hashicorp/go-safetemp v1.0.0 // indirect
 	github.com/hashicorp/go-secure-stdlib/parseutil v0.1.7 // indirect
 	github.com/hashicorp/go-secure-stdlib/strutil v0.1.2 // indirect
 	github.com/hashicorp/go-sockaddr v1.0.2 // indirect
+	github.com/hashicorp/go-uuid v1.0.3 // indirect
+	github.com/hashicorp/go-version v1.6.0 // indirect
 	github.com/hashicorp/hcl v1.0.0 // indirect
+	github.com/hashicorp/terraform-plugin-log v0.9.0 // indirect
+	github.com/hashicorp/terraform-registry-address v0.0.0-20220623143253-7d51757b572c // indirect
+	github.com/hashicorp/terraform-svchost v0.1.0 // indirect
 	github.com/hashicorp/vault/api v1.9.0 // indirect
 	github.com/howeyc/gopass v0.0.0-20210920133722-c8aef6fb66ef // indirect
 	github.com/huandu/xstrings v1.4.0 // indirect
 	github.com/imdario/mergo v0.3.13 // indirect
 	github.com/inconshreveable/mousetrap v1.1.0 // indirect
 	github.com/jmespath/go-jmespath v0.4.0 // indirect
+	github.com/klauspost/compress v1.15.11 // indirect
+	github.com/kr/text v0.2.0 // indirect
 	github.com/lib/pq v1.10.7 // indirect
 	github.com/mattn/go-colorable v0.1.13 // indirect
 	github.com/mattn/go-isatty v0.0.17 // indirect
 	github.com/mattn/go-runewidth v0.0.14 // indirect
+	github.com/mitchellh/cli v1.1.5 // indirect
+	github.com/mitchellh/colorstring v0.0.0-20190213212951-d06e56a500db // indirect
 	github.com/mitchellh/copystructure v1.2.0 // indirect
 	github.com/mitchellh/go-homedir v1.1.0 // indirect
+	github.com/mitchellh/go-testing-interface v1.14.1 // indirect
 	github.com/mitchellh/go-wordwrap v1.0.1 // indirect
 	github.com/mitchellh/mapstructure v1.5.0 // indirect
 	github.com/mitchellh/reflectwalk v1.0.2 // indirect
 	github.com/pkg/errors v0.9.1 // indirect
 	github.com/pkg/term v1.1.0 // indirect
+	github.com/posener/complete v1.2.3 // indirect
 	github.com/rivo/uniseg v0.4.4 // indirect
+	github.com/rogpeppe/go-internal v1.9.0 // indirect
 	github.com/ryanuber/go-glob v1.0.0 // indirect
 	github.com/sergi/go-diff v1.3.1 // indirect
+	github.com/shopspring/decimal v1.3.1 // indirect
 	github.com/sirupsen/logrus v1.9.0 // indirect
+	github.com/spf13/afero v1.2.2 // indirect
+	github.com/spf13/cast v1.5.0 // indirect
 	github.com/spf13/pflag v1.0.5 // indirect
+	github.com/ulikunitz/xz v0.5.10 // indirect
+	github.com/vmihailenco/msgpack/v5 v5.3.5 // indirect
+	github.com/vmihailenco/tagparser/v2 v2.0.0 // indirect
 	github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e // indirect
+	github.com/zclconf/go-cty-yaml v1.0.3 // indirect
 	go.mozilla.org/gopgagent v0.0.0-20170926210634-4d7ea76ff71a // indirect
 	go.opencensus.io v0.24.0 // indirect
+	go.opentelemetry.io/otel v1.16.0 // indirect
+	go.opentelemetry.io/otel/trace v1.16.0 // indirect
+	golang.org/x/exp v0.0.0-20230510235704-dd950f8aeaea // indirect
+	golang.org/x/mod v0.8.0 // indirect
 	golang.org/x/net v0.8.0 // indirect
 	golang.org/x/oauth2 v0.6.0 // indirect
-	golang.org/x/sys v0.6.0 // indirect
+	golang.org/x/sys v0.11.0 // indirect
 	golang.org/x/term v0.6.0 // indirect
 	golang.org/x/text v0.8.0 // indirect
 	golang.org/x/time v0.3.0 // indirect
+	golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 // indirect
 	google.golang.org/api v0.111.0 // indirect
 	google.golang.org/appengine v1.6.7 // indirect
 	google.golang.org/genproto v0.0.0-20230306155012-7f2fa6fef1f4 // indirect