Add disabling TLS Certificate Verification support for Secure S3 Stor…

…age (#1554) Signed-off-by: Anisur Rahman <[email protected]>
stashed · Feb 9, 2024 · 8fa0835 · 8fa0835
1 parent 6d5ad36
commit 8fa0835
Show file tree

Hide file tree

Showing 24 changed files with 196 additions and 62 deletions.
diff --git a/go.mod b/go.mod
@@ -39,13 +39,13 @@ require (
 	kmodules.xyz/constants v0.0.0-20230304030334-d2d1f28732a5
 	kmodules.xyz/csi-utils v0.25.4
 	kmodules.xyz/custom-resources v0.29.0
-	kmodules.xyz/objectstore-api v0.29.0
+	kmodules.xyz/objectstore-api v0.29.1
 	kmodules.xyz/offshoot-api v0.29.0
 	kmodules.xyz/openshift v0.29.0
 	kmodules.xyz/prober v0.29.0
 	kmodules.xyz/webhook-runtime v0.29.0
 	sigs.k8s.io/controller-runtime v0.16.3
-	stash.appscode.dev/apimachinery v0.32.1-0.20240101013736-ef308633d8b2
+	stash.appscode.dev/apimachinery v0.32.1-0.20240206075719-41610d0ce38f
 )
 
 require (

diff --git a/go.sum b/go.sum
@@ -1649,8 +1649,8 @@ kmodules.xyz/custom-resources v0.29.0 h1:RaDM2+wSVXiwIvLqmkTVYpwoH83AC8wruXe2p2r
 kmodules.xyz/custom-resources v0.29.0/go.mod h1:MzZyXtxdg1PDxGk3RTTO1Xv3KiVqZnIonSwmxVbagOY=
 kmodules.xyz/go-containerregistry v0.0.12 h1:Tl32QGmSqRVm9PUEb/f3dgDeu9zW5fVzt3qmAFIE37I=
 kmodules.xyz/go-containerregistry v0.0.12/go.mod h1:KgeNg0hDsgeda+qc0NzWk0iVRdF0+ZIg/oRzGoYh78I=
-kmodules.xyz/objectstore-api v0.29.0 h1:dK53fQXdoboyW/EyBBAMjykT8u7jstKrM1DS4RJvhEU=
-kmodules.xyz/objectstore-api v0.29.0/go.mod h1:Kxmv6F7Kd/7EoKX3X2xIzhHT++zlj2qdXLcp/8avUYI=
+kmodules.xyz/objectstore-api v0.29.1 h1:uUsjf8KU0w4LYowSEOnl0AbHT3hsHIu1wNLHqGe1o6s=
+kmodules.xyz/objectstore-api v0.29.1/go.mod h1:xG+5awH1SXYKxwN/+k1FEQvzixd5tgNqEN/1LEiB2FE=
 kmodules.xyz/offshoot-api v0.29.0 h1:GHLhxxT9jU1N8+FvOCCeJNyU5g0duYS46UGrs6AHNLY=
 kmodules.xyz/offshoot-api v0.29.0/go.mod h1:5NxhBblXoDHWStx9HCDJR2KFTwYjEZ7i1Id3jelIunw=
 kmodules.xyz/openshift v0.29.0 h1:8PXjeQ+usUGkLnYUMSbZrMg+i1OIYBe9UWOeBf2FRzU=
@@ -1684,7 +1684,7 @@ sigs.k8s.io/yaml v1.1.0/go.mod h1:UJmg0vDUVViEyp3mgSv9WPwZCDxu4rQW1olrI1uml+o=
 sigs.k8s.io/yaml v1.2.0/go.mod h1:yfXDCHCao9+ENCvLSE62v9VSji2MKu5jeNfTrofGhJc=
 sigs.k8s.io/yaml v1.4.0 h1:Mk1wCc2gy/F0THH0TAp1QYyJNzRm2KCLy3o5ASXVI5E=
 sigs.k8s.io/yaml v1.4.0/go.mod h1:Ejl7/uTz7PSA4eKMyQCUTnhZYNmLIl+5c2lQPGR2BPY=
-stash.appscode.dev/apimachinery v0.32.1-0.20240101013736-ef308633d8b2 h1:dePrbjp7o57sKe33K1ppaWQK/Ely4QgxmCcab0sDOpY=
-stash.appscode.dev/apimachinery v0.32.1-0.20240101013736-ef308633d8b2/go.mod h1:hTslVqyx20fF2i2s/m0rqXD+pZwnI2oG3k5zPzsDnXQ=
+stash.appscode.dev/apimachinery v0.32.1-0.20240206075719-41610d0ce38f h1:0B35Biy3T5cNWlfMKr2LPgDaXv6C7IeDBYHOeqBaLko=
+stash.appscode.dev/apimachinery v0.32.1-0.20240206075719-41610d0ce38f/go.mod h1:5ZunLyvEChKp4LpPJq8mTKQss3bsT93S/Tqu9BRvQTA=
 x-helm.dev/apimachinery v0.0.16 h1:Eb160xcdH9fMVHak5QSWYWxoaReytch+A7kk25QWjx0=
 x-helm.dev/apimachinery v0.0.16/go.mod h1:05brgFw5oWOX7OTXT090SQojqXjbttqWfqoJo+ejBU4=
diff --git a/pkg/cmds/backup_pvc.go b/pkg/cmds/backup_pvc.go
@@ -140,6 +140,7 @@ func NewCmdBackupPVC() *cobra.Command {
 	cmd.Flags().StringVar(&opt.setupOpt.Provider, "provider", opt.setupOpt.Provider, "Backend provider (i.e. gcs, s3, azure etc)")
 	cmd.Flags().StringVar(&opt.setupOpt.Bucket, "bucket", opt.setupOpt.Bucket, "Name of the cloud bucket/container (keep empty for local backend)")
 	cmd.Flags().StringVar(&opt.setupOpt.Endpoint, "endpoint", opt.setupOpt.Endpoint, "Endpoint for s3/s3 compatible backend or REST server URL")
+	cmd.Flags().BoolVar(&opt.setupOpt.InsecureTLS, "insecure-tls", opt.setupOpt.InsecureTLS, "InsecureTLS for TLS secure s3/s3 compatible backend")
 	cmd.Flags().StringVar(&opt.setupOpt.Region, "region", opt.setupOpt.Region, "Region for s3/s3 compatible backend")
 	cmd.Flags().StringVar(&opt.setupOpt.Path, "path", opt.setupOpt.Path, "Directory inside the bucket where backed up data will be stored")
 	cmd.Flags().StringVar(&opt.setupOpt.ScratchDir, "scratch-dir", opt.setupOpt.ScratchDir, "Temporary directory")

diff --git a/pkg/cmds/restore_pvc.go b/pkg/cmds/restore_pvc.go
@@ -113,6 +113,7 @@ func NewCmdRestorePVC() *cobra.Command {
 	cmd.Flags().StringVar(&opt.setupOpt.Provider, "provider", opt.setupOpt.Provider, "Backend provider (i.e. gcs, s3, azure etc)")
 	cmd.Flags().StringVar(&opt.setupOpt.Bucket, "bucket", opt.setupOpt.Bucket, "Name of the cloud bucket/container (keep empty for local backend)")
 	cmd.Flags().StringVar(&opt.setupOpt.Endpoint, "endpoint", opt.setupOpt.Endpoint, "Endpoint for s3/s3 compatible backend or REST server URL")
+	cmd.Flags().BoolVar(&opt.setupOpt.InsecureTLS, "insecure-tls", opt.setupOpt.InsecureTLS, "InsecureTLS for TLS secure s3/s3 compatible backend")
 	cmd.Flags().StringVar(&opt.setupOpt.Region, "region", opt.setupOpt.Region, "Region for s3/s3 compatible backend")
 	cmd.Flags().StringVar(&opt.setupOpt.Path, "path", opt.setupOpt.Path, "Directory inside the bucket where backed up data will be stored")
 	cmd.Flags().StringVar(&opt.setupOpt.ScratchDir, "scratch-dir", opt.setupOpt.ScratchDir, "Temporary directory")

diff --git a/pkg/cmds/update_status.go b/pkg/cmds/update_status.go
@@ -84,6 +84,7 @@ func NewCmdUpdateStatus() *cobra.Command {
 	cmd.Flags().StringVar(&opt.SetupOpt.Provider, "provider", opt.SetupOpt.Provider, "Backend provider (i.e. gcs, s3, azure etc)")
 	cmd.Flags().StringVar(&opt.SetupOpt.Bucket, "bucket", opt.SetupOpt.Bucket, "Name of the cloud bucket/container (keep empty for local backend)")
 	cmd.Flags().StringVar(&opt.SetupOpt.Endpoint, "endpoint", opt.SetupOpt.Endpoint, "Endpoint for s3/s3 compatible backend or REST server URL")
+	cmd.Flags().BoolVar(&opt.SetupOpt.InsecureTLS, "insecure-tls", opt.SetupOpt.InsecureTLS, "InsecureTLS for TLS secure s3/s3 compatible backend")
 	cmd.Flags().StringVar(&opt.SetupOpt.Region, "region", opt.SetupOpt.Region, "Region for s3/s3 compatible backend")
 	cmd.Flags().StringVar(&opt.SetupOpt.Path, "path", opt.SetupOpt.Path, "Directory inside the bucket where backed up data will be stored")
 	cmd.Flags().StringVar(&opt.SetupOpt.ScratchDir, "scratch-dir", opt.SetupOpt.ScratchDir, "Temporary directory")

diff --git a/pkg/resolver/inputs.go b/pkg/resolver/inputs.go
@@ -164,8 +164,11 @@ func (r *TaskOptions) setRepositoryVariables() error {
 		vars[apis.RepositorySecretName] = r.Repository.Spec.Backend.StorageSecretName
 		vars[apis.RepositorySecretNamespace] = r.Repository.Namespace
 	}
-	if r.Repository.Spec.Backend.S3 != nil && r.Repository.Spec.Backend.S3.Endpoint != "" {
-		vars[apis.RepositoryEndpoint] = r.Repository.Spec.Backend.S3.Endpoint
+	if r.Repository.Spec.Backend.S3 != nil {
+		vars[apis.RepositoryInsecureTLS] = strconv.FormatBool(r.Repository.Spec.Backend.S3.InsecureTLS)
+		if r.Repository.Spec.Backend.S3.Endpoint != "" {
+			vars[apis.RepositoryEndpoint] = r.Repository.Spec.Backend.S3.Endpoint
+		}
 	}
 	endpoint, found := r.Repository.Spec.Backend.Endpoint()
 	if found {

diff --git a/pkg/util/crds.go b/pkg/util/crds.go
@@ -100,6 +100,7 @@ func updateStatusFunction(image docker.Docker) *api_v1beta1.Function {
 				"--provider=${REPOSITORY_PROVIDER:=}",
 				"--bucket=${REPOSITORY_BUCKET:=}",
 				"--endpoint=${REPOSITORY_ENDPOINT:=}",
+				"--insecure-tls=${REPOSITORY_INSECURE_TLS:=}",
 				"--region=${REPOSITORY_REGION:=}",
 				"--path=${REPOSITORY_PREFIX:=}",
 				"--enable-cache=${ENABLE_CACHE:=true}",
@@ -134,6 +135,7 @@ func pvcBackupFunction(image docker.Docker) *api_v1beta1.Function {
 				"--provider=${REPOSITORY_PROVIDER:=}",
 				"--bucket=${REPOSITORY_BUCKET:=}",
 				"--endpoint=${REPOSITORY_ENDPOINT:=}",
+				"--insecure-tls=${REPOSITORY_INSECURE_TLS:=}",
 				"--region=${REPOSITORY_REGION:=}",
 				"--path=${REPOSITORY_PREFIX:=}",
 				"--enable-cache=${ENABLE_CACHE:=true}",
@@ -184,6 +186,7 @@ func pvcRestoreFunction(image docker.Docker) *api_v1beta1.Function {
 				"--provider=${REPOSITORY_PROVIDER:=}",
 				"--bucket=${REPOSITORY_BUCKET:=}",
 				"--endpoint=${REPOSITORY_ENDPOINT:=}",
+				"--insecure-tls=${REPOSITORY_INSECURE_TLS:=}",
 				"--region=${REPOSITORY_REGION:=}",
 				"--path=${REPOSITORY_PREFIX:=}",
 				"--enable-cache=${ENABLE_CACHE:=true}",

diff --git a/pkg/util/options.go b/pkg/util/options.go
@@ -95,13 +95,15 @@ func SetupOptionsForRepository(repository api_v1alpha1.Repository, extraOpt Extr
 	}
 	endpoint, _ := repository.Spec.Backend.Endpoint()
 	region, _ := repository.Spec.Backend.Region()
+	insecureTLS := repository.Spec.Backend.InsecureTLS()
 
 	return restic.SetupOptions{
 		Provider:       provider,
 		Bucket:         bucket,
 		Path:           prefix,
 		Endpoint:       endpoint,
 		Region:         region,
+		InsecureTLS:    insecureTLS,
 		CacertFile:     extraOpt.CacertFile,
 		StorageSecret:  extraOpt.StorageSecret,
 		ScratchDir:     extraOpt.ScratchDir,

diff --git a/vendor/kmodules.xyz/objectstore-api/api/v1/generated.pb.go b/vendor/kmodules.xyz/objectstore-api/api/v1/generated.pb.go
diff --git a/vendor/kmodules.xyz/objectstore-api/api/v1/generated.proto b/vendor/kmodules.xyz/objectstore-api/api/v1/generated.proto
diff --git a/vendor/kmodules.xyz/objectstore-api/api/v1/helpers.go b/vendor/kmodules.xyz/objectstore-api/api/v1/helpers.go
@@ -163,3 +163,11 @@ func (backend Backend) Region() (string, bool) {
 	}
 	return "", false
 }
+
+// InsecureTLS returns insecureTLS of S3/S3 compatible backend
+func (backend Backend) InsecureTLS() bool {
+	if backend.S3 != nil {
+		return backend.S3.InsecureTLS
+	}
+	return false
+}
diff --git a/vendor/kmodules.xyz/objectstore-api/api/v1/openapi_generated.go b/vendor/kmodules.xyz/objectstore-api/api/v1/openapi_generated.go
diff --git a/vendor/kmodules.xyz/objectstore-api/api/v1/types.go b/vendor/kmodules.xyz/objectstore-api/api/v1/types.go
@@ -98,10 +98,11 @@ type LocalSpec struct {
 }
 
 type S3Spec struct {
-	Endpoint string `json:"endpoint" protobuf:"bytes,1,opt,name=endpoint"`
-	Bucket   string `json:"bucket" protobuf:"bytes,2,opt,name=bucket"`
-	Prefix   string `json:"prefix,omitempty" protobuf:"bytes,3,opt,name=prefix"`
-	Region   string `json:"region,omitempty" protobuf:"bytes,4,opt,name=region"`
+	Endpoint    string `json:"endpoint" protobuf:"bytes,1,opt,name=endpoint"`
+	Bucket      string `json:"bucket" protobuf:"bytes,2,opt,name=bucket"`
+	Prefix      string `json:"prefix,omitempty" protobuf:"bytes,3,opt,name=prefix"`
+	Region      string `json:"region,omitempty" protobuf:"bytes,4,opt,name=region"`
+	InsecureTLS bool   `json:"insecureTLS,omitempty" protobuf:"varint,5,opt,name=insecureTLS"`
 }
 
 type GCSSpec struct {