From 15e49eea333ca6afb1d713d6223efed466595882 Mon Sep 17 00:00:00 2001 From: Chris Mellard Date: Sat, 18 Jul 2020 23:45:22 +1200 Subject: [PATCH] Added in empty implementation of AKSBucketProvider Implemented Container Exists check Implemented Azure container creation Added in logic to retrieve storage account keys Encapsulated the storage account parsing down close the CLI which appears to be the only part that requires them as separate arguments Fixing up houndci review --- pkg/cloud/aks/aks.go | 121 +++++++++++++++++++++++ pkg/cloud/aks/storage/bucket_provider.go | 76 ++++++++++++++ pkg/cloud/factory/factory.go | 3 + 3 files changed, 200 insertions(+) create mode 100644 pkg/cloud/aks/storage/bucket_provider.go diff --git a/pkg/cloud/aks/aks.go b/pkg/cloud/aks/aks.go index 477e792426..21d635800e 100644 --- a/pkg/cloud/aks/aks.go +++ b/pkg/cloud/aks/aks.go @@ -3,6 +3,9 @@ package aks import ( b64 "encoding/base64" "encoding/json" + "fmt" + "github.com/pkg/errors" + "regexp" "strings" "github.com/jenkins-x/jx-logging/pkg/log" @@ -28,6 +31,10 @@ type acr struct { Name string `json:"name"` } +type containerExists struct { + Exists bool `json:"exists"` +} + type password struct { Name string `json:"name"` Value string `json:"value"` @@ -46,6 +53,17 @@ type config struct { Auths map[string]*auth `json:"auths,omitempty"` } +// Interface for Azure Storage commands +type AzureStorage interface { + ContainerExists(bucketURL string) (bool, error) + CreateContainer(bucketURL string) error + GetStorageAccessKey(storageAccount string) (string, error) +} + +var ( + azureContainerURIRegExp = regexp.MustCompile(`https://(?P\w+)\.blob\.core\.windows\.net/(?P\w+)`) +) + // NewAzureRunnerWithCommander specific the command runner for Azure CLI. func NewAzureRunnerWithCommander(runner util.Commander) *AzureRunner { return &AzureRunner{ @@ -242,3 +260,106 @@ func (az *AzureRunner) azureCLI(args ...string) (string, error) { az.Runner.SetArgs(args) return az.Runner.RunWithoutRetry() } + +func parseContainerURL(bucketURL string) (string, string, error) { + match := azureContainerURIRegExp.FindStringSubmatch(bucketURL) + if len(match) == 3 { + return match[1], match[2], nil + } + return "", "", errors.New(fmt.Sprintf("Azure Blob Container Url %s could not be parsed to determine storage account and container name", bucketURL)) +} + +// ContainerExists checks if an Azure Storage Container exists +func (az *AzureRunner) ContainerExists(bucketURL string) (bool, error) { + storageAccount, bucketName, err := parseContainerURL(bucketURL) + if err != nil { + return false, err + } + + accessKey, err := az.GetStorageAccessKey(storageAccount) + if err != nil { + return false, err + } + + bucketExistsArgs := []string{ + "storage", + "container", + "exists", + "-n", + bucketName, + "--account-name", + storageAccount, + "--account-key", + accessKey, + } + + cmdResult, err := az.azureCLI(bucketExistsArgs...) + + if err != nil { + log.Logger().Infof("Error checking bucket exists: %s, %s", cmdResult, err) + return false, err + } + + containerExists := containerExists{} + err = json.Unmarshal([]byte(cmdResult), &containerExists) + if err != nil { + return false, errors.Wrap(err, "unmarshalling Azure container exists command") + } + return containerExists.Exists, nil + +} + +func (az *AzureRunner) CreateContainer(bucketURL string) error { + storageAccount, bucketName, err := parseContainerURL(bucketURL) + if err != nil { + return err + } + + accessKey, err := az.GetStorageAccessKey(storageAccount) + if err != nil { + return err + } + + createContainerArgs := []string{ + "storage", + "container", + "create", + "-n", + bucketName, + "--account-name", + storageAccount, + "--fail-on-exist", + "--account-key", + accessKey, + } + + cmdResult, err := az.azureCLI(createContainerArgs...) + + if err != nil { + log.Logger().Infof("Error creating bucket: %s, %s", cmdResult, err) + return err + } + + return nil +} + +func (az *AzureRunner) GetStorageAccessKey(storageAccount string) (string, error) { + getStorageAccessKeyArgs := []string{ + "storage", + "account", + "keys", + "list", + "-n", + storageAccount, + "--query", + "[?keyName=='key1'].value | [0]", + } + + cmdResult, err := az.azureCLI(getStorageAccessKeyArgs...) + + if err != nil { + return "", err + } + + return cmdResult, nil +} diff --git a/pkg/cloud/aks/storage/bucket_provider.go b/pkg/cloud/aks/storage/bucket_provider.go new file mode 100644 index 0000000000..5833e8e276 --- /dev/null +++ b/pkg/cloud/aks/storage/bucket_provider.go @@ -0,0 +1,76 @@ +package storage + +import ( + "fmt" + "github.com/jenkins-x/jx-logging/pkg/log" + "github.com/jenkins-x/jx/v2/pkg/cloud/aks" + "github.com/jenkins-x/jx/v2/pkg/cloud/buckets" + "github.com/jenkins-x/jx/v2/pkg/config" + "github.com/jenkins-x/jx/v2/pkg/util" + "github.com/pkg/errors" + uuid "github.com/satori/go.uuid" + "io" + "strings" +) + +type AKSBucketProvider struct { + Requirements *config.RequirementsConfig + AzureStorage aks.AzureStorage +} + +// CreateNewBucketForCluster creates a new dynamic bucket +func (b *AKSBucketProvider) CreateNewBucketForCluster(clusterName string, bucketKind string) (string, error) { + uuid4, _ := uuid.NewV4() + bucketName := fmt.Sprintf("%s-%s-%s", clusterName, bucketKind, uuid4.String()) + + // Max length is 63, https://docs.microsoft.com/en-us/rest/api/storageservices/naming-and-referencing-containers--blobs--and-metadata + if len(bucketName) > 63 { + bucketName = bucketName[:63] + } + bucketName = strings.TrimRight(bucketName, "-") + bucketURL := fmt.Sprintf("https://%s.blob.core.windows.net/%s", b.Requirements.Velero.ServiceAccount, bucketName) + err := b.EnsureBucketIsCreated(bucketURL) + if err != nil { + return bucketURL, errors.Wrapf(err, "failed to create bucket %s", bucketURL) + } + + return bucketURL, nil +} + +// EnsureBucketIsCreated ensures the bucket URL is created +func (b *AKSBucketProvider) EnsureBucketIsCreated(bucketURL string) error { + + exists, err := b.AzureStorage.ContainerExists(bucketURL) + if err != nil { + return errors.Wrap(err, "checking if the provided container exists") + } + if exists { + return nil + } + + log.Logger().Infof("The bucket %s does not exist so lets create it", util.ColorInfo(bucketURL)) + err = b.AzureStorage.CreateContainer(bucketURL) + if err != nil { + return errors.Wrapf(err, "there was a problem creating the bucket with URL %s", + bucketURL) + } + return nil +} + +// UploadFileToBucket is yet to be implemented for this provider +func (b *AKSBucketProvider) UploadFileToBucket(r io.Reader, outputName string, bucketURL string) (string, error) { + return "", nil +} + +// DownloadFileFromBucket is yet to be implemented for this provider +func (b *AKSBucketProvider) DownloadFileFromBucket(bucketURL string) (io.ReadCloser, error) { + return nil, nil +} + +// NewAKSBucketProvider create a new provider for AKS +func NewAKSBucketProvider(requirements *config.RequirementsConfig) buckets.Provider { + return &AKSBucketProvider{ + Requirements: requirements, + AzureStorage: aks.NewAzureRunner(), + } +} diff --git a/pkg/cloud/factory/factory.go b/pkg/cloud/factory/factory.go index fbe865a533..e343568bae 100644 --- a/pkg/cloud/factory/factory.go +++ b/pkg/cloud/factory/factory.go @@ -4,6 +4,7 @@ import ( v1 "github.com/jenkins-x/jx-api/pkg/apis/jenkins.io/v1" "github.com/jenkins-x/jx-logging/pkg/log" "github.com/jenkins-x/jx/v2/pkg/cloud" + aksStorage "github.com/jenkins-x/jx/v2/pkg/cloud/aks/storage" amazonStorage "github.com/jenkins-x/jx/v2/pkg/cloud/amazon/storage" "github.com/jenkins-x/jx/v2/pkg/cloud/buckets" "github.com/jenkins-x/jx/v2/pkg/cloud/gke/storage" @@ -23,6 +24,8 @@ func NewBucketProvider(requirements *config.RequirementsConfig) buckets.Provider fallthrough case cloud.AWS: return amazonStorage.NewAmazonBucketProvider(requirements) + case cloud.AKS: + return aksStorage.NewAKSBucketProvider(requirements) default: // we have an implementation for GKE / EKS but not for AKS so we should fall back to default // but we don't have every func implemented