Skip to content

Commit

Permalink
Added GCS backend (#216)
Browse files Browse the repository at this point in the history 
* cluster.dev google backend
---------

Co-authored-by: romanprog <[email protected]>
Co-authored-by: romanprog <[email protected]>
  • Loading branch information
@rayman-g @arti-shalb @romanprog
3 people authored Sep 20, 2023
1 parent 00e6c0d commit d561dca
Show file tree
Hide file tree
Showing 3 changed files with 228 additions and 21 deletions.
6 changes: 5 additions & 1 deletion go.mod
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@ module github.com/shalb/cluster.dev
go 1.20

require (
cloud.google.com/go/storage v1.28.1
github.com/Masterminds/semver v1.5.0
github.com/Masterminds/sprig v2.22.0+incompatible
github.com/apex/log v1.9.0
Expand All @@ -22,12 +23,15 @@ require (
github.com/zclconf/go-cty v1.13.0
go.mozilla.org/sops/v3 v3.7.3
golang.org/x/crypto v0.7.0
golang.org/x/oauth2 v0.6.0
google.golang.org/api v0.111.0
gopkg.in/yaml.v3 v3.0.1
)

require (
cloud.google.com/go/compute v1.18.0 // indirect
cloud.google.com/go/compute/metadata v0.2.3 // indirect
cloud.google.com/go/iam v0.12.0 // indirect
filippo.io/age v1.1.1 // indirect
github.com/Azure/azure-sdk-for-go v68.0.0+incompatible // indirect
github.com/Azure/go-autorest v14.2.0+incompatible // indirect
Expand Down Expand Up @@ -115,11 +119,11 @@ require (
go.opentelemetry.io/otel/trace v1.16.0 // indirect
golang.org/x/exp v0.0.0-20230510235704-dd950f8aeaea // indirect
golang.org/x/net v0.8.0 // indirect
golang.org/x/oauth2 v0.6.0 // indirect
golang.org/x/sys v0.11.0 // indirect
golang.org/x/term v0.6.0 // indirect
golang.org/x/text v0.8.0 // indirect
golang.org/x/time v0.3.0 // indirect
golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 // indirect
google.golang.org/api v0.111.0 // indirect
google.golang.org/appengine v1.6.7 // indirect
google.golang.org/genproto v0.0.0-20230306155012-7f2fa6fef1f4 // indirect
Expand Down
5 changes: 3 additions & 2 deletions pkg/backend/gcs/factory.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,15 +16,16 @@ func (f *Factory) New(config []byte, name string, p *project.Project) (project.B
name: name,
ProjectPtr: p,
}
state := map[string]interface{}{}
err := yaml.Unmarshal(config, &bk)
if err != nil {
return nil, utils.ResolveYamlError(config, err)
}
if bk.Prefix != "" {
bk.Prefix += "/"
}
bk.state, err = getStateMap(bk)
return &bk, nil
bk.state = state
return &bk, bk.Configure()
}

func init() {
Expand Down
238 changes: 220 additions & 18 deletions pkg/backend/gcs/gcs.go
View file
Original file line number Diff line number Diff line change
@@ -1,28 +1,131 @@
package gcs

import (
"context"
"encoding/base64"
"encoding/json"
"fmt"
"io/ioutil"
"os"
"strings"

"cloud.google.com/go/storage"
"github.com/hashicorp/hcl/v2/hclwrite"
"github.com/shalb/cluster.dev/pkg/hcltools"
"github.com/shalb/cluster.dev/pkg/project"
"github.com/shalb/cluster.dev/pkg/utils"
"github.com/zclconf/go-cty/cty"
"golang.org/x/oauth2"
"google.golang.org/api/impersonate"
"google.golang.org/api/option"
"gopkg.in/yaml.v3"

"github.com/hashicorp/hcl/v2/hclwrite"
)

// Backend - describe s3 backend for interface package.backend.
// Backend - describe GCS backend for interface package.backend.
type Backend struct {
name string `yaml:"-"`
Bucket string `yaml:"bucket"`
Credentials string `yaml:"credentials,omitempty"`
ImpersonateServiceAccount string `yaml:"impersonate_service_account,omitempty"`
AccessToken string `yaml:"access_token,omitempty"`
EncryptionKey string `yaml:"encryption_key,omitempty"`
Prefix string `yaml:"prefix"`
state map[string]interface{} `yaml:"-"`
ProjectPtr *project.Project `yaml:"-"`
storageClient *storage.Client `yaml:"-"`
storageContext context.Context
name string `yaml:"-"`
Bucket string `yaml:"bucket"`
Credentials string `yaml:"credentials,omitempty"`
ImpersonateSA string `yaml:"impersonate_service_account,omitempty"`
ImpersonateSADelegates []string `yaml:"impersonate_service_account_delegates,omitempty"`
AccessToken string `yaml:"access_token,omitempty"`
Prefix string `yaml:"prefix"`
encryptionKey []byte `yaml:"encryption_key,omitempty"`
StorageCustomEndpoint string `yaml:"storage_custom_endpoint,omitempty"`
state map[string]interface{} `yaml:"-"`
ProjectPtr *project.Project `yaml:"-"`
}

func (b *Backend) Configure() error {
if b.storageClient != nil {
return nil
}

ctx := context.Background()
b.storageContext = ctx

b.Prefix = strings.TrimLeft(b.Prefix, "/")
if b.Prefix != "" && !strings.HasSuffix(b.Prefix, "/") {
b.Prefix = b.Prefix + "/"
}

var opts []option.ClientOption
var credOptions []option.ClientOption

// Add credential source
var creds string
var tokenSource oauth2.TokenSource

if b.AccessToken != "" {
tokenSource = oauth2.StaticTokenSource(&oauth2.Token{
AccessToken: b.AccessToken,
})
} else if b.Credentials != "" {
creds = b.Credentials
} else if v := os.Getenv("GOOGLE_BACKEND_CREDENTIALS"); v != "" {
creds = v
} else {
creds = os.Getenv("GOOGLE_CREDENTIALS")
}

if tokenSource != nil {
credOptions = append(credOptions, option.WithTokenSource(tokenSource))
} else if creds != "" {
contents, err := ReadPathOrContents(creds)
if err != nil {
return fmt.Errorf("error loading credentials: %s", err.Error())
}

if !json.Valid([]byte(contents)) {
return fmt.Errorf("the string provided in credentials is neither valid json nor a valid file path")
}

credOptions = append(credOptions, option.WithCredentialsJSON([]byte(contents)))
}

if b.ImpersonateSA != "" {
ts, err := impersonate.CredentialsTokenSource(ctx, impersonate.CredentialsConfig{
TargetPrincipal: b.ImpersonateSA,
Scopes: []string{storage.ScopeReadWrite},
Delegates: b.ImpersonateSADelegates,
}, credOptions...)
if err != nil {
return err
}
opts = append(opts, option.WithTokenSource(ts))
} else {
opts = append(opts, credOptions...)
}

if b.StorageCustomEndpoint != "" {
endpoint := option.WithEndpoint(b.StorageCustomEndpoint)
opts = append(opts, endpoint)
}
client, err := storage.NewClient(b.storageContext, opts...)
if err != nil {
return fmt.Errorf("storage.NewClient() failed: %v", err.Error())
}

b.storageClient = client

key := b.encryptionKey

if len(key) > 0 {
kc, err := ReadPathOrContents(string(key))
if err != nil {
return fmt.Errorf("error loading encryption key: %s", err.Error())
}

k, err := base64.StdEncoding.DecodeString(kc)
if err != nil {
return fmt.Errorf("error decoding encryption key: %s", err.Error())
}
b.encryptionKey = k
}

return nil
}

func (b *Backend) State() map[string]interface{} {
Expand Down Expand Up @@ -99,17 +202,116 @@ func getStateMap(in Backend) (res map[string]interface{}, err error) {
}

func (b *Backend) LockState() error {
return fmt.Errorf("cdev state gcs not supported")
}
lockKey := fmt.Sprintf("cdev.%s.lock", b.ProjectPtr.Name())

// Create a context.
ctx := context.Background()

// Check if the lock object exists.
_, err := b.storageClient.Bucket(b.Bucket).Object(lockKey).Attrs(ctx)
if err == nil {
return fmt.Errorf("lock state file found, the state is locked")
}

sessionID := utils.RandString(10)

// Create the lock object with the sessionID.
lockObject := b.storageClient.Bucket(b.Bucket).Object(lockKey)
w := lockObject.NewWriter(ctx)
defer w.Close()

if _, err := w.Write([]byte(sessionID)); err != nil {
return fmt.Errorf("can't save lock state file: %v", err.Error())
}

// Sleep and read the sessionID from the lock object.
// Compare it with the generated sessionID.

return nil
}
func (b *Backend) UnlockState() error {
return fmt.Errorf("cdev state gcs not supported")
lockKey := fmt.Sprintf("cdev.%s.lock", b.ProjectPtr.Name())

// Create a context.
ctx := context.Background()

// Delete the lock object.
return b.storageClient.Bucket(b.Bucket).Object(lockKey).Delete(ctx)
}

func (b *Backend) WriteState(stateData string) error {
return fmt.Errorf("cdev state gcs not supported")
}
stateKey := fmt.Sprintf("cdev.%s.state", b.ProjectPtr.Name())

// Create a context.
ctx := context.Background()

// Create or overwrite the state object with stateData.
stateObject := b.storageClient.Bucket(b.Bucket).Object(stateKey)
w := stateObject.NewWriter(ctx)
defer w.Close()

if _, err := w.Write([]byte(stateData)); err != nil {
return fmt.Errorf("can't save state file: %v", err.Error())
}

return nil
}
func (b *Backend) ReadState() (string, error) {
return "", fmt.Errorf("cdev state gcs not supported")
stateKey := fmt.Sprintf("cdev.%s.state", b.ProjectPtr.Name())

// Create a context.
ctx := context.Background()

// Check if the object exists.
_, err := b.storageClient.Bucket(b.Bucket).Object(stateKey).Attrs(ctx)
if err != nil {
if err == storage.ErrObjectNotExist {
// fmt.Printf("Object '%s' does not exist in bucket '%s'\n", stateKey, b.Bucket)
return "", nil
}
fmt.Printf("Error checking object existence: %v\n", err)
return "", err
}

// Read the state object.
stateObject := b.storageClient.Bucket(b.Bucket).Object(stateKey)
r, err := stateObject.NewReader(ctx)
if err != nil {
return "", err
}
defer r.Close()

stateData, err := ioutil.ReadAll(r)
if err != nil {
return "", err
}

return string(stateData), nil
}

// ReadPathOrContents reads the contents of a file if the input is a file path,
// or returns the input as is if it's not a file path.
func ReadPathOrContents(input string) (string, error) {
// Check if the input contains a path separator (e.g., '/').
if strings.Contains(input, "/") {
// Treat input as a file path and read its contents.
contents, err := readFileContents(input)
if err != nil {
return "", err
}
return contents, nil
}

// Input is not a file path, return it as is.
return input, nil
}

// readFileContents reads the contents of a file given its path.
func readFileContents(filePath string) (string, error) {
// Read the file contents.
contents, err := ioutil.ReadFile(filePath)
if err != nil {
return "", err
}
return string(contents), nil
}

0 comments on commit d561dca

Please sign in to comment.