Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

chore: Storage integration with custom protocol #3213

Merged
merged 4 commits into from
Nov 26, 2024
Merged

chore: Storage integration with custom protocol #3213

Show file tree
Hide file tree
Changes from 2 commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
c09da00
wip
sfc-gh-jcieslak Nov 21, 2024
c35bf68
wip
sfc-gh-jcieslak Nov 21, 2024
2e60e0b
changes after review
sfc-gh-jcieslak Nov 21, 2024
db785c3
Fix tests
sfc-gh-jcieslak Nov 22, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 4 additions & 4 deletions pkg/acceptance/helpers/storage_integration_client.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -54,10 +54,10 @@ func (c *StorageIntegrationClient) CreateS3(t *testing.T, awsBucketUrl, awsRoleA

id := c.ids.RandomAccountObjectIdentifier()
req := sdk.NewCreateStorageIntegrationRequest(id, true, s3AllowedLocations).
WithIfNotExists(sdk.Bool(true)).
WithS3StorageProviderParams(sdk.NewS3StorageParamsRequest(awsRoleArn)).
WithIfNotExists(true).
WithS3StorageProviderParams(*sdk.NewS3StorageParamsRequest(sdk.RegularS3Protocol, awsRoleArn)).
WithStorageBlockedLocations(s3BlockedLocations).
WithComment(sdk.String("some comment"))
WithComment("some comment")

err := c.client().Create(ctx, req)
require.NoError(t, err)
Expand All @@ -73,7 +73,7 @@ func (c *StorageIntegrationClient) DropFunc(t *testing.T, id sdk.AccountObjectId
ctx := context.Background()

return func() {
err := c.client().Drop(ctx, sdk.NewDropStorageIntegrationRequest(id).WithIfExists(sdk.Bool(true)))
err := c.client().Drop(ctx, sdk.NewDropStorageIntegrationRequest(id).WithIfExists(true))
require.NoError(t, err)
}
}
51 changes: 28 additions & 23 deletions pkg/resources/storage_integration.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@ import (
"context"
"fmt"
"log"
"slices"
"strings"

"github.com/Snowflake-Labs/terraform-provider-snowflake/pkg/internal/provider"
Expand Down Expand Up @@ -52,12 +53,11 @@ var storageIntegrationSchema = map[string]*schema.Schema{
Optional: true,
Description: "Explicitly prohibits external stages that use the integration from referencing one or more storage locations.",
},
// TODO (SNOW-1015282): Remove S3gov option before going into V1
"storage_provider": {
Type: schema.TypeString,
Required: true,
ForceNew: true,
ValidateFunc: validation.StringInSlice([]string{"S3", "S3gov", "GCS", "AZURE", "S3GOV"}, false),
Type: schema.TypeString,
Required: true,
ForceNew: true,
ValidateDiagFunc: StringInSlice(append(sdk.AsStringList(sdk.AllS3Protocols), "GCS", "AZURE"), true),
sfc-gh-jmichalak marked this conversation as resolved.
Show resolved Hide resolved
},
"storage_aws_external_id": {
Type: schema.TypeString,
Expand Down Expand Up @@ -140,7 +140,7 @@ func CreateStorageIntegration(d *schema.ResourceData, meta any) error {
req := sdk.NewCreateStorageIntegrationRequest(name, enabled, storageAllowedLocations)

if v, ok := d.GetOk("comment"); ok {
req.WithComment(sdk.String(v.(string)))
req.WithComment(v.(string))
}

if _, ok := d.GetOk("storage_blocked_locations"); ok {
Expand All @@ -154,28 +154,33 @@ func CreateStorageIntegration(d *schema.ResourceData, meta any) error {
req.WithStorageBlockedLocations(storageBlockedLocations)
}

storageProvider := d.Get("storage_provider").(string)
storageProvider := strings.ToUpper(d.Get("storage_provider").(string))
sfc-gh-jmichalak marked this conversation as resolved.
Show resolved Hide resolved

switch {
case slices.Contains(sdk.AllS3Protocols, sdk.S3Protocol(storageProvider)):
s3Protocol, err := sdk.ToS3Protocol(storageProvider)
if err != nil {
return err
}

switch storageProvider {
case "S3", "S3GOV", "S3gov":
v, ok := d.GetOk("storage_aws_role_arn")
if !ok {
return fmt.Errorf("if you use the S3 storage provider you must specify a storage_aws_role_arn")
}

s3Params := sdk.NewS3StorageParamsRequest(v.(string))
s3Params := sdk.NewS3StorageParamsRequest(s3Protocol, v.(string))
if _, ok := d.GetOk("storage_aws_object_acl"); ok {
s3Params.WithStorageAwsObjectAcl(sdk.String(d.Get("storage_aws_object_acl").(string)))
s3Params.WithStorageAwsObjectAcl(d.Get("storage_aws_object_acl").(string))
}
req.WithS3StorageProviderParams(s3Params)
case "AZURE":
req.WithS3StorageProviderParams(*s3Params)
case storageProvider == "AZURE":
v, ok := d.GetOk("azure_tenant_id")
if !ok {
return fmt.Errorf("if you use the Azure storage provider you must specify an azure_tenant_id")
}
req.WithAzureStorageProviderParams(sdk.NewAzureStorageParamsRequest(sdk.String(v.(string))))
case "GCS":
req.WithGCSStorageProviderParams(sdk.NewGCSStorageParamsRequest())
req.WithAzureStorageProviderParams(*sdk.NewAzureStorageParamsRequest(sdk.String(v.(string))))
case storageProvider == "GCS":
req.WithGCSStorageProviderParams(*sdk.NewGCSStorageParamsRequest())
default:
return fmt.Errorf("unexpected provider %v", storageProvider)
}
Expand Down Expand Up @@ -295,7 +300,7 @@ func UpdateStorageIntegration(d *schema.ResourceData, meta any) error {

if d.HasChange("comment") {
runSetStatement = true
setReq.WithComment(sdk.String(d.Get("comment").(string)))
setReq.WithComment(d.Get("comment").(string))
}

if d.HasChange("enabled") {
Expand All @@ -320,7 +325,7 @@ func UpdateStorageIntegration(d *schema.ResourceData, meta any) error {
v := d.Get("storage_blocked_locations").([]interface{})
if len(v) == 0 {
if err := client.StorageIntegrations.Alter(ctx, sdk.NewAlterStorageIntegrationRequest(id).
WithUnset(sdk.NewStorageIntegrationUnsetRequest().WithStorageBlockedLocations(sdk.Bool(true)))); err != nil {
WithUnset(*sdk.NewStorageIntegrationUnsetRequest().WithStorageBlockedLocations(true))); err != nil {
return fmt.Errorf("error unsetting storage_blocked_locations, err = %w", err)
}
} else {
Expand All @@ -342,25 +347,25 @@ func UpdateStorageIntegration(d *schema.ResourceData, meta any) error {

if d.HasChange("storage_aws_object_acl") {
if v, ok := d.GetOk("storage_aws_object_acl"); ok {
s3SetParams.WithStorageAwsObjectAcl(sdk.String(v.(string)))
s3SetParams.WithStorageAwsObjectAcl(v.(string))
} else {
if err := client.StorageIntegrations.Alter(ctx, sdk.NewAlterStorageIntegrationRequest(id).
WithUnset(sdk.NewStorageIntegrationUnsetRequest().WithStorageAwsObjectAcl(sdk.Bool(true)))); err != nil {
WithUnset(*sdk.NewStorageIntegrationUnsetRequest().WithStorageAwsObjectAcl(true))); err != nil {
return fmt.Errorf("error unsetting storage_aws_object_acl, err = %w", err)
}
}
}

setReq.WithS3Params(s3SetParams)
setReq.WithS3Params(*s3SetParams)
}

if d.HasChange("azure_tenant_id") {
runSetStatement = true
setReq.WithAzureParams(sdk.NewSetAzureStorageParamsRequest(d.Get("azure_tenant_id").(string)))
setReq.WithAzureParams(*sdk.NewSetAzureStorageParamsRequest(d.Get("azure_tenant_id").(string)))
}

if runSetStatement {
if err := client.StorageIntegrations.Alter(ctx, sdk.NewAlterStorageIntegrationRequest(id).WithSet(setReq)); err != nil {
if err := client.StorageIntegrations.Alter(ctx, sdk.NewAlterStorageIntegrationRequest(id).WithSet(*setReq)); err != nil {
return fmt.Errorf("error updating storage integration, err = %w", err)
}
}
Expand Down
30 changes: 27 additions & 3 deletions pkg/sdk/storage_integration_def.go
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,9 +1,32 @@
package sdk

import g "github.com/Snowflake-Labs/terraform-provider-snowflake/pkg/sdk/poc/generator"
import (
"fmt"

Check failure on line 4 in pkg/sdk/storage_integration_def.go

View workflow job for this annotation

GitHub Actions / reviewdog 

[golangci] reported by reviewdog 🐶
File is not `gofumpt`-ed (gofumpt)

Raw Output:
pkg/sdk/storage_integration_def.go:4: File is not `gofumpt`-ed (gofumpt)
	"fmt"
g "github.com/Snowflake-Labs/terraform-provider-snowflake/pkg/sdk/poc/generator"

Check failure on line 5 in pkg/sdk/storage_integration_def.go

View workflow job for this annotation

GitHub Actions / reviewdog 

[golangci] reported by reviewdog 🐶
File is not `goimports`-ed (goimports)

Raw Output:
pkg/sdk/storage_integration_def.go:5: File is not `goimports`-ed (goimports)
	g "github.com/Snowflake-Labs/terraform-provider-snowflake/pkg/sdk/poc/generator"
"strings"

Check failure on line 6 in pkg/sdk/storage_integration_def.go

View workflow job for this annotation

GitHub Actions / reviewdog 

[golangci] reported by reviewdog 🐶
File is not `gofumpt`-ed (gofumpt)

Raw Output:
pkg/sdk/storage_integration_def.go:6: File is not `gofumpt`-ed (gofumpt)
	"strings"
)

//go:generate go run ./poc/main.go

type S3Protocol string

const (
RegularS3Protocol S3Protocol = "S3"
GovS3Protocol S3Protocol = "S3GOV"
ChinaS3Protocol S3Protocol = "S3CHINA"
)

var AllS3Protocols = []S3Protocol{RegularS3Protocol, GovS3Protocol, ChinaS3Protocol}

func ToS3Protocol(s string) (S3Protocol, error) {
switch protocol := S3Protocol(strings.ToUpper(s)); protocol {
case RegularS3Protocol, GovS3Protocol, ChinaS3Protocol:
return protocol, nil
default:
return "", fmt.Errorf("invalid S3 protocol: %s", s)
}
}

var StorageLocationDef = g.NewQueryStruct("StorageLocation").Text("Path", g.KeywordOptions().SingleQuotes().Required())

var StorageIntegrationDef = g.NewInterface(
Expand All @@ -23,7 +46,8 @@
OptionalQueryStructField(
"S3StorageProviderParams",
g.NewQueryStruct("S3StorageParams").
PredefinedQueryStructField("storageProvider", "string", g.StaticOptions().SQL("STORAGE_PROVIDER = 'S3'")).
PredefinedQueryStructField("Protocol", g.KindOfT[S3Protocol](), g.ParameterOptions().SQL("STORAGE_PROVIDER").SingleQuotes().Required()).
//PredefinedQueryStructField("storageProvider", "string", g.StaticOptions().SQL("STORAGE_PROVIDER = 'S3'")).

Check failure on line 50 in pkg/sdk/storage_integration_def.go

View workflow job for this annotation

GitHub Actions / reviewdog 

[golangci] reported by reviewdog 🐶
commentFormatting: put a space between `//` and comment text (gocritic)

Raw Output:
pkg/sdk/storage_integration_def.go:50:6: commentFormatting: put a space between `//` and comment text (gocritic)
					//PredefinedQueryStructField("storageProvider", "string", g.StaticOptions().SQL("STORAGE_PROVIDER = 'S3'")).
					^
sfc-gh-jmichalak marked this conversation as resolved.
Show resolved Hide resolved
TextAssignment("STORAGE_AWS_ROLE_ARN", g.ParameterOptions().SingleQuotes().Required()).
OptionalTextAssignment("STORAGE_AWS_OBJECT_ACL", g.ParameterOptions().SingleQuotes()),
g.KeywordOptions(),
Expand Down Expand Up @@ -73,7 +97,7 @@
TextAssignment("AZURE_TENANT_ID", g.ParameterOptions().SingleQuotes().Required()),
g.KeywordOptions(),
).
BooleanAssignment("ENABLED", g.ParameterOptions()).
OptionalBooleanAssignment("ENABLED", g.ParameterOptions()).
ListAssignment("STORAGE_ALLOWED_LOCATIONS", "StorageLocation", g.ParameterOptions().Parentheses()).
ListAssignment("STORAGE_BLOCKED_LOCATIONS", "StorageLocation", g.ParameterOptions().Parentheses()).
OptionalComment(),
Expand Down
82 changes: 42 additions & 40 deletions pkg/sdk/storage_integration_dto_builders_gen.go
View file
Open in desktop

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

3 changes: 2 additions & 1 deletion pkg/sdk/storage_integration_dto_gen.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -24,7 +24,8 @@ type CreateStorageIntegrationRequest struct {
}

type S3StorageParamsRequest struct {
StorageAwsRoleArn string // required
Protocol S3Protocol // required
StorageAwsRoleArn string // required
StorageAwsObjectAcl *string
}

Expand Down
Loading
Loading