Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

feat: Add tables to the SDK #2042

Merged
merged 52 commits into from
Dec 8, 2023
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
52 commits
Select commit Hold shift + click to select a range
308b413
qs
sfc-gh-pbosak Jul 28, 2023
aca16a7
chantes
sfc-gh-pbosak Aug 9, 2023
ce7bfa0
Merge branch 'main' into tables-sdk-v2
sfc-gh-pbosak Aug 9, 2023
1e70498
changes
sfc-gh-pbosak Aug 14, 2023
0937ea8
changes
sfc-gh-pbosak Aug 14, 2023
ea178e5
Merge branch 'main' into tables-sdk-v2
sfc-gh-pbosak Aug 14, 2023
c0e6a08
stuff
sfc-gh-pbosak Aug 17, 2023
a0b61ea
Merge branch 'main' into tables-sdk-v2
sfc-gh-pbosak Aug 17, 2023
38148d6
changes
sfc-gh-pbosak Aug 28, 2023
30cc775
changes
sfc-gh-pbosak Aug 31, 2023
5b796f4
changes
sfc-gh-pbosak Sep 1, 2023
5932fae
Merge branch 'main' of github.com:Snowflake-Labs/terraform-provider-s…
sfc-gh-pbosak Sep 1, 2023
e2523df
changes
sfc-gh-pbosak Sep 1, 2023
bba3a83
fixes
sfc-gh-pbosak Sep 1, 2023
e4642af
fixes
sfc-gh-pbosak Sep 1, 2023
0a659ff
changes
sfc-gh-pbosak Sep 1, 2023
06ae64c
Merge branch 'main' into tables-sdk-v2
sfc-gh-asawicki Nov 20, 2023
cb1ea66
Fix minor stuff
sfc-gh-asawicki Nov 20, 2023
03d4f9d
Merge branch 'main' into tables-sdk-v2
sfc-gh-asawicki Nov 22, 2023
f193057
Use our assertion helper
sfc-gh-asawicki Nov 28, 2023
0b8d154
Fix tests
sfc-gh-asawicki Nov 28, 2023
d5ff1a7
Use our errors
sfc-gh-asawicki Nov 28, 2023
c4931f7
Use common schema and db
sfc-gh-asawicki Nov 28, 2023
0cd1860
Show by id using in syntax
sfc-gh-asawicki Nov 28, 2023
06bf8ff
Create database using SDK
sfc-gh-asawicki Nov 28, 2023
e099f72
Rename to toOpts()
sfc-gh-asawicki Nov 28, 2023
90febf6
Merge branch 'main' into tables-sdk-v2
sfc-gh-asawicki Dec 1, 2023
ee9ada1
Extract getting table columns to helper
sfc-gh-asawicki Dec 1, 2023
b1de9f9
Fix column assertions
sfc-gh-asawicki Dec 1, 2023
4611b0e
Refactor column assertions
sfc-gh-asawicki Dec 1, 2023
7748dbd
Merge two create table helper methods
sfc-gh-asawicki Dec 1, 2023
4fef9da
Add skip file x and x%
sfc-gh-asawicki Dec 1, 2023
78fcab9
Fix tests and files
sfc-gh-asawicki Dec 1, 2023
2f1d516
Add describe and fix integration tests
sfc-gh-asawicki Dec 1, 2023
09b62ba
Update comments
sfc-gh-asawicki Dec 1, 2023
252e513
Fix linter
sfc-gh-asawicki Dec 1, 2023
304ea1f
Merge branch 'main' into tables-sdk-v2
sfc-gh-asawicki Dec 1, 2023
554b251
Fix reviewdog
sfc-gh-asawicki Dec 1, 2023
56ed0c8
Fix test
sfc-gh-asawicki Dec 1, 2023
e5f3b0d
Fix after review part 1
sfc-gh-asawicki Dec 5, 2023
cf73cb5
Fix after review part 2
sfc-gh-asawicki Dec 5, 2023
517bb5d
Fix after review part 3
sfc-gh-asawicki Dec 5, 2023
3afd536
Add out of line constraint validations (WIP)
sfc-gh-asawicki Dec 5, 2023
bc46fb8
Add type presence validation
sfc-gh-asawicki Dec 6, 2023
76f50c7
Validate out of line constraints
sfc-gh-asawicki Dec 6, 2023
05b6c71
Allow multiple out of line column constraints
sfc-gh-asawicki Dec 6, 2023
f691f14
Validate out of line in alter
sfc-gh-asawicki Dec 6, 2023
551672c
Use query in create table as select
sfc-gh-asawicki Dec 6, 2023
d0aed52
Fix linter
sfc-gh-asawicki Dec 6, 2023
9acd22c
Fix after review
sfc-gh-asawicki Dec 7, 2023
ea2e35a
Merge branch 'main' into tables-sdk-v2
sfc-gh-asawicki Dec 7, 2023
6f924bd
Fix after review
sfc-gh-asawicki Dec 7, 2023
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 3 additions & 0 deletions Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -68,6 +68,9 @@ test: ## run unit and integration tests
test-acceptance: ## run acceptance tests
TF_ACC=1 go test -run "^TestAcc_" -v -cover -timeout=30m ./...

test-integration: ## run SDK integration tests
go test -run "^TestInt_" -v -cover -timeout=20m ./...

test-architecture: ## check architecture constraints between packages
go test ./pkg/architests/... -v

Expand Down
6 changes: 3 additions & 3 deletions pkg/resources/file_format.go
Original file line number Diff line number Diff line change
Expand Up @@ -428,7 +428,7 @@ func CreateFileFormat(d *schema.ResourceData, meta interface{}) error {
}
nullIf = append(nullIf, sdk.NullString{S: s.(string)})
}
opts.JSONNullIf = &nullIf
opts.JSONNullIf = nullIf
}
if v, ok := d.GetOk("file_extension"); ok {
opts.JSONFileExtension = sdk.String(v.(string))
Expand Down Expand Up @@ -649,7 +649,7 @@ func ReadFileFormat(d *schema.ResourceData, meta interface{}) error {
return err
}
nullIf := []string{}
for _, s := range *fileFormat.Options.JSONNullIf {
for _, s := range fileFormat.Options.JSONNullIf {
nullIf = append(nullIf, s.S)
}
if err := d.Set("null_if", nullIf); err != nil {
Expand Down Expand Up @@ -939,7 +939,7 @@ func UpdateFileFormat(d *schema.ResourceData, meta interface{}) error {
}
nullIf = append(nullIf, sdk.NullString{S: s.(string)})
}
opts.Set.JSONNullIf = &nullIf
opts.Set.JSONNullIf = nullIf
runSet = true
}
if d.HasChange("file_extension") {
Expand Down
2 changes: 2 additions & 0 deletions pkg/sdk/client.go
Original file line number Diff line number Diff line change
Expand Up @@ -52,6 +52,7 @@ type Client struct {
Shares Shares
Stages Stages
Streams Streams
Tables Tables
Tags Tags
Tasks Tasks
Users Users
Expand Down Expand Up @@ -181,6 +182,7 @@ func (c *Client) initialize() {
c.Stages = &stages{client: c}
c.Streams = &streams{client: c}
c.SystemFunctions = &systemFunctions{client: c}
c.Tables = &tables{client: c}
c.Tags = &tags{client: c}
c.Tasks = &tasks{client: c}
c.Users = &users{client: c}
Expand Down
115 changes: 115 additions & 0 deletions pkg/sdk/common_table_types.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,115 @@
package sdk

import "errors"

type TableRowAccessPolicy struct {
rowAccessPolicy bool `ddl:"static" sql:"ROW ACCESS POLICY"`
Name SchemaObjectIdentifier `ddl:"identifier"`
On []string `ddl:"keyword,parentheses" sql:"ON"`
}

// ColumnInlineConstraint is based on https://docs.snowflake.com/en/sql-reference/sql/create-table-constraint#inline-unique-primary-foreign-key.
type ColumnInlineConstraint struct {
sfc-gh-jcieslak marked this conversation as resolved.
Show resolved Hide resolved
Name *string `ddl:"parameter,no_equals" sql:"CONSTRAINT"`
Type ColumnConstraintType `ddl:"keyword"`
ForeignKey *InlineForeignKey `ddl:"keyword" sql:"FOREIGN KEY"`

// optional
Enforced *bool `ddl:"keyword" sql:"ENFORCED"`
NotEnforced *bool `ddl:"keyword" sql:"NOT ENFORCED"`
Deferrable *bool `ddl:"keyword" sql:"DEFERRABLE"`
NotDeferrable *bool `ddl:"keyword" sql:"NOT DEFERRABLE"`
InitiallyDeferred *bool `ddl:"keyword" sql:"INITIALLY DEFERRED"`
InitiallyImmediate *bool `ddl:"keyword" sql:"INITIALLY IMMEDIATE"`
Enable *bool `ddl:"keyword" sql:"ENABLE"`
Disable *bool `ddl:"keyword" sql:"DISABLE"`
Validate *bool `ddl:"keyword" sql:"VALIDATE"`
NoValidate *bool `ddl:"keyword" sql:"NOVALIDATE"`
Rely *bool `ddl:"keyword" sql:"RELY"`
NoRely *bool `ddl:"keyword" sql:"NORELY"`
}

func (v *ColumnInlineConstraint) validate() error {
var errs []error
switch v.Type {
case ColumnConstraintTypeForeignKey:
if !valueSet(v.ForeignKey) {
errs = append(errs, errNotSet("ColumnInlineConstraint", "ForeignKey"))
} else {
if err := v.ForeignKey.validate(); err != nil {
errs = append(errs, err)
}
}
case ColumnConstraintTypeUnique, ColumnConstraintTypePrimaryKey:
if valueSet(v.ForeignKey) {
errs = append(errs, errSet("ColumnInlineConstraint", "ForeignKey"))
}
default:
errs = append(errs, errInvalidValue("ColumnInlineConstraint", "Type", string(v.Type)))
}
if moreThanOneValueSet(v.Enforced, v.NotEnforced) {
errs = append(errs, errMoreThanOneOf("ColumnInlineConstraint", "Enforced", "NotEnforced"))
}
if moreThanOneValueSet(v.Deferrable, v.NotDeferrable) {
errs = append(errs, errMoreThanOneOf("ColumnInlineConstraint", "Deferrable", "NotDeferrable"))
}
if moreThanOneValueSet(v.InitiallyDeferred, v.InitiallyImmediate) {
errs = append(errs, errMoreThanOneOf("ColumnInlineConstraint", "InitiallyDeferred", "InitiallyImmediate"))
}
if moreThanOneValueSet(v.Enable, v.Disable) {
errs = append(errs, errMoreThanOneOf("ColumnInlineConstraint", "Enable", "Disable"))
}
if moreThanOneValueSet(v.Validate, v.NoValidate) {
errs = append(errs, errMoreThanOneOf("ColumnInlineConstraint", "Validate", "Novalidate"))
}
if moreThanOneValueSet(v.Rely, v.NoRely) {
errs = append(errs, errMoreThanOneOf("ColumnInlineConstraint", "Rely", "Norely"))
}
return errors.Join(errs...)
}

type ColumnConstraintType string

const (
ColumnConstraintTypeUnique ColumnConstraintType = "UNIQUE"
ColumnConstraintTypePrimaryKey ColumnConstraintType = "PRIMARY KEY"
ColumnConstraintTypeForeignKey ColumnConstraintType = "FOREIGN KEY"
)

type InlineForeignKey struct {
TableName string `ddl:"keyword" sql:"REFERENCES"`
ColumnName []string `ddl:"keyword,parentheses"`
Match *MatchType `ddl:"keyword" sql:"MATCH"`
On *ForeignKeyOnAction `ddl:"keyword" sql:"ON"`
}

func (v *InlineForeignKey) validate() error {
sfc-gh-jcieslak marked this conversation as resolved.
Show resolved Hide resolved
var errs []error
if !valueSet(v.TableName) {
errs = append(errs, errNotSet("InlineForeignKey", "TableName"))
}
return errors.Join(errs...)
}

type MatchType string

var (
FullMatchType MatchType = "FULL"
SimpleMatchType MatchType = "SIMPLE"
PartialMatchType MatchType = "PARTIAL"
)

type ForeignKeyOnAction struct {
OnUpdate *ForeignKeyAction `ddl:"parameter,no_equals" sql:"ON UPDATE"`
OnDelete *ForeignKeyAction `ddl:"parameter,no_equals" sql:"ON DELETE"`
}

type ForeignKeyAction string

const (
ForeignKeyCascadeAction ForeignKeyAction = "CASCADE"
ForeignKeySetNullAction ForeignKeyAction = "SET NULL"
ForeignKeySetDefaultAction ForeignKeyAction = "SET DEFAULT"
ForeignKeyRestrictAction ForeignKeyAction = "RESTRICT"
ForeignKeyNoAction ForeignKeyAction = "NO ACTION"
)
59 changes: 0 additions & 59 deletions pkg/sdk/common_types.go
Original file line number Diff line number Diff line change
Expand Up @@ -129,65 +129,6 @@ func (row *propertyRow) toIntProperty() *IntProperty {
}
}

type RowAccessPolicy struct {
rowAccessPolicy bool `ddl:"static" sql:"ROW ACCESS POLICY"`
Name SchemaObjectIdentifier `ddl:"identifier"`
On []string `ddl:"keyword,parentheses" sql:"ON"`
}

type ColumnInlineConstraint struct {
NotNull *bool `ddl:"keyword" sql:"NOT NULL"`
Name *string `ddl:"parameter,no_equals" sql:"CONSTRAINT"`
Type *ColumnConstraintType `ddl:"keyword"`
ForeignKey *InlineForeignKey `ddl:"keyword" sql:"FOREIGN KEY"`

// optional
Enforced *bool `ddl:"keyword" sql:"ENFORCED"`
NotEnforced *bool `ddl:"keyword" sql:"NOT ENFORCED"`
Deferrable *bool `ddl:"keyword" sql:"DEFERRABLE"`
NotDeferrable *bool `ddl:"keyword" sql:"NOT DEFERRABLE"`
InitiallyDeferred *bool `ddl:"keyword" sql:"INITIALLY DEFERRED"`
InitiallyImmediate *bool `ddl:"keyword" sql:"INITIALLY IMMEDIATE"`
Enable *bool `ddl:"keyword" sql:"ENABLE"`
Disable *bool `ddl:"keyword" sql:"DISABLE"`
Validate *bool `ddl:"keyword" sql:"VALIDATE"`
NoValidate *bool `ddl:"keyword" sql:"NOVALIDATE"`
Rely *bool `ddl:"keyword" sql:"RELY"`
NoRely *bool `ddl:"keyword" sql:"NORELY"`
}

type ColumnConstraintType string

var (
ColumnConstraintTypeUnique ColumnConstraintType = "UNIQUE"
ColumnConstraintTypePrimaryKey ColumnConstraintType = "PRIMARY KEY"
ColumnConstraintTypeForeignKey ColumnConstraintType = "FOREIGN KEY"
)

type InlineForeignKey struct {
TableName string `ddl:"keyword" sql:"REFERENCES"`
ColumnName []string `ddl:"keyword,parentheses"`
Match *MatchType `ddl:"keyword" sql:"MATCH"`
On *ForeignKeyOnAction `ddl:"keyword" sql:"ON"`
}

func (v *InlineForeignKey) validate() error {
return nil
}

type MatchType string

var (
FullMatchType MatchType = "FULL"
SimpleMatchType MatchType = "SIMPLE"
PartialMatchType MatchType = "PARTIAL"
)

type ForeignKeyOnAction struct {
OnUpdate *bool `ddl:"parameter,no_equals" sql:"ON UPDATE"`
OnDelete *bool `ddl:"parameter,no_equals" sql:"ON DELETE"`
}

func (row *propertyRow) toBoolProperty() *BoolProperty {
var value bool
if row.Value != "" && row.Value != "null" {
Expand Down
1 change: 1 addition & 0 deletions pkg/sdk/data_types.go
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@ import (
"golang.org/x/exp/slices"
)

// DataType is based on https://docs.snowflake.com/en/sql-reference/intro-summary-data-types.
type DataType string

const (
Expand Down
12 changes: 12 additions & 0 deletions pkg/sdk/errors.go
Original file line number Diff line number Diff line change
Expand Up @@ -52,6 +52,10 @@ func errNotSet(structName string, fieldNames ...string) error {
return newError(fmt.Sprintf("%v fields: %v should be set", structName, fieldNames), 2)
}

func errSet(structName string, fieldNames ...string) error {
return newError(fmt.Sprintf("%v fields: %v should not be set", structName, fieldNames), 2)
}

func errExactlyOneOf(structName string, fieldNames ...string) error {
return newError(fmt.Sprintf("exactly one of %s fields %v must be set", structName, fieldNames), 2)
}
Expand All @@ -60,6 +64,14 @@ func errAtLeastOneOf(structName string, fieldNames ...string) error {
return newError(fmt.Sprintf("at least one of %s fields %v must be set", structName, fieldNames), 2)
}

func errMoreThanOneOf(structName string, fieldNames ...string) error {
return newError(fmt.Sprintf("more than one field (%v) of %s cannot be set", fieldNames, structName), 2)
}

func errInvalidValue(structName string, fieldName string, invalidValue string) error {
return newError(fmt.Sprintf("invalid value %s of struct %s field: %s", invalidValue, structName, fieldName), 2)
}

func decodeDriverError(err error) error {
if err == nil {
return nil
Expand Down
10 changes: 6 additions & 4 deletions pkg/sdk/external_tables.go
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@ type ExternalTables interface {
CreateWithManualPartitioning(ctx context.Context, req *CreateWithManualPartitioningExternalTableRequest) error
CreateDeltaLake(ctx context.Context, req *CreateDeltaLakeExternalTableRequest) error
CreateUsingTemplate(ctx context.Context, req *CreateExternalTableUsingTemplateRequest) error
// TODO: Add alter options from https://docs.snowflake.com/en/sql-reference/sql/alter-table#external-table-column-actions-exttablecolumnaction (for better UX)
Alter(ctx context.Context, req *AlterExternalTableRequest) error
AlterPartitions(ctx context.Context, req *AlterExternalTablePartitionRequest) error
Drop(ctx context.Context, req *DropExternalTableRequest) error
Expand Down Expand Up @@ -74,14 +75,15 @@ type CreateExternalTableOptions struct {
AwsSnsTopic *string `ddl:"parameter,single_quotes" sql:"AWS_SNS_TOPIC"`
CopyGrants *bool `ddl:"keyword" sql:"COPY GRANTS"`
Comment *string `ddl:"parameter,single_quotes" sql:"COMMENT"`
RowAccessPolicy *RowAccessPolicy `ddl:"keyword"`
RowAccessPolicy *TableRowAccessPolicy `ddl:"keyword"`
Tag []TagAssociation `ddl:"keyword,parentheses" sql:"TAG"`
}

type ExternalTableColumn struct {
Name string `ddl:"keyword"`
Type DataType `ddl:"keyword"`
AsExpression []string `ddl:"keyword,parentheses" sql:"AS"`
NotNull *bool `ddl:"keyword" sql:"NOT NULL"`
InlineConstraint *ColumnInlineConstraint
}

Expand Down Expand Up @@ -205,7 +207,7 @@ type CreateWithManualPartitioningExternalTableOptions struct {
FileFormat []ExternalTableFileFormat `ddl:"parameter,parentheses" sql:"FILE_FORMAT"`
CopyGrants *bool `ddl:"keyword" sql:"COPY GRANTS"`
Comment *string `ddl:"parameter,single_quotes" sql:"COMMENT"`
RowAccessPolicy *RowAccessPolicy `ddl:"keyword"`
RowAccessPolicy *TableRowAccessPolicy `ddl:"keyword"`
Tag []TagAssociation `ddl:"keyword,parentheses" sql:"TAG"`
}

Expand All @@ -227,7 +229,7 @@ type CreateDeltaLakeExternalTableOptions struct {
DeltaTableFormat *bool `ddl:"keyword" sql:"TABLE_FORMAT = DELTA"`
CopyGrants *bool `ddl:"keyword" sql:"COPY GRANTS"`
Comment *string `ddl:"parameter,single_quotes" sql:"COMMENT"`
RowAccessPolicy *RowAccessPolicy `ddl:"keyword"`
RowAccessPolicy *TableRowAccessPolicy `ddl:"keyword"`
Tag []TagAssociation `ddl:"keyword,parentheses" sql:"TAG"`
}

Expand All @@ -248,7 +250,7 @@ type CreateExternalTableUsingTemplateOptions struct {
FileFormat []ExternalTableFileFormat `ddl:"parameter,parentheses" sql:"FILE_FORMAT"`
AwsSnsTopic *string `ddl:"parameter,single_quotes" sql:"AWS_SNS_TOPIC"`
Comment *string `ddl:"parameter,single_quotes" sql:"COMMENT"`
RowAccessPolicy *RowAccessPolicy `ddl:"keyword"`
RowAccessPolicy *TableRowAccessPolicy `ddl:"keyword"`
Tag []TagAssociation `ddl:"keyword,parentheses" sql:"TAG"`
}

Expand Down
Loading
Loading