diff --git a/README.md b/README.md index 37186bdc03..be219e7f91 100644 --- a/README.md +++ b/README.md @@ -67,7 +67,7 @@ Integration status - indicates if given resource / datasource is using new SDK. | Managed Account | ❌ | snowflake_managed_account | snowflake_managed_account | ❌ | | User | ✅ | snowflake_user | snowflake_user | ✅ | | Database Role | ✅ | snowflake_database_role | snowflake_database_role | ✅ | -| Role | 👨‍💻 | snowflake_role | snowflake_role | 👨‍💻 | +| Role | ✅ | snowflake_role | snowflake_role | 👨‍💻 | | Grant Privilege to Application Role | ❌ | snowflake_grant_privileges_to_application_role | snowflake_grants | ❌ | | Grant Privilege to Database Role | ✅ | snowflake_grant_privileges_to_database_role | snowflake_grants | 👨‍💻 | | Grant Privilege to Role | ❌ | snowflake_grant_privileges_to_role | snowflake_grants | ✅ | @@ -96,7 +96,7 @@ Integration status - indicates if given resource / datasource is using new SDK. | Share | ✅ | snowflake_share | snowflake_share | ✅ | | Table | 👨‍💻 | snowflake_table | snowflake_table | ❌ | | Dynamic Table | ❌ | snowflake_dynamic_table | snowflake_dynamic_table | ❌ | -| External Table | 👨‍💻 | snowflake_external_table | snowflake_external_table | ❌ | +| External Table | ✅ | snowflake_external_table | snowflake_external_table | ❌ | | Event Table | ❌ | snowflake_event_table | snowflake_event_table | ❌ | | View | ❌ | snowflake_view | snowflake_view | ❌ | | Materialized View | ❌ | snowflake_materialized_view | snowflake_materialized_view | ❌ | diff --git a/pkg/sdk/client.go b/pkg/sdk/client.go index 5dbe19017d..691638a7e3 100644 --- a/pkg/sdk/client.go +++ b/pkg/sdk/client.go @@ -31,6 +31,7 @@ type Client struct { Comments Comments Databases Databases DatabaseRoles DatabaseRoles + ExternalTables ExternalTables FailoverGroups FailoverGroups FileFormats FileFormats Grants Grants @@ -128,6 +129,7 @@ func (c *Client) initialize() { c.ConversionFunctions = &conversionFunctions{client: c} c.Databases = &databases{client: c} c.DatabaseRoles = &databaseRoles{client: c} + c.ExternalTables = &externalTables{client: c} c.FailoverGroups = &failoverGroups{client: c} c.FileFormats = &fileFormats{client: c} c.Grants = &grants{client: c} diff --git a/pkg/sdk/common_types.go b/pkg/sdk/common_types.go index 080984cebd..26a1330804 100644 --- a/pkg/sdk/common_types.go +++ b/pkg/sdk/common_types.go @@ -129,6 +129,65 @@ func (row *propertyRow) toIntProperty() *IntProperty { } } +type RowAccessPolicy struct { + rowAccessPolicy bool `ddl:"static" sql:"ROW ACCESS POLICY"` + Name SchemaObjectIdentifier `ddl:"identifier"` + On []string `ddl:"keyword,parentheses" sql:"ON"` +} + +type ColumnInlineConstraint struct { + NotNull *bool `ddl:"keyword" sql:"NOT NULL"` + Name *string `ddl:"parameter,no_equals" sql:"CONSTRAINT"` + Type *ColumnConstraintType `ddl:"keyword"` + ForeignKey *InlineForeignKey `ddl:"keyword" sql:"FOREIGN KEY"` + + // optional + Enforced *bool `ddl:"keyword" sql:"ENFORCED"` + NotEnforced *bool `ddl:"keyword" sql:"NOT ENFORCED"` + Deferrable *bool `ddl:"keyword" sql:"DEFERRABLE"` + NotDeferrable *bool `ddl:"keyword" sql:"NOT DEFERRABLE"` + InitiallyDeferred *bool `ddl:"keyword" sql:"INITIALLY DEFERRED"` + InitiallyImmediate *bool `ddl:"keyword" sql:"INITIALLY IMMEDIATE"` + Enable *bool `ddl:"keyword" sql:"ENABLE"` + Disable *bool `ddl:"keyword" sql:"DISABLE"` + Validate *bool `ddl:"keyword" sql:"VALIDATE"` + NoValidate *bool `ddl:"keyword" sql:"NOVALIDATE"` + Rely *bool `ddl:"keyword" sql:"RELY"` + NoRely *bool `ddl:"keyword" sql:"NORELY"` +} + +type ColumnConstraintType string + +var ( + ColumnConstraintTypeUnique ColumnConstraintType = "UNIQUE" + ColumnConstraintTypePrimaryKey ColumnConstraintType = "PRIMARY KEY" + ColumnConstraintTypeForeignKey ColumnConstraintType = "FOREIGN KEY" +) + +type InlineForeignKey struct { + TableName string `ddl:"keyword" sql:"REFERENCES"` + ColumnName []string `ddl:"keyword,parentheses"` + Match *MatchType `ddl:"keyword" sql:"MATCH"` + On *ForeignKeyOnAction `ddl:"keyword" sql:"ON"` +} + +func (v *InlineForeignKey) validate() error { + return nil +} + +type MatchType string + +var ( + FullMatchType MatchType = "FULL" + SimpleMatchType MatchType = "SIMPLE" + PartialMatchType MatchType = "PARTIAL" +) + +type ForeignKeyOnAction struct { + OnUpdate *bool `ddl:"parameter,no_equals" sql:"ON UPDATE"` + OnDelete *bool `ddl:"parameter,no_equals" sql:"ON DELETE"` +} + func (row *propertyRow) toBoolProperty() *BoolProperty { var value bool if row.Value != "" && row.Value != "null" { diff --git a/pkg/sdk/data_types.go b/pkg/sdk/data_types.go index 0e1a4965f8..132f2a94d5 100644 --- a/pkg/sdk/data_types.go +++ b/pkg/sdk/data_types.go @@ -13,10 +13,12 @@ const ( DataTypeNumber DataType = "NUMBER" DataTypeFloat DataType = "FLOAT" DataTypeVARCHAR DataType = "VARCHAR" + DataTypeString DataType = "STRING" DataTypeBinary DataType = "BINARY" DataTypeBoolean DataType = "BOOLEAN" DataTypeDate DataType = "DATE" DataTypeTime DataType = "TIME" + DataTypeTimestamp DataType = "TIMESTAMP" DataTypeTimestampLTZ DataType = "TIMESTAMP_LTZ" DataTypeTimestampNTZ DataType = "TIMESTAMP_NTZ" DataTypeTimestampTZ DataType = "TIMESTAMP_TZ" diff --git a/pkg/sdk/errors.go b/pkg/sdk/errors.go index 021103350a..60acd8e449 100644 --- a/pkg/sdk/errors.go +++ b/pkg/sdk/errors.go @@ -19,8 +19,12 @@ var ( errInvalidObjectIdentifier = errors.New("invalid object identifier") ) -func errOneOf(fieldNames ...string) error { - return fmt.Errorf("fields %v are incompatible and cannot be set at once", fieldNames) +func errOneOf(structName string, fieldNames ...string) error { + return fmt.Errorf("%v fields: %v are incompatible and cannot be set at the same time", structName, fieldNames) +} + +func errNotSet(structName string, fieldNames ...string) error { + return fmt.Errorf("%v fields: %v should be set", structName, fieldNames) } func errExactlyOneOf(fieldNames ...string) error { diff --git a/pkg/sdk/external_tables.go b/pkg/sdk/external_tables.go new file mode 100644 index 0000000000..8cddfc82b3 --- /dev/null +++ b/pkg/sdk/external_tables.go @@ -0,0 +1,464 @@ +package sdk + +import ( + "context" + "database/sql" + "time" +) + +var ( + _ convertibleRow[ExternalTable] = (*externalTableRow)(nil) + _ convertibleRow[ExternalTableColumnDetails] = (*externalTableColumnDetailsRow)(nil) + _ convertibleRow[ExternalTableStageDetails] = (*externalTableStageDetailsRow)(nil) +) + +type ExternalTables interface { + Create(ctx context.Context, req *CreateExternalTableRequest) error + CreateWithManualPartitioning(ctx context.Context, req *CreateWithManualPartitioningExternalTableRequest) error + CreateDeltaLake(ctx context.Context, req *CreateDeltaLakeExternalTableRequest) error + CreateUsingTemplate(ctx context.Context, req *CreateExternalTableUsingTemplateRequest) error + Alter(ctx context.Context, req *AlterExternalTableRequest) error + AlterPartitions(ctx context.Context, req *AlterExternalTablePartitionRequest) error + Drop(ctx context.Context, req *DropExternalTableRequest) error + Show(ctx context.Context, req *ShowExternalTableRequest) ([]ExternalTable, error) + ShowByID(ctx context.Context, req *ShowExternalTableByIDRequest) (*ExternalTable, error) + DescribeColumns(ctx context.Context, req *DescribeExternalTableColumnsRequest) ([]ExternalTableColumnDetails, error) + DescribeStage(ctx context.Context, req *DescribeExternalTableStageRequest) ([]ExternalTableStageDetails, error) +} + +type ExternalTable struct { + CreatedOn time.Time + Name string + DatabaseName string + SchemaName string + Invalid bool + InvalidReason string + Owner string + Comment string + Stage string + Location string + FileFormatName string + FileFormatType string + Cloud string + Region string + NotificationChannel string + LastRefreshedOn time.Time + TableFormat string + LastRefreshDetails string + OwnerRoleType string +} + +func (v *ExternalTable) ID() AccountObjectIdentifier { + return NewAccountObjectIdentifier(v.Name) +} + +func (v *ExternalTable) ObjectType() ObjectType { + return ObjectTypeExternalTable +} + +// CreateExternalTableOptions based on https://docs.snowflake.com/en/sql-reference/sql/create-external-table +type CreateExternalTableOptions struct { + create bool `ddl:"static" sql:"CREATE"` + OrReplace *bool `ddl:"keyword" sql:"OR REPLACE"` + externalTable bool `ddl:"static" sql:"EXTERNAL TABLE"` + IfNotExists *bool `ddl:"keyword" sql:"IF NOT EXISTS"` + name AccountObjectIdentifier `ddl:"identifier"` + Columns []ExternalTableColumn `ddl:"list,parentheses"` + CloudProviderParams *CloudProviderParams + PartitionBy []string `ddl:"keyword,parentheses" sql:"PARTITION BY"` + Location string `ddl:"parameter" sql:"LOCATION"` + RefreshOnCreate *bool `ddl:"parameter" sql:"REFRESH_ON_CREATE"` + AutoRefresh *bool `ddl:"parameter" sql:"AUTO_REFRESH"` + Pattern *string `ddl:"parameter,single_quotes" sql:"PATTERN"` + FileFormat []ExternalTableFileFormat `ddl:"parameter,parentheses" sql:"FILE_FORMAT"` + AwsSnsTopic *string `ddl:"parameter,single_quotes" sql:"AWS_SNS_TOPIC"` + CopyGrants *bool `ddl:"keyword" sql:"COPY GRANTS"` + Comment *string `ddl:"parameter,single_quotes" sql:"COMMENT"` + RowAccessPolicy *RowAccessPolicy `ddl:"keyword"` + Tag []TagAssociation `ddl:"keyword,parentheses" sql:"TAG"` +} + +type ExternalTableColumn struct { + Name string `ddl:"keyword"` + Type DataType `ddl:"keyword"` + AsExpression []string `ddl:"keyword,parentheses" sql:"AS"` + InlineConstraint *ColumnInlineConstraint +} + +type CloudProviderParams struct { + // One of + GoogleCloudStorageIntegration *string `ddl:"parameter,single_quotes" sql:"INTEGRATION"` + MicrosoftAzureIntegration *string `ddl:"parameter,single_quotes" sql:"INTEGRATION"` +} + +type ExternalTableFileFormat struct { + Name *string `ddl:"parameter,single_quotes" sql:"FORMAT_NAME"` + Type *ExternalTableFileFormatType `ddl:"parameter" sql:"TYPE"` + Options *ExternalTableFileFormatTypeOptions +} + +type ExternalTableFileFormatType string + +var ( + ExternalTableFileFormatTypeCSV ExternalTableFileFormatType = "CSV" + ExternalTableFileFormatTypeJSON ExternalTableFileFormatType = "JSON" + ExternalTableFileFormatTypeAvro ExternalTableFileFormatType = "AVRO" + ExternalTableFileFormatTypeORC ExternalTableFileFormatType = "ORC" + ExternalTableFileFormatTypeParquet ExternalTableFileFormatType = "PARQUET" +) + +type ExternalTableFileFormatTypeOptions struct { + // CSV type options + CSVCompression *ExternalTableCsvCompression `ddl:"parameter" sql:"COMPRESSION"` + CSVRecordDelimiter *string `ddl:"parameter,single_quotes" sql:"RECORD_DELIMITER"` + CSVFieldDelimiter *string `ddl:"parameter,single_quotes" sql:"FIELD_DELIMITER"` + CSVSkipHeader *int `ddl:"parameter" sql:"SKIP_HEADER"` + CSVSkipBlankLines *bool `ddl:"parameter" sql:"SKIP_BLANK_LINES"` + CSVEscapeUnenclosedField *string `ddl:"parameter,single_quotes" sql:"ESCAPE_UNENCLOSED_FIELD"` + CSVTrimSpace *bool `ddl:"parameter" sql:"TRIM_SPACE"` + CSVFieldOptionallyEnclosedBy *string `ddl:"parameter,single_quotes" sql:"FIELD_OPTIONALLY_ENCLOSED_BY"` + CSVNullIf *[]NullString `ddl:"parameter,parentheses" sql:"NULL_IF"` + CSVEmptyFieldAsNull *bool `ddl:"parameter" sql:"EMPTY_FIELD_AS_NULL"` + CSVEncoding *CSVEncoding `ddl:"parameter,single_quotes" sql:"ENCODING"` + + // JSON type options + JSONCompression *ExternalTableJsonCompression `ddl:"parameter" sql:"COMPRESSION"` + JSONAllowDuplicate *bool `ddl:"parameter" sql:"ALLOW_DUPLICATE"` + JSONStripOuterArray *bool `ddl:"parameter" sql:"STRIP_OUTER_ARRAY"` + JSONStripNullValues *bool `ddl:"parameter" sql:"STRIP_NULL_VALUES"` + JSONReplaceInvalidCharacters *bool `ddl:"parameter" sql:"REPLACE_INVALID_CHARACTERS"` + + // AVRO type options + AvroCompression *ExternalTableAvroCompression `ddl:"parameter" sql:"COMPRESSION"` + AvroReplaceInvalidCharacters *bool `ddl:"parameter" sql:"REPLACE_INVALID_CHARACTERS"` + + // ORC type options + ORCTrimSpace *bool `ddl:"parameter" sql:"TRIM_SPACE"` + ORCReplaceInvalidCharacters *bool `ddl:"parameter" sql:"REPLACE_INVALID_CHARACTERS"` + ORCNullIf *[]NullString `ddl:"parameter,parentheses" sql:"NULL_IF"` + + // PARQUET type options + ParquetCompression *ExternalTableParquetCompression `ddl:"parameter" sql:"COMPRESSION"` + ParquetBinaryAsText *bool `ddl:"parameter" sql:"BINARY_AS_TEXT"` + ParquetReplaceInvalidCharacters *bool `ddl:"parameter" sql:"REPLACE_INVALID_CHARACTERS"` +} + +type ExternalTableCsvCompression string + +var ( + ExternalTableCsvCompressionAuto ExternalTableCsvCompression = "AUTO" + ExternalTableCsvCompressionGzip ExternalTableCsvCompression = "GZIP" + ExternalTableCsvCompressionBz2 ExternalTableCsvCompression = "BZ2" + ExternalTableCsvCompressionBrotli ExternalTableCsvCompression = "BROTLI" + ExternalTableCsvCompressionZstd ExternalTableCsvCompression = "ZSTD" + ExternalTableCsvCompressionDeflate ExternalTableCsvCompression = "DEFLATE" + ExternalTableCsvCompressionRawDeflate ExternalTableCsvCompression = "RAW_DEFALTE" + ExternalTableCsvCompressionNone ExternalTableCsvCompression = "NONE" +) + +type ExternalTableJsonCompression string + +var ( + ExternalTableJsonCompressionAuto ExternalTableJsonCompression = "AUTO" + ExternalTableJsonCompressionGzip ExternalTableJsonCompression = "GZIP" + ExternalTableJsonCompressionBz2 ExternalTableJsonCompression = "BZ2" + ExternalTableJsonCompressionBrotli ExternalTableJsonCompression = "BROTLI" + ExternalTableJsonCompressionZstd ExternalTableJsonCompression = "ZSTD" + ExternalTableJsonCompressionDeflate ExternalTableJsonCompression = "DEFLATE" + ExternalTableJsonCompressionRawDeflate ExternalTableJsonCompression = "RAW_DEFLATE" + ExternalTableJsonCompressionNone ExternalTableJsonCompression = "NONE" +) + +type ExternalTableAvroCompression string + +var ( + ExternalTableAvroCompressionAuto ExternalTableAvroCompression = "AUTO" + ExternalTableAvroCompressionGzip ExternalTableAvroCompression = "GZIP" + ExternalTableAvroCompressionBz2 ExternalTableAvroCompression = "BZ2" + ExternalTableAvroCompressionBrotli ExternalTableAvroCompression = "BROTLI" + ExternalTableAvroCompressionZstd ExternalTableAvroCompression = "ZSTD" + ExternalTableAvroCompressionDeflate ExternalTableAvroCompression = "DEFLATE" + ExternalTableAvroCompressionRawDeflate ExternalTableAvroCompression = "RAW_DEFLATE" + ExternalTableAvroCompressionNone ExternalTableAvroCompression = "NONE" +) + +type ExternalTableParquetCompression string + +var ( + ExternalTableParquetCompressionAuto ExternalTableParquetCompression = "AUTO" + ExternalTableParquetCompressionSnappy ExternalTableParquetCompression = "SNAPPY" + ExternalTableParquetCompressionNone ExternalTableParquetCompression = "NONE" +) + +// CreateWithManualPartitioningExternalTableOptions based on https://docs.snowflake.com/en/sql-reference/sql/create-external-table +type CreateWithManualPartitioningExternalTableOptions struct { + create bool `ddl:"static" sql:"CREATE"` + OrReplace *bool `ddl:"keyword" sql:"OR REPLACE"` + externalTable bool `ddl:"static" sql:"EXTERNAL TABLE"` + IfNotExists *bool `ddl:"keyword" sql:"IF NOT EXISTS"` + name AccountObjectIdentifier `ddl:"identifier"` + Columns []ExternalTableColumn `ddl:"list,parentheses"` + CloudProviderParams *CloudProviderParams + PartitionBy []string `ddl:"keyword,parentheses" sql:"PARTITION BY"` + Location string `ddl:"parameter" sql:"LOCATION"` + UserSpecifiedPartitionType *bool `ddl:"keyword" sql:"PARTITION_TYPE = USER_SPECIFIED"` + FileFormat []ExternalTableFileFormat `ddl:"parameter,parentheses" sql:"FILE_FORMAT"` + CopyGrants *bool `ddl:"keyword" sql:"COPY GRANTS"` + Comment *string `ddl:"parameter,single_quotes" sql:"COMMENT"` + RowAccessPolicy *RowAccessPolicy `ddl:"keyword"` + Tag []TagAssociation `ddl:"keyword,parentheses" sql:"TAG"` +} + +// CreateDeltaLakeExternalTableOptions based on https://docs.snowflake.com/en/sql-reference/sql/create-external-table +type CreateDeltaLakeExternalTableOptions struct { + create bool `ddl:"static" sql:"CREATE"` + OrReplace *bool `ddl:"keyword" sql:"OR REPLACE"` + externalTable bool `ddl:"static" sql:"EXTERNAL TABLE"` + IfNotExists *bool `ddl:"keyword" sql:"IF NOT EXISTS"` + name AccountObjectIdentifier `ddl:"identifier"` + Columns []ExternalTableColumn `ddl:"list,parentheses"` + CloudProviderParams *CloudProviderParams + PartitionBy []string `ddl:"keyword,parentheses" sql:"PARTITION BY"` + Location string `ddl:"parameter" sql:"LOCATION"` + RefreshOnCreate *bool `ddl:"parameter" sql:"REFRESH_ON_CREATE"` + AutoRefresh *bool `ddl:"parameter" sql:"AUTO_REFRESH"` + UserSpecifiedPartitionType *bool `ddl:"keyword" sql:"PARTITION_TYPE = USER_SPECIFIED"` + FileFormat []ExternalTableFileFormat `ddl:"parameter,parentheses" sql:"FILE_FORMAT"` + DeltaTableFormat *bool `ddl:"keyword" sql:"TABLE_FORMAT = DELTA"` + CopyGrants *bool `ddl:"keyword" sql:"COPY GRANTS"` + Comment *string `ddl:"parameter,single_quotes" sql:"COMMENT"` + RowAccessPolicy *RowAccessPolicy `ddl:"keyword"` + Tag []TagAssociation `ddl:"keyword,parentheses" sql:"TAG"` +} + +// CreateExternalTableUsingTemplateOptions based on https://docs.snowflake.com/en/sql-reference/sql/create-external-table#variant-syntax +type CreateExternalTableUsingTemplateOptions struct { + create bool `ddl:"static" sql:"CREATE"` + OrReplace *bool `ddl:"keyword" sql:"OR REPLACE"` + externalTable bool `ddl:"static" sql:"EXTERNAL TABLE"` + name AccountObjectIdentifier `ddl:"identifier"` + CopyGrants *bool `ddl:"keyword" sql:"COPY GRANTS"` + Query []string `ddl:"parameter,no_equals,parentheses" sql:"USING TEMPLATE"` + CloudProviderParams *CloudProviderParams + PartitionBy []string `ddl:"keyword,parentheses" sql:"PARTITION BY"` + Location string `ddl:"parameter" sql:"LOCATION"` + RefreshOnCreate *bool `ddl:"parameter" sql:"REFRESH_ON_CREATE"` + AutoRefresh *bool `ddl:"parameter" sql:"AUTO_REFRESH"` + Pattern *string `ddl:"parameter,single_quotes" sql:"PATTERN"` + FileFormat []ExternalTableFileFormat `ddl:"parameter,parentheses" sql:"FILE_FORMAT"` + AwsSnsTopic *string `ddl:"parameter,single_quotes" sql:"AWS_SNS_TOPIC"` + Comment *string `ddl:"parameter,single_quotes" sql:"COMMENT"` + RowAccessPolicy *RowAccessPolicy `ddl:"keyword"` + Tag []TagAssociation `ddl:"keyword,parentheses" sql:"TAG"` +} + +// AlterExternalTableOptions based on https://docs.snowflake.com/en/sql-reference/sql/alter-external-table +type AlterExternalTableOptions struct { + alterExternalTable bool `ddl:"static" sql:"ALTER EXTERNAL TABLE"` + IfExists *bool `ddl:"keyword" sql:"IF EXISTS"` + name AccountObjectIdentifier `ddl:"identifier"` + // One of + Refresh *RefreshExternalTable `ddl:"keyword" sql:"REFRESH"` + AddFiles []ExternalTableFile `ddl:"keyword,no_quotes,parentheses" sql:"ADD FILES"` + RemoveFiles []ExternalTableFile `ddl:"keyword,no_quotes,parentheses" sql:"REMOVE FILES"` + AutoRefresh *bool `ddl:"parameter" sql:"SET AUTO_REFRESH"` + SetTag []TagAssociation `ddl:"keyword" sql:"SET TAG"` + UnsetTag []ObjectIdentifier `ddl:"keyword" sql:"UNSET TAG"` +} + +type RefreshExternalTable struct { + Path string `ddl:"parameter,no_equals,single_quotes"` +} + +type ExternalTableFile struct { + Name string `ddl:"keyword,single_quotes"` +} + +// AlterExternalTablePartitionOptions based on https://docs.snowflake.com/en/sql-reference/sql/alter-external-table +type AlterExternalTablePartitionOptions struct { + alterExternalTable bool `ddl:"static" sql:"ALTER EXTERNAL TABLE"` + IfExists *bool `ddl:"keyword" sql:"IF EXISTS"` + name AccountObjectIdentifier `ddl:"identifier"` + AddPartitions []Partition `ddl:"keyword,parentheses" sql:"ADD PARTITION"` + DropPartition *bool `ddl:"keyword" sql:"DROP PARTITION"` + Location string `ddl:"parameter,no_equals,single_quotes" sql:"LOCATION"` +} + +type Partition struct { + ColumnName string `ddl:"keyword"` + Value string `ddl:"parameter,single_quotes"` +} + +// DropExternalTableOptions based on https://docs.snowflake.com/en/sql-reference/sql/drop-external-table +type DropExternalTableOptions struct { + dropExternalTable bool `ddl:"static" sql:"DROP EXTERNAL TABLE"` + IfExists *bool `ddl:"keyword" sql:"IF EXISTS"` + name AccountObjectIdentifier `ddl:"identifier"` + DropOption *ExternalTableDropOption +} + +type ExternalTableDropOption struct { + Restrict *bool `ddl:"keyword" sql:"RESTRICT"` + Cascade *bool `ddl:"keyword" sql:"CASCADE"` +} + +// ShowExternalTableOptions based on https://docs.snowflake.com/en/sql-reference/sql/show-external-tables +type ShowExternalTableOptions struct { + show bool `ddl:"static" sql:"SHOW"` + Terse *bool `ddl:"keyword" sql:"TERSE"` + externalTables bool `ddl:"static" sql:"EXTERNAL TABLES"` + Like *Like `ddl:"keyword" sql:"LIKE"` + In *In `ddl:"keyword" sql:"IN"` + StartsWith *string `ddl:"parameter,single_quotes,no_equals" sql:"STARTS WITH"` + LimitFrom *LimitFrom `ddl:"keyword" sql:"LIMIT"` +} + +type externalTableRow struct { + CreatedOn time.Time + Name string + DatabaseName string + SchemaName string + Invalid bool + InvalidReason sql.NullString + Owner string + Comment string + Stage string + Location string + FileFormatName string + FileFormatType string + Cloud string + Region string + NotificationChannel sql.NullString + LastRefreshedOn sql.NullTime + TableFormat string + LastRefreshDetails sql.NullString + OwnerRoleType string +} + +func (e externalTableRow) convert() *ExternalTable { + et := &ExternalTable{ + CreatedOn: e.CreatedOn, + Name: e.Name, + DatabaseName: e.DatabaseName, + SchemaName: e.DatabaseName, + Invalid: e.Invalid, + Owner: e.Owner, + Stage: e.Stage, + Location: e.Location, + FileFormatName: e.FileFormatName, + FileFormatType: e.FileFormatType, + Cloud: e.Cloud, + Region: e.Region, + TableFormat: e.TableFormat, + OwnerRoleType: e.OwnerRoleType, + Comment: e.Comment, + } + if e.InvalidReason.Valid { + et.InvalidReason = e.InvalidReason.String + } + if e.NotificationChannel.Valid { + et.NotificationChannel = e.NotificationChannel.String + } + if e.LastRefreshedOn.Valid { + et.LastRefreshedOn = e.LastRefreshedOn.Time + } + if e.LastRefreshDetails.Valid { + et.LastRefreshDetails = e.LastRefreshDetails.String + } + return et +} + +// describeExternalTableColumns based on https://docs.snowflake.com/en/sql-reference/sql/desc-external-table +type describeExternalTableColumns struct { + describeExternalTable bool `ddl:"static" sql:"DESCRIBE EXTERNAL TABLE"` + name AccountObjectIdentifier `ddl:"identifier"` + columnsType bool `ddl:"static" sql:"TYPE = COLUMNS"` +} + +type ExternalTableColumnDetails struct { + Name string + Type DataType + Kind string + IsNullable bool + Default *string + IsPrimary bool + IsUnique bool + Check *bool + Expression *string + Comment *string + PolicyName *string +} + +// externalTableColumnDetailsRow based on https://docs.snowflake.com/en/sql-reference/sql/desc-external-table +type externalTableColumnDetailsRow struct { + Name string `db:"name"` + Type DataType `db:"type"` + Kind string `db:"kind"` + IsNullable string `db:"null?"` + Default sql.NullString `db:"default"` + IsPrimary string `db:"primary key"` + IsUnique string `db:"unique key"` + Check sql.NullString `db:"check"` + Expression sql.NullString `db:"expression"` + Comment sql.NullString `db:"comment"` + PolicyName sql.NullString `db:"policy name"` +} + +func (r externalTableColumnDetailsRow) convert() *ExternalTableColumnDetails { + details := &ExternalTableColumnDetails{ + Name: r.Name, + Type: r.Type, + Kind: r.Kind, + IsNullable: r.IsNullable == "Y", + IsPrimary: r.IsPrimary == "Y", + IsUnique: r.IsUnique == "Y", + } + if r.Default.Valid { + details.Default = String(r.Default.String) + } + if r.Check.Valid { + details.Check = Bool(r.Check.String == "Y") + } + if r.Expression.Valid { + details.Expression = String(r.Expression.String) + } + if r.Comment.Valid { + details.Comment = String(r.Comment.String) + } + if r.PolicyName.Valid { + details.PolicyName = String(r.PolicyName.String) + } + return details +} + +type describeExternalTableStage struct { + describeExternalTable bool `ddl:"static" sql:"DESCRIBE EXTERNAL TABLE"` + name AccountObjectIdentifier `ddl:"identifier"` + stageType bool `ddl:"static" sql:"TYPE = STAGE"` +} + +type ExternalTableStageDetails struct { + ParentProperty string + Property string + PropertyType string + PropertyValue string + PropertyDefault string +} + +type externalTableStageDetailsRow struct { + ParentProperty string `db:"parent_property"` + Property string `db:"property"` + PropertyType string `db:"property_type"` + PropertyValue string `db:"property_value"` + PropertyDefault string `db:"property_default"` +} + +func (r externalTableStageDetailsRow) convert() *ExternalTableStageDetails { + return &ExternalTableStageDetails{ + ParentProperty: r.ParentProperty, + Property: r.Property, + PropertyType: r.PropertyType, + PropertyValue: r.PropertyValue, + PropertyDefault: r.PropertyDefault, + } +} diff --git a/pkg/sdk/external_tables_dto.go b/pkg/sdk/external_tables_dto.go new file mode 100644 index 0000000000..9a998bcf9f --- /dev/null +++ b/pkg/sdk/external_tables_dto.go @@ -0,0 +1,680 @@ +package sdk + +//go:generate go run ./dto-builder-generator/main.go + +type CreateExternalTableRequest struct { + orReplace *bool + ifNotExists *bool + name AccountObjectIdentifier // required + columns []*ExternalTableColumnRequest + cloudProviderParams *CloudProviderParamsRequest + partitionBy []string + location string // required + refreshOnCreate *bool + autoRefresh *bool + pattern *string + fileFormat *ExternalTableFileFormatRequest // required + awsSnsTopic *string + copyGrants *bool + comment *string + rowAccessPolicy *RowAccessPolicyRequest + tag []*TagAssociationRequest +} + +type ExternalTableColumnRequest struct { + name string // required + dataType DataType // required + asExpression string // required + inlineConstraint *ColumnInlineConstraintRequest +} + +func (v ExternalTableColumnRequest) toOpts() ExternalTableColumn { + var inlineConstraint *ColumnInlineConstraint + if v.inlineConstraint != nil { + inlineConstraint = v.inlineConstraint.toOpts() + } + + return ExternalTableColumn{ + Name: v.name, + Type: v.dataType, + AsExpression: []string{v.asExpression}, + InlineConstraint: inlineConstraint, + } +} + +func (v *ColumnInlineConstraintRequest) toOpts() *ColumnInlineConstraint { + return &ColumnInlineConstraint{ + NotNull: v.notNull, + Name: &v.name, + Type: &v.constraintType, + ForeignKey: v.foreignKey, + Enforced: v.enforced, + NotEnforced: v.notEnforced, + Deferrable: v.deferrable, + NotDeferrable: v.notDeferrable, + InitiallyDeferred: v.initiallyDeferred, + InitiallyImmediate: v.initiallyImmediate, + Enable: v.enable, + Disable: v.disable, + Validate: v.validate, + NoValidate: v.noValidate, + Rely: v.rely, + NoRely: v.noRely, + } +} + +type ColumnInlineConstraintRequest struct { + notNull *bool + name string // required + constraintType ColumnConstraintType // required + foreignKey *InlineForeignKey + + // optional + enforced *bool + notEnforced *bool + deferrable *bool + notDeferrable *bool + initiallyDeferred *bool + initiallyImmediate *bool + enable *bool + disable *bool + validate *bool + noValidate *bool + rely *bool + noRely *bool +} + +type InlineForeignKeyRequest struct { + tableName string // required + columnName []string + match *MatchType + on *ForeignKeyOnActionRequest +} + +type ForeignKeyOnActionRequest struct { + onUpdate *bool + onDelete *bool +} + +type CloudProviderParamsRequest struct { + googleCloudStorageIntegration *string + microsoftAzureIntegration *string +} + +func (v *CloudProviderParamsRequest) toOpts() *CloudProviderParams { + return &CloudProviderParams{ + GoogleCloudStorageIntegration: v.googleCloudStorageIntegration, + MicrosoftAzureIntegration: v.microsoftAzureIntegration, + } +} + +type ExternalTableFileFormatRequest struct { + name *string + fileFormatType *ExternalTableFileFormatType + options *ExternalTableFileFormatTypeOptionsRequest +} + +func (v *ExternalTableFileFormatTypeOptionsRequest) toOpts() *ExternalTableFileFormatTypeOptions { + var csvNullIf []NullString + if v.csvNullIf != nil { + for _, n := range *v.csvNullIf { + csvNullIf = append(csvNullIf, n.toOpts()) + } + } + + var orcNullIf []NullString + if v.orcNullIf != nil { + for _, n := range *v.orcNullIf { + orcNullIf = append(orcNullIf, n.toOpts()) + } + } + + return &ExternalTableFileFormatTypeOptions{ + CSVCompression: v.csvCompression, + CSVRecordDelimiter: v.csvRecordDelimiter, + CSVFieldDelimiter: v.csvFieldDelimiter, + CSVSkipHeader: v.csvSkipHeader, + CSVSkipBlankLines: v.csvSkipBlankLines, + CSVEscapeUnenclosedField: v.csvEscapeUnenclosedField, + CSVTrimSpace: v.csvTrimSpace, + CSVFieldOptionallyEnclosedBy: v.csvFieldOptionallyEnclosedBy, + CSVNullIf: &csvNullIf, + CSVEmptyFieldAsNull: v.csvEmptyFieldAsNull, + CSVEncoding: v.csvEncoding, + JSONCompression: v.jsonCompression, + JSONAllowDuplicate: v.jsonAllowDuplicate, + JSONStripOuterArray: v.jsonStripOuterArray, + JSONStripNullValues: v.jsonStripNullValues, + JSONReplaceInvalidCharacters: v.jsonReplaceInvalidCharacters, + AvroCompression: v.avroCompression, + AvroReplaceInvalidCharacters: v.avroReplaceInvalidCharacters, + ORCTrimSpace: v.orcTrimSpace, + ORCReplaceInvalidCharacters: v.orcReplaceInvalidCharacters, + ORCNullIf: &orcNullIf, + ParquetCompression: v.parquetCompression, + ParquetBinaryAsText: v.parquetBinaryAsText, + ParquetReplaceInvalidCharacters: v.parquetReplaceInvalidCharacters, + } +} + +func (v ExternalTableFileFormatRequest) toOpts() ExternalTableFileFormat { + var options *ExternalTableFileFormatTypeOptions + if v.options != nil { + options = v.options.toOpts() + } + + return ExternalTableFileFormat{ + Name: v.name, + Type: v.fileFormatType, + Options: options, + } +} + +type ExternalTableFileFormatTypeOptionsRequest struct { + // CSV type options + csvCompression *ExternalTableCsvCompression + csvRecordDelimiter *string + csvFieldDelimiter *string + csvSkipHeader *int + csvSkipBlankLines *bool + csvEscapeUnenclosedField *string + csvTrimSpace *bool + csvFieldOptionallyEnclosedBy *string + csvNullIf *[]NullStringRequest + csvEmptyFieldAsNull *bool + csvEncoding *CSVEncoding + + // JSON type options + jsonCompression *ExternalTableJsonCompression + jsonAllowDuplicate *bool + jsonStripOuterArray *bool + jsonStripNullValues *bool + jsonReplaceInvalidCharacters *bool + + // AVRO type options + avroCompression *ExternalTableAvroCompression + avroReplaceInvalidCharacters *bool + + // ORC type options + orcTrimSpace *bool + orcReplaceInvalidCharacters *bool + orcNullIf *[]NullStringRequest + + // PARQUET type options + parquetCompression *ExternalTableParquetCompression + parquetBinaryAsText *bool + parquetReplaceInvalidCharacters *bool +} + +type NullStringRequest struct { + str string +} + +func (v NullStringRequest) toOpts() NullString { + return NullString{ + S: v.str, + } +} + +type RowAccessPolicyRequest struct { + name SchemaObjectIdentifier // required + on []string // required +} + +func (v *RowAccessPolicyRequest) toOpts() *RowAccessPolicy { + return nil +} + +type TagAssociationRequest struct { + name ObjectIdentifier // required + value string // required +} + +func (v TagAssociationRequest) toOpts() TagAssociation { + return TagAssociation{ + Name: v.name, + Value: v.value, + } +} + +func (v *CreateExternalTableRequest) toOpts() *CreateExternalTableOptions { + columns := make([]ExternalTableColumn, len(v.columns)) + if v.columns != nil { + for i, c := range v.columns { + columns[i] = c.toOpts() + } + } + + var fileFormat []ExternalTableFileFormat + if v.fileFormat != nil { + fileFormat = []ExternalTableFileFormat{v.fileFormat.toOpts()} + } + + var cloudProviderParams *CloudProviderParams + if v.cloudProviderParams != nil { + cloudProviderParams = v.cloudProviderParams.toOpts() + } + + var rowAccessPolicy *RowAccessPolicy + if v.rowAccessPolicy != nil { + rowAccessPolicy = v.rowAccessPolicy.toOpts() + } + + tag := make([]TagAssociation, len(v.tag)) + if v.tag != nil { + for i, t := range v.tag { + tag[i] = t.toOpts() + } + } + + return &CreateExternalTableOptions{ + OrReplace: v.orReplace, + IfNotExists: v.ifNotExists, + name: v.name, + Columns: columns, + CloudProviderParams: cloudProviderParams, + Location: v.location, + RefreshOnCreate: v.refreshOnCreate, + AutoRefresh: v.autoRefresh, + Pattern: v.pattern, + FileFormat: fileFormat, + AwsSnsTopic: v.awsSnsTopic, + CopyGrants: v.copyGrants, + Comment: v.comment, + RowAccessPolicy: rowAccessPolicy, + Tag: tag, + } +} + +type CreateWithManualPartitioningExternalTableRequest struct { + orReplace *bool + ifNotExists *bool + name AccountObjectIdentifier // required + columns []*ExternalTableColumnRequest + cloudProviderParams *CloudProviderParamsRequest + partitionBy []string + location string // required + userSpecifiedPartitionType *bool + fileFormat *ExternalTableFileFormatRequest // required + copyGrants *bool + comment *string + rowAccessPolicy *RowAccessPolicyRequest + tag []*TagAssociationRequest +} + +func (v *CreateWithManualPartitioningExternalTableRequest) toOpts() *CreateWithManualPartitioningExternalTableOptions { + columns := make([]ExternalTableColumn, len(v.columns)) + if v.columns != nil { + for i, c := range v.columns { + columns[i] = c.toOpts() + } + } + + var cloudProviderParams *CloudProviderParams + if v.cloudProviderParams != nil { + cloudProviderParams = v.cloudProviderParams.toOpts() + } + + var fileFormat []ExternalTableFileFormat + if v.fileFormat != nil { + fileFormat = []ExternalTableFileFormat{v.fileFormat.toOpts()} + } + + var rowAccessPolicy *RowAccessPolicy + if v.rowAccessPolicy != nil { + rowAccessPolicy = v.rowAccessPolicy.toOpts() + } + + tag := make([]TagAssociation, len(v.tag)) + if v.tag != nil { + for i, t := range v.tag { + tag[i] = t.toOpts() + } + } + + return &CreateWithManualPartitioningExternalTableOptions{ + OrReplace: v.orReplace, + IfNotExists: v.ifNotExists, + name: v.name, + Columns: columns, + CloudProviderParams: cloudProviderParams, + PartitionBy: v.partitionBy, + Location: v.location, + UserSpecifiedPartitionType: v.userSpecifiedPartitionType, + FileFormat: fileFormat, + CopyGrants: v.copyGrants, + Comment: v.comment, + RowAccessPolicy: rowAccessPolicy, + Tag: tag, + } +} + +type CreateDeltaLakeExternalTableRequest struct { + orReplace *bool + ifNotExists *bool + name AccountObjectIdentifier // required + columns []*ExternalTableColumnRequest + cloudProviderParams *CloudProviderParamsRequest + partitionBy []string + location string // required + userSpecifiedPartitionType *bool + refreshOnCreate *bool + autoRefresh *bool + fileFormat *ExternalTableFileFormatRequest // required + deltaTableFormat *bool + copyGrants *bool + comment *string + rowAccessPolicy *RowAccessPolicyRequest + tag []*TagAssociationRequest +} + +func (v *CreateDeltaLakeExternalTableRequest) toOpts() *CreateDeltaLakeExternalTableOptions { + columns := make([]ExternalTableColumn, len(v.columns)) + if v.columns != nil { + for i, c := range v.columns { + columns[i] = c.toOpts() + } + } + + var cloudProviderParams *CloudProviderParams + if v.cloudProviderParams != nil { + cloudProviderParams = v.cloudProviderParams.toOpts() + } + + var fileFormat []ExternalTableFileFormat + if v.fileFormat != nil { + fileFormat = []ExternalTableFileFormat{v.fileFormat.toOpts()} + } + + var rowAccessPolicy *RowAccessPolicy + if v.rowAccessPolicy != nil { + rowAccessPolicy = v.rowAccessPolicy.toOpts() + } + + tag := make([]TagAssociation, len(v.tag)) + if v.tag != nil { + for i, t := range v.tag { + tag[i] = t.toOpts() + } + } + + return &CreateDeltaLakeExternalTableOptions{ + OrReplace: v.orReplace, + IfNotExists: v.ifNotExists, + name: v.name, + Columns: columns, + CloudProviderParams: cloudProviderParams, + PartitionBy: v.partitionBy, + Location: v.location, + UserSpecifiedPartitionType: v.userSpecifiedPartitionType, + RefreshOnCreate: v.refreshOnCreate, + AutoRefresh: v.autoRefresh, + FileFormat: fileFormat, + DeltaTableFormat: v.deltaTableFormat, + CopyGrants: v.copyGrants, + Comment: v.comment, + RowAccessPolicy: rowAccessPolicy, + Tag: tag, + } +} + +type CreateExternalTableUsingTemplateRequest struct { + orReplace *bool + name AccountObjectIdentifier // required + copyGrants *bool + query string + cloudProviderParams *CloudProviderParamsRequest + partitionBy []string + location string // required + refreshOnCreate *bool + autoRefresh *bool + pattern *string + fileFormat *ExternalTableFileFormatRequest // required + awsSnsTopic *string + comment *string + rowAccessPolicy *RowAccessPolicyRequest + tag []*TagAssociationRequest +} + +func (v *CreateExternalTableUsingTemplateRequest) toOpts() *CreateExternalTableUsingTemplateOptions { + var cloudProviderParams *CloudProviderParams + if v.cloudProviderParams != nil { + cloudProviderParams = v.cloudProviderParams.toOpts() + } + + var fileFormat []ExternalTableFileFormat + if v.fileFormat != nil { + fileFormat = []ExternalTableFileFormat{v.fileFormat.toOpts()} + } + + var rowAccessPolicy *RowAccessPolicy + if v.rowAccessPolicy != nil { + rowAccessPolicy = v.rowAccessPolicy.toOpts() + } + + tag := make([]TagAssociation, len(v.tag)) + if v.tag != nil { + for i, t := range v.tag { + tag[i] = t.toOpts() + } + } + + return &CreateExternalTableUsingTemplateOptions{ + OrReplace: v.orReplace, + name: v.name, + CopyGrants: v.copyGrants, + Query: []string{v.query}, + CloudProviderParams: cloudProviderParams, + PartitionBy: v.partitionBy, + Location: v.location, + RefreshOnCreate: v.refreshOnCreate, + AutoRefresh: v.autoRefresh, + Pattern: v.pattern, + FileFormat: fileFormat, + AwsSnsTopic: v.awsSnsTopic, + Comment: v.comment, + RowAccessPolicy: rowAccessPolicy, + Tag: tag, + } +} + +type AlterExternalTableRequest struct { + ifExists *bool + name AccountObjectIdentifier // required + refresh *RefreshExternalTableRequest + addFiles []*ExternalTableFileRequest + removeFiles []*ExternalTableFileRequest + autoRefresh *bool + setTag []*TagAssociationRequest + unsetTag []ObjectIdentifier +} + +type RefreshExternalTableRequest struct { + path string // required +} + +type ExternalTableFileRequest struct { + name string // required +} + +func (v *AlterExternalTableRequest) toOpts() *AlterExternalTableOptions { + var refresh *RefreshExternalTable + if v.refresh != nil { + refresh = &RefreshExternalTable{ + Path: v.refresh.path, + } + } + + addFiles := make([]ExternalTableFile, len(v.addFiles)) + if v.addFiles != nil { + for i, f := range v.addFiles { + addFiles[i] = ExternalTableFile{ + Name: f.name, + } + } + } + + removeFiles := make([]ExternalTableFile, len(v.removeFiles)) + if v.removeFiles != nil { + for i, f := range v.removeFiles { + removeFiles[i] = ExternalTableFile{ + Name: f.name, + } + } + } + + setTag := make([]TagAssociation, len(v.setTag)) + if v.setTag != nil { + for i, t := range v.setTag { + setTag[i] = t.toOpts() + } + } + + return &AlterExternalTableOptions{ + IfExists: v.ifExists, + name: v.name, + Refresh: refresh, + AddFiles: addFiles, + RemoveFiles: removeFiles, + AutoRefresh: v.autoRefresh, + SetTag: setTag, + UnsetTag: v.unsetTag, + } +} + +type AlterExternalTablePartitionRequest struct { + ifExists *bool + name AccountObjectIdentifier // required + addPartitions []*PartitionRequest + dropPartition *bool + location string +} + +type PartitionRequest struct { + columnName string // required + value string // required +} + +func (v *AlterExternalTablePartitionRequest) toOpts() *AlterExternalTablePartitionOptions { + addPartitions := make([]Partition, len(v.addPartitions)) + if v.addPartitions != nil { + for i, p := range v.addPartitions { + addPartitions[i] = Partition{ + ColumnName: p.columnName, + Value: p.value, + } + } + } + + return &AlterExternalTablePartitionOptions{ + IfExists: v.ifExists, + name: v.name, + AddPartitions: addPartitions, + DropPartition: v.dropPartition, + Location: v.location, + } +} + +type DropExternalTableRequest struct { + ifExists *bool + name AccountObjectIdentifier // required + dropOption *ExternalTableDropOptionRequest +} + +type ExternalTableDropOptionRequest struct { + restrict *bool + cascade *bool +} + +func (v *ExternalTableDropOptionRequest) toOpts() *ExternalTableDropOption { + return &ExternalTableDropOption{ + Restrict: v.restrict, + Cascade: v.cascade, + } +} + +func (v *DropExternalTableRequest) toOpts() *DropExternalTableOptions { + var dropOption *ExternalTableDropOption + if v.dropOption != nil { + dropOption = v.dropOption.toOpts() + } + + return &DropExternalTableOptions{ + IfExists: v.ifExists, + name: v.name, + DropOption: dropOption, + } +} + +type ShowExternalTableRequest struct { + terse *bool + like *string + in *ShowExternalTableInRequest + startsWith *string + limitFrom *LimitFromRequest +} + +type ShowExternalTableInRequest struct { + account *bool + database AccountObjectIdentifier + schema DatabaseObjectIdentifier +} + +func (v *ShowExternalTableInRequest) toOpts() *In { + return &In{ + Account: v.account, + Database: v.database, + Schema: v.schema, + } +} + +type LimitFromRequest struct { + rows *int + from *string +} + +func (v *LimitFromRequest) toOpts() *LimitFrom { + return &LimitFrom{ + Rows: v.rows, + From: v.from, + } +} + +func (v *ShowExternalTableRequest) toOpts() *ShowExternalTableOptions { + var like *Like + if v.like != nil { + like = &Like{ + Pattern: v.like, + } + } + + var in *In + if v.in != nil { + in = v.in.toOpts() + } + + var limitFrom *LimitFrom + if v.limitFrom != nil { + limitFrom = v.limitFrom.toOpts() + } + + return &ShowExternalTableOptions{ + Terse: v.terse, + Like: like, + In: in, + StartsWith: v.startsWith, + LimitFrom: limitFrom, + } +} + +type ShowExternalTableByIDRequest struct { + id AccountObjectIdentifier // required +} + +type DescribeExternalTableColumnsRequest struct { + id AccountObjectIdentifier // required +} + +type DescribeExternalTableStageRequest struct { + id AccountObjectIdentifier // required +} diff --git a/pkg/sdk/external_tables_dto_builders_gen.go b/pkg/sdk/external_tables_dto_builders_gen.go new file mode 100644 index 0000000000..ef2a4e94dd --- /dev/null +++ b/pkg/sdk/external_tables_dto_builders_gen.go @@ -0,0 +1,828 @@ +// Code generated by dto builder generator; DO NOT EDIT. + +package sdk + +import () + +func NewCreateExternalTableRequest( + name AccountObjectIdentifier, + location string, + fileFormat *ExternalTableFileFormatRequest, +) *CreateExternalTableRequest { + s := CreateExternalTableRequest{} + s.name = name + s.location = location + s.fileFormat = fileFormat + return &s +} + +func (s *CreateExternalTableRequest) WithOrReplace(orReplace *bool) *CreateExternalTableRequest { + s.orReplace = orReplace + return s +} + +func (s *CreateExternalTableRequest) WithIfNotExists(ifNotExists *bool) *CreateExternalTableRequest { + s.ifNotExists = ifNotExists + return s +} + +func (s *CreateExternalTableRequest) WithColumns(columns []*ExternalTableColumnRequest) *CreateExternalTableRequest { + s.columns = columns + return s +} + +func (s *CreateExternalTableRequest) WithCloudProviderParams(cloudProviderParams *CloudProviderParamsRequest) *CreateExternalTableRequest { + s.cloudProviderParams = cloudProviderParams + return s +} + +func (s *CreateExternalTableRequest) WithPartitionBy(partitionBy []string) *CreateExternalTableRequest { + s.partitionBy = partitionBy + return s +} + +func (s *CreateExternalTableRequest) WithRefreshOnCreate(refreshOnCreate *bool) *CreateExternalTableRequest { + s.refreshOnCreate = refreshOnCreate + return s +} + +func (s *CreateExternalTableRequest) WithAutoRefresh(autoRefresh *bool) *CreateExternalTableRequest { + s.autoRefresh = autoRefresh + return s +} + +func (s *CreateExternalTableRequest) WithPattern(pattern *string) *CreateExternalTableRequest { + s.pattern = pattern + return s +} + +func (s *CreateExternalTableRequest) WithAwsSnsTopic(awsSnsTopic *string) *CreateExternalTableRequest { + s.awsSnsTopic = awsSnsTopic + return s +} + +func (s *CreateExternalTableRequest) WithCopyGrants(copyGrants *bool) *CreateExternalTableRequest { + s.copyGrants = copyGrants + return s +} + +func (s *CreateExternalTableRequest) WithComment(comment *string) *CreateExternalTableRequest { + s.comment = comment + return s +} + +func (s *CreateExternalTableRequest) WithRowAccessPolicy(rowAccessPolicy *RowAccessPolicyRequest) *CreateExternalTableRequest { + s.rowAccessPolicy = rowAccessPolicy + return s +} + +func (s *CreateExternalTableRequest) WithTag(tag []*TagAssociationRequest) *CreateExternalTableRequest { + s.tag = tag + return s +} + +func NewExternalTableColumnRequest( + name string, + dataType DataType, + asExpression string, +) *ExternalTableColumnRequest { + s := ExternalTableColumnRequest{} + s.name = name + s.dataType = dataType + s.asExpression = asExpression + return &s +} + +func (s *ExternalTableColumnRequest) WithInlineConstraint(inlineConstraint *ColumnInlineConstraintRequest) *ExternalTableColumnRequest { + s.inlineConstraint = inlineConstraint + return s +} + +func NewColumnInlineConstraintRequest( + name string, + constraintType ColumnConstraintType, +) *ColumnInlineConstraintRequest { + s := ColumnInlineConstraintRequest{} + s.name = name + s.constraintType = constraintType + return &s +} + +func (s *ColumnInlineConstraintRequest) WithNotNull(notNull *bool) *ColumnInlineConstraintRequest { + s.notNull = notNull + return s +} + +func (s *ColumnInlineConstraintRequest) WithForeignKey(foreignKey *InlineForeignKey) *ColumnInlineConstraintRequest { + s.foreignKey = foreignKey + return s +} + +func (s *ColumnInlineConstraintRequest) WithEnforced(enforced *bool) *ColumnInlineConstraintRequest { + s.enforced = enforced + return s +} + +func (s *ColumnInlineConstraintRequest) WithNotEnforced(notEnforced *bool) *ColumnInlineConstraintRequest { + s.notEnforced = notEnforced + return s +} + +func (s *ColumnInlineConstraintRequest) WithDeferrable(deferrable *bool) *ColumnInlineConstraintRequest { + s.deferrable = deferrable + return s +} + +func (s *ColumnInlineConstraintRequest) WithNotDeferrable(notDeferrable *bool) *ColumnInlineConstraintRequest { + s.notDeferrable = notDeferrable + return s +} + +func (s *ColumnInlineConstraintRequest) WithInitiallyDeferred(initiallyDeferred *bool) *ColumnInlineConstraintRequest { + s.initiallyDeferred = initiallyDeferred + return s +} + +func (s *ColumnInlineConstraintRequest) WithInitiallyImmediate(initiallyImmediate *bool) *ColumnInlineConstraintRequest { + s.initiallyImmediate = initiallyImmediate + return s +} + +func (s *ColumnInlineConstraintRequest) WithEnable(enable *bool) *ColumnInlineConstraintRequest { + s.enable = enable + return s +} + +func (s *ColumnInlineConstraintRequest) WithDisable(disable *bool) *ColumnInlineConstraintRequest { + s.disable = disable + return s +} + +func (s *ColumnInlineConstraintRequest) WithValidate(validate *bool) *ColumnInlineConstraintRequest { + s.validate = validate + return s +} + +func (s *ColumnInlineConstraintRequest) WithNoValidate(noValidate *bool) *ColumnInlineConstraintRequest { + s.noValidate = noValidate + return s +} + +func (s *ColumnInlineConstraintRequest) WithRely(rely *bool) *ColumnInlineConstraintRequest { + s.rely = rely + return s +} + +func (s *ColumnInlineConstraintRequest) WithNoRely(noRely *bool) *ColumnInlineConstraintRequest { + s.noRely = noRely + return s +} + +func NewInlineForeignKeyRequest( + tableName string, +) *InlineForeignKeyRequest { + s := InlineForeignKeyRequest{} + s.tableName = tableName + return &s +} + +func (s *InlineForeignKeyRequest) WithColumnName(columnName []string) *InlineForeignKeyRequest { + s.columnName = columnName + return s +} + +func (s *InlineForeignKeyRequest) WithMatch(match *MatchType) *InlineForeignKeyRequest { + s.match = match + return s +} + +func (s *InlineForeignKeyRequest) WithOn(on *ForeignKeyOnActionRequest) *InlineForeignKeyRequest { + s.on = on + return s +} + +func NewForeignKeyOnActionRequest() *ForeignKeyOnActionRequest { + return &ForeignKeyOnActionRequest{} +} + +func (s *ForeignKeyOnActionRequest) WithOnUpdate(onUpdate *bool) *ForeignKeyOnActionRequest { + s.onUpdate = onUpdate + return s +} + +func (s *ForeignKeyOnActionRequest) WithOnDelete(onDelete *bool) *ForeignKeyOnActionRequest { + s.onDelete = onDelete + return s +} + +func NewCloudProviderParamsRequest() *CloudProviderParamsRequest { + return &CloudProviderParamsRequest{} +} + +func (s *CloudProviderParamsRequest) WithGoogleCloudStorageIntegration(googleCloudStorageIntegration *string) *CloudProviderParamsRequest { + s.googleCloudStorageIntegration = googleCloudStorageIntegration + return s +} + +func (s *CloudProviderParamsRequest) WithMicrosoftAzureIntegration(microsoftAzureIntegration *string) *CloudProviderParamsRequest { + s.microsoftAzureIntegration = microsoftAzureIntegration + return s +} + +func NewExternalTableFileFormatRequest() *ExternalTableFileFormatRequest { + return &ExternalTableFileFormatRequest{} +} + +func (s *ExternalTableFileFormatRequest) WithName(name *string) *ExternalTableFileFormatRequest { + s.name = name + return s +} + +func (s *ExternalTableFileFormatRequest) WithFileFormatType(fileFormatType *ExternalTableFileFormatType) *ExternalTableFileFormatRequest { + s.fileFormatType = fileFormatType + return s +} + +func (s *ExternalTableFileFormatRequest) WithOptions(options *ExternalTableFileFormatTypeOptionsRequest) *ExternalTableFileFormatRequest { + s.options = options + return s +} + +func NewExternalTableFileFormatTypeOptionsRequest() *ExternalTableFileFormatTypeOptionsRequest { + return &ExternalTableFileFormatTypeOptionsRequest{} +} + +func (s *ExternalTableFileFormatTypeOptionsRequest) WithCsvCompression(csvCompression *ExternalTableCsvCompression) *ExternalTableFileFormatTypeOptionsRequest { + s.csvCompression = csvCompression + return s +} + +func (s *ExternalTableFileFormatTypeOptionsRequest) WithCsvRecordDelimiter(csvRecordDelimiter *string) *ExternalTableFileFormatTypeOptionsRequest { + s.csvRecordDelimiter = csvRecordDelimiter + return s +} + +func (s *ExternalTableFileFormatTypeOptionsRequest) WithCsvFieldDelimiter(csvFieldDelimiter *string) *ExternalTableFileFormatTypeOptionsRequest { + s.csvFieldDelimiter = csvFieldDelimiter + return s +} + +func (s *ExternalTableFileFormatTypeOptionsRequest) WithCsvSkipHeader(csvSkipHeader *int) *ExternalTableFileFormatTypeOptionsRequest { + s.csvSkipHeader = csvSkipHeader + return s +} + +func (s *ExternalTableFileFormatTypeOptionsRequest) WithCsvSkipBlankLines(csvSkipBlankLines *bool) *ExternalTableFileFormatTypeOptionsRequest { + s.csvSkipBlankLines = csvSkipBlankLines + return s +} + +func (s *ExternalTableFileFormatTypeOptionsRequest) WithCsvEscapeUnenclosedField(csvEscapeUnenclosedField *string) *ExternalTableFileFormatTypeOptionsRequest { + s.csvEscapeUnenclosedField = csvEscapeUnenclosedField + return s +} + +func (s *ExternalTableFileFormatTypeOptionsRequest) WithCsvTrimSpace(csvTrimSpace *bool) *ExternalTableFileFormatTypeOptionsRequest { + s.csvTrimSpace = csvTrimSpace + return s +} + +func (s *ExternalTableFileFormatTypeOptionsRequest) WithCsvFieldOptionallyEnclosedBy(csvFieldOptionallyEnclosedBy *string) *ExternalTableFileFormatTypeOptionsRequest { + s.csvFieldOptionallyEnclosedBy = csvFieldOptionallyEnclosedBy + return s +} + +func (s *ExternalTableFileFormatTypeOptionsRequest) WithCsvNullIf(csvNullIf *[]NullStringRequest) *ExternalTableFileFormatTypeOptionsRequest { + s.csvNullIf = csvNullIf + return s +} + +func (s *ExternalTableFileFormatTypeOptionsRequest) WithCsvEmptyFieldAsNull(csvEmptyFieldAsNull *bool) *ExternalTableFileFormatTypeOptionsRequest { + s.csvEmptyFieldAsNull = csvEmptyFieldAsNull + return s +} + +func (s *ExternalTableFileFormatTypeOptionsRequest) WithCsvEncoding(csvEncoding *CSVEncoding) *ExternalTableFileFormatTypeOptionsRequest { + s.csvEncoding = csvEncoding + return s +} + +func (s *ExternalTableFileFormatTypeOptionsRequest) WithJsonCompression(jsonCompression *ExternalTableJsonCompression) *ExternalTableFileFormatTypeOptionsRequest { + s.jsonCompression = jsonCompression + return s +} + +func (s *ExternalTableFileFormatTypeOptionsRequest) WithJsonAllowDuplicate(jsonAllowDuplicate *bool) *ExternalTableFileFormatTypeOptionsRequest { + s.jsonAllowDuplicate = jsonAllowDuplicate + return s +} + +func (s *ExternalTableFileFormatTypeOptionsRequest) WithJsonStripOuterArray(jsonStripOuterArray *bool) *ExternalTableFileFormatTypeOptionsRequest { + s.jsonStripOuterArray = jsonStripOuterArray + return s +} + +func (s *ExternalTableFileFormatTypeOptionsRequest) WithJsonStripNullValues(jsonStripNullValues *bool) *ExternalTableFileFormatTypeOptionsRequest { + s.jsonStripNullValues = jsonStripNullValues + return s +} + +func (s *ExternalTableFileFormatTypeOptionsRequest) WithJsonReplaceInvalidCharacters(jsonReplaceInvalidCharacters *bool) *ExternalTableFileFormatTypeOptionsRequest { + s.jsonReplaceInvalidCharacters = jsonReplaceInvalidCharacters + return s +} + +func (s *ExternalTableFileFormatTypeOptionsRequest) WithAvroCompression(avroCompression *ExternalTableAvroCompression) *ExternalTableFileFormatTypeOptionsRequest { + s.avroCompression = avroCompression + return s +} + +func (s *ExternalTableFileFormatTypeOptionsRequest) WithAvroReplaceInvalidCharacters(avroReplaceInvalidCharacters *bool) *ExternalTableFileFormatTypeOptionsRequest { + s.avroReplaceInvalidCharacters = avroReplaceInvalidCharacters + return s +} + +func (s *ExternalTableFileFormatTypeOptionsRequest) WithOrcTrimSpace(orcTrimSpace *bool) *ExternalTableFileFormatTypeOptionsRequest { + s.orcTrimSpace = orcTrimSpace + return s +} + +func (s *ExternalTableFileFormatTypeOptionsRequest) WithOrcReplaceInvalidCharacters(orcReplaceInvalidCharacters *bool) *ExternalTableFileFormatTypeOptionsRequest { + s.orcReplaceInvalidCharacters = orcReplaceInvalidCharacters + return s +} + +func (s *ExternalTableFileFormatTypeOptionsRequest) WithOrcNullIf(orcNullIf *[]NullStringRequest) *ExternalTableFileFormatTypeOptionsRequest { + s.orcNullIf = orcNullIf + return s +} + +func (s *ExternalTableFileFormatTypeOptionsRequest) WithParquetCompression(parquetCompression *ExternalTableParquetCompression) *ExternalTableFileFormatTypeOptionsRequest { + s.parquetCompression = parquetCompression + return s +} + +func (s *ExternalTableFileFormatTypeOptionsRequest) WithParquetBinaryAsText(parquetBinaryAsText *bool) *ExternalTableFileFormatTypeOptionsRequest { + s.parquetBinaryAsText = parquetBinaryAsText + return s +} + +func (s *ExternalTableFileFormatTypeOptionsRequest) WithParquetReplaceInvalidCharacters(parquetReplaceInvalidCharacters *bool) *ExternalTableFileFormatTypeOptionsRequest { + s.parquetReplaceInvalidCharacters = parquetReplaceInvalidCharacters + return s +} + +func NewNullStringRequest() *NullStringRequest { + return &NullStringRequest{} +} + +func (s *NullStringRequest) WithStr(str string) *NullStringRequest { + s.str = str + return s +} + +func NewRowAccessPolicyRequest( + name SchemaObjectIdentifier, + on []string, +) *RowAccessPolicyRequest { + s := RowAccessPolicyRequest{} + s.name = name + s.on = on + return &s +} + +func NewTagAssociationRequest( + name ObjectIdentifier, + value string, +) *TagAssociationRequest { + s := TagAssociationRequest{} + s.name = name + s.value = value + return &s +} + +func NewCreateWithManualPartitioningExternalTableRequest( + name AccountObjectIdentifier, + location string, + fileFormat *ExternalTableFileFormatRequest, +) *CreateWithManualPartitioningExternalTableRequest { + s := CreateWithManualPartitioningExternalTableRequest{} + s.name = name + s.location = location + s.fileFormat = fileFormat + return &s +} + +func (s *CreateWithManualPartitioningExternalTableRequest) WithOrReplace(orReplace *bool) *CreateWithManualPartitioningExternalTableRequest { + s.orReplace = orReplace + return s +} + +func (s *CreateWithManualPartitioningExternalTableRequest) WithIfNotExists(ifNotExists *bool) *CreateWithManualPartitioningExternalTableRequest { + s.ifNotExists = ifNotExists + return s +} + +func (s *CreateWithManualPartitioningExternalTableRequest) WithColumns(columns []*ExternalTableColumnRequest) *CreateWithManualPartitioningExternalTableRequest { + s.columns = columns + return s +} + +func (s *CreateWithManualPartitioningExternalTableRequest) WithCloudProviderParams(cloudProviderParams *CloudProviderParamsRequest) *CreateWithManualPartitioningExternalTableRequest { + s.cloudProviderParams = cloudProviderParams + return s +} + +func (s *CreateWithManualPartitioningExternalTableRequest) WithPartitionBy(partitionBy []string) *CreateWithManualPartitioningExternalTableRequest { + s.partitionBy = partitionBy + return s +} + +func (s *CreateWithManualPartitioningExternalTableRequest) WithUserSpecifiedPartitionType(userSpecifiedPartitionType *bool) *CreateWithManualPartitioningExternalTableRequest { + s.userSpecifiedPartitionType = userSpecifiedPartitionType + return s +} + +func (s *CreateWithManualPartitioningExternalTableRequest) WithCopyGrants(copyGrants *bool) *CreateWithManualPartitioningExternalTableRequest { + s.copyGrants = copyGrants + return s +} + +func (s *CreateWithManualPartitioningExternalTableRequest) WithComment(comment *string) *CreateWithManualPartitioningExternalTableRequest { + s.comment = comment + return s +} + +func (s *CreateWithManualPartitioningExternalTableRequest) WithRowAccessPolicy(rowAccessPolicy *RowAccessPolicyRequest) *CreateWithManualPartitioningExternalTableRequest { + s.rowAccessPolicy = rowAccessPolicy + return s +} + +func (s *CreateWithManualPartitioningExternalTableRequest) WithTag(tag []*TagAssociationRequest) *CreateWithManualPartitioningExternalTableRequest { + s.tag = tag + return s +} + +func NewCreateDeltaLakeExternalTableRequest( + name AccountObjectIdentifier, + location string, + fileFormat *ExternalTableFileFormatRequest, +) *CreateDeltaLakeExternalTableRequest { + s := CreateDeltaLakeExternalTableRequest{} + s.name = name + s.location = location + s.fileFormat = fileFormat + return &s +} + +func (s *CreateDeltaLakeExternalTableRequest) WithOrReplace(orReplace *bool) *CreateDeltaLakeExternalTableRequest { + s.orReplace = orReplace + return s +} + +func (s *CreateDeltaLakeExternalTableRequest) WithIfNotExists(ifNotExists *bool) *CreateDeltaLakeExternalTableRequest { + s.ifNotExists = ifNotExists + return s +} + +func (s *CreateDeltaLakeExternalTableRequest) WithColumns(columns []*ExternalTableColumnRequest) *CreateDeltaLakeExternalTableRequest { + s.columns = columns + return s +} + +func (s *CreateDeltaLakeExternalTableRequest) WithCloudProviderParams(cloudProviderParams *CloudProviderParamsRequest) *CreateDeltaLakeExternalTableRequest { + s.cloudProviderParams = cloudProviderParams + return s +} + +func (s *CreateDeltaLakeExternalTableRequest) WithPartitionBy(partitionBy []string) *CreateDeltaLakeExternalTableRequest { + s.partitionBy = partitionBy + return s +} + +func (s *CreateDeltaLakeExternalTableRequest) WithUserSpecifiedPartitionType(userSpecifiedPartitionType *bool) *CreateDeltaLakeExternalTableRequest { + s.userSpecifiedPartitionType = userSpecifiedPartitionType + return s +} + +func (s *CreateDeltaLakeExternalTableRequest) WithRefreshOnCreate(refreshOnCreate *bool) *CreateDeltaLakeExternalTableRequest { + s.refreshOnCreate = refreshOnCreate + return s +} + +func (s *CreateDeltaLakeExternalTableRequest) WithAutoRefresh(autoRefresh *bool) *CreateDeltaLakeExternalTableRequest { + s.autoRefresh = autoRefresh + return s +} + +func (s *CreateDeltaLakeExternalTableRequest) WithDeltaTableFormat(deltaTableFormat *bool) *CreateDeltaLakeExternalTableRequest { + s.deltaTableFormat = deltaTableFormat + return s +} + +func (s *CreateDeltaLakeExternalTableRequest) WithCopyGrants(copyGrants *bool) *CreateDeltaLakeExternalTableRequest { + s.copyGrants = copyGrants + return s +} + +func (s *CreateDeltaLakeExternalTableRequest) WithComment(comment *string) *CreateDeltaLakeExternalTableRequest { + s.comment = comment + return s +} + +func (s *CreateDeltaLakeExternalTableRequest) WithRowAccessPolicy(rowAccessPolicy *RowAccessPolicyRequest) *CreateDeltaLakeExternalTableRequest { + s.rowAccessPolicy = rowAccessPolicy + return s +} + +func (s *CreateDeltaLakeExternalTableRequest) WithTag(tag []*TagAssociationRequest) *CreateDeltaLakeExternalTableRequest { + s.tag = tag + return s +} + +func NewCreateExternalTableUsingTemplateRequest( + name AccountObjectIdentifier, + location string, + fileFormat *ExternalTableFileFormatRequest, +) *CreateExternalTableUsingTemplateRequest { + s := CreateExternalTableUsingTemplateRequest{} + s.name = name + s.location = location + s.fileFormat = fileFormat + return &s +} + +func (s *CreateExternalTableUsingTemplateRequest) WithOrReplace(orReplace *bool) *CreateExternalTableUsingTemplateRequest { + s.orReplace = orReplace + return s +} + +func (s *CreateExternalTableUsingTemplateRequest) WithCopyGrants(copyGrants *bool) *CreateExternalTableUsingTemplateRequest { + s.copyGrants = copyGrants + return s +} + +func (s *CreateExternalTableUsingTemplateRequest) WithQuery(query string) *CreateExternalTableUsingTemplateRequest { + s.query = query + return s +} + +func (s *CreateExternalTableUsingTemplateRequest) WithCloudProviderParams(cloudProviderParams *CloudProviderParamsRequest) *CreateExternalTableUsingTemplateRequest { + s.cloudProviderParams = cloudProviderParams + return s +} + +func (s *CreateExternalTableUsingTemplateRequest) WithPartitionBy(partitionBy []string) *CreateExternalTableUsingTemplateRequest { + s.partitionBy = partitionBy + return s +} + +func (s *CreateExternalTableUsingTemplateRequest) WithRefreshOnCreate(refreshOnCreate *bool) *CreateExternalTableUsingTemplateRequest { + s.refreshOnCreate = refreshOnCreate + return s +} + +func (s *CreateExternalTableUsingTemplateRequest) WithAutoRefresh(autoRefresh *bool) *CreateExternalTableUsingTemplateRequest { + s.autoRefresh = autoRefresh + return s +} + +func (s *CreateExternalTableUsingTemplateRequest) WithPattern(pattern *string) *CreateExternalTableUsingTemplateRequest { + s.pattern = pattern + return s +} + +func (s *CreateExternalTableUsingTemplateRequest) WithAwsSnsTopic(awsSnsTopic *string) *CreateExternalTableUsingTemplateRequest { + s.awsSnsTopic = awsSnsTopic + return s +} + +func (s *CreateExternalTableUsingTemplateRequest) WithComment(comment *string) *CreateExternalTableUsingTemplateRequest { + s.comment = comment + return s +} + +func (s *CreateExternalTableUsingTemplateRequest) WithRowAccessPolicy(rowAccessPolicy *RowAccessPolicyRequest) *CreateExternalTableUsingTemplateRequest { + s.rowAccessPolicy = rowAccessPolicy + return s +} + +func (s *CreateExternalTableUsingTemplateRequest) WithTag(tag []*TagAssociationRequest) *CreateExternalTableUsingTemplateRequest { + s.tag = tag + return s +} + +func NewAlterExternalTableRequest( + name AccountObjectIdentifier, +) *AlterExternalTableRequest { + s := AlterExternalTableRequest{} + s.name = name + return &s +} + +func (s *AlterExternalTableRequest) WithIfExists(ifExists *bool) *AlterExternalTableRequest { + s.ifExists = ifExists + return s +} + +func (s *AlterExternalTableRequest) WithRefresh(refresh *RefreshExternalTableRequest) *AlterExternalTableRequest { + s.refresh = refresh + return s +} + +func (s *AlterExternalTableRequest) WithAddFiles(addFiles []*ExternalTableFileRequest) *AlterExternalTableRequest { + s.addFiles = addFiles + return s +} + +func (s *AlterExternalTableRequest) WithRemoveFiles(removeFiles []*ExternalTableFileRequest) *AlterExternalTableRequest { + s.removeFiles = removeFiles + return s +} + +func (s *AlterExternalTableRequest) WithAutoRefresh(autoRefresh *bool) *AlterExternalTableRequest { + s.autoRefresh = autoRefresh + return s +} + +func (s *AlterExternalTableRequest) WithSetTag(setTag []*TagAssociationRequest) *AlterExternalTableRequest { + s.setTag = setTag + return s +} + +func (s *AlterExternalTableRequest) WithUnsetTag(unsetTag []ObjectIdentifier) *AlterExternalTableRequest { + s.unsetTag = unsetTag + return s +} + +func NewRefreshExternalTableRequest( + path string, +) *RefreshExternalTableRequest { + s := RefreshExternalTableRequest{} + s.path = path + return &s +} + +func NewExternalTableFileRequest( + name string, +) *ExternalTableFileRequest { + s := ExternalTableFileRequest{} + s.name = name + return &s +} + +func NewAlterExternalTablePartitionRequest( + name AccountObjectIdentifier, +) *AlterExternalTablePartitionRequest { + s := AlterExternalTablePartitionRequest{} + s.name = name + return &s +} + +func (s *AlterExternalTablePartitionRequest) WithIfExists(ifExists *bool) *AlterExternalTablePartitionRequest { + s.ifExists = ifExists + return s +} + +func (s *AlterExternalTablePartitionRequest) WithAddPartitions(addPartitions []*PartitionRequest) *AlterExternalTablePartitionRequest { + s.addPartitions = addPartitions + return s +} + +func (s *AlterExternalTablePartitionRequest) WithDropPartition(dropPartition *bool) *AlterExternalTablePartitionRequest { + s.dropPartition = dropPartition + return s +} + +func (s *AlterExternalTablePartitionRequest) WithLocation(location string) *AlterExternalTablePartitionRequest { + s.location = location + return s +} + +func NewPartitionRequest( + columnName string, + value string, +) *PartitionRequest { + s := PartitionRequest{} + s.columnName = columnName + s.value = value + return &s +} + +func NewDropExternalTableRequest( + name AccountObjectIdentifier, +) *DropExternalTableRequest { + s := DropExternalTableRequest{} + s.name = name + return &s +} + +func (s *DropExternalTableRequest) WithIfExists(ifExists *bool) *DropExternalTableRequest { + s.ifExists = ifExists + return s +} + +func (s *DropExternalTableRequest) WithDropOption(dropOption *ExternalTableDropOptionRequest) *DropExternalTableRequest { + s.dropOption = dropOption + return s +} + +func NewExternalTableDropOptionRequest() *ExternalTableDropOptionRequest { + return &ExternalTableDropOptionRequest{} +} + +func (s *ExternalTableDropOptionRequest) WithRestrict(restrict *bool) *ExternalTableDropOptionRequest { + s.restrict = restrict + return s +} + +func (s *ExternalTableDropOptionRequest) WithCascade(cascade *bool) *ExternalTableDropOptionRequest { + s.cascade = cascade + return s +} + +func NewShowExternalTableRequest() *ShowExternalTableRequest { + return &ShowExternalTableRequest{} +} + +func (s *ShowExternalTableRequest) WithTerse(terse *bool) *ShowExternalTableRequest { + s.terse = terse + return s +} + +func (s *ShowExternalTableRequest) WithLike(like *string) *ShowExternalTableRequest { + s.like = like + return s +} + +func (s *ShowExternalTableRequest) WithIn(in *ShowExternalTableInRequest) *ShowExternalTableRequest { + s.in = in + return s +} + +func (s *ShowExternalTableRequest) WithStartsWith(startsWith *string) *ShowExternalTableRequest { + s.startsWith = startsWith + return s +} + +func (s *ShowExternalTableRequest) WithLimitFrom(limitFrom *LimitFromRequest) *ShowExternalTableRequest { + s.limitFrom = limitFrom + return s +} + +func NewShowExternalTableInRequest() *ShowExternalTableInRequest { + return &ShowExternalTableInRequest{} +} + +func (s *ShowExternalTableInRequest) WithAccount(account *bool) *ShowExternalTableInRequest { + s.account = account + return s +} + +func (s *ShowExternalTableInRequest) WithDatabase(database AccountObjectIdentifier) *ShowExternalTableInRequest { + s.database = database + return s +} + +func (s *ShowExternalTableInRequest) WithSchema(schema DatabaseObjectIdentifier) *ShowExternalTableInRequest { + s.schema = schema + return s +} + +func NewLimitFromRequest() *LimitFromRequest { + return &LimitFromRequest{} +} + +func (s *LimitFromRequest) WithRows(rows *int) *LimitFromRequest { + s.rows = rows + return s +} + +func (s *LimitFromRequest) WithFrom(from *string) *LimitFromRequest { + s.from = from + return s +} + +func NewShowExternalTableByIDRequest( + id AccountObjectIdentifier, +) *ShowExternalTableByIDRequest { + s := ShowExternalTableByIDRequest{} + s.id = id + return &s +} + +func NewDescribeExternalTableColumnsRequest( + id AccountObjectIdentifier, +) *DescribeExternalTableColumnsRequest { + s := DescribeExternalTableColumnsRequest{} + s.id = id + return &s +} + +func NewDescribeExternalTableStageRequest( + id AccountObjectIdentifier, +) *DescribeExternalTableStageRequest { + s := DescribeExternalTableStageRequest{} + s.id = id + return &s +} diff --git a/pkg/sdk/external_tables_impl.go b/pkg/sdk/external_tables_impl.go new file mode 100644 index 0000000000..1c65bf7f46 --- /dev/null +++ b/pkg/sdk/external_tables_impl.go @@ -0,0 +1,79 @@ +package sdk + +import "context" + +var _ ExternalTables = (*externalTables)(nil) + +type externalTables struct { + client *Client +} + +func (v *externalTables) Create(ctx context.Context, req *CreateExternalTableRequest) error { + return validateAndExec(v.client, ctx, req.toOpts()) +} + +func (v *externalTables) CreateWithManualPartitioning(ctx context.Context, req *CreateWithManualPartitioningExternalTableRequest) error { + return validateAndExec(v.client, ctx, req.toOpts()) +} + +func (v *externalTables) CreateDeltaLake(ctx context.Context, req *CreateDeltaLakeExternalTableRequest) error { + return validateAndExec(v.client, ctx, req.toOpts()) +} + +func (v *externalTables) CreateUsingTemplate(ctx context.Context, req *CreateExternalTableUsingTemplateRequest) error { + return validateAndExec(v.client, ctx, req.toOpts()) +} + +func (v *externalTables) Alter(ctx context.Context, req *AlterExternalTableRequest) error { + return validateAndExec(v.client, ctx, req.toOpts()) +} + +func (v *externalTables) AlterPartitions(ctx context.Context, req *AlterExternalTablePartitionRequest) error { + return validateAndExec(v.client, ctx, req.toOpts()) +} + +func (v *externalTables) Drop(ctx context.Context, req *DropExternalTableRequest) error { + return validateAndExec(v.client, ctx, req.toOpts()) +} + +func (v *externalTables) Show(ctx context.Context, req *ShowExternalTableRequest) ([]ExternalTable, error) { + dbRows, err := validateAndQuery[externalTableRow](v.client, ctx, req.toOpts()) + if err != nil { + return nil, err + } + resultList := convertRows[externalTableRow, ExternalTable](dbRows) + return resultList, nil +} + +func (v *externalTables) ShowByID(ctx context.Context, req *ShowExternalTableByIDRequest) (*ExternalTable, error) { + if !validObjectidentifier(req.id) { + return nil, errInvalidObjectIdentifier + } + + externalTables, err := v.client.ExternalTables.Show(ctx, NewShowExternalTableRequest().WithLike(String(req.id.Name()))) + if err != nil { + return nil, err + } + + return findOne(externalTables, func(t ExternalTable) bool { return t.ID() == req.id }) +} + +func (v *externalTables) DescribeColumns(ctx context.Context, req *DescribeExternalTableColumnsRequest) ([]ExternalTableColumnDetails, error) { + rows, err := validateAndQuery[externalTableColumnDetailsRow](v.client, ctx, &describeExternalTableColumns{ + name: req.id, + }) + if err != nil { + return nil, err + } + return convertRows[externalTableColumnDetailsRow, ExternalTableColumnDetails](rows), nil +} + +func (v *externalTables) DescribeStage(ctx context.Context, req *DescribeExternalTableStageRequest) ([]ExternalTableStageDetails, error) { + rows, err := validateAndQuery[externalTableStageDetailsRow](v.client, ctx, &describeExternalTableStage{ + name: req.id, + }) + if err != nil { + return nil, err + } + return convertRows[externalTableStageDetailsRow, ExternalTableStageDetails](rows), nil +} diff --git a/pkg/sdk/external_tables_integration_test.go b/pkg/sdk/external_tables_integration_test.go new file mode 100644 index 0000000000..062cadabe0 --- /dev/null +++ b/pkg/sdk/external_tables_integration_test.go @@ -0,0 +1,400 @@ +package sdk + +import ( + "context" + "fmt" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestInt_ExternalTables(t *testing.T) { + client := testClient(t) + ctx := context.Background() + + db, cleanupDB := createDatabase(t, client) + t.Cleanup(cleanupDB) + + schema, _ := createSchema(t, client, db) + + err := client.Sessions.UseDatabase(ctx, db.ID()) + require.NoError(t, err) + err = client.Sessions.UseSchema(ctx, schema.ID()) + require.NoError(t, err) + + stageID := NewAccountObjectIdentifier("EXTERNAL_TABLE_STAGE") + stageLocation := "@external_table_stage" + _, _ = createStageWithURL(t, client, stageID, "s3://snowflake-workshop-lab/weather-nyc") + + tag, _ := createTag(t, client, db, schema) + + defaultColumns := func() []*ExternalTableColumnRequest { + return []*ExternalTableColumnRequest{ + NewExternalTableColumnRequest("filename", DataTypeString, "metadata$filename::string"), + NewExternalTableColumnRequest("city", DataTypeString, "value:city:findname::string"), + NewExternalTableColumnRequest("time", DataTypeTimestamp, "to_timestamp(value:time::int)"), + NewExternalTableColumnRequest("weather", DataTypeVariant, "value:weather::variant"), + } + } + + columns := defaultColumns() + columnsWithPartition := append(defaultColumns(), []*ExternalTableColumnRequest{ + NewExternalTableColumnRequest("weather_date", DataTypeDate, "to_date(to_timestamp(value:time::int))"), + NewExternalTableColumnRequest("part_date", DataTypeDate, "parse_json(metadata$external_table_partition):weather_date::date"), + }...) + + minimalCreateExternalTableReq := func(id AccountObjectIdentifier) *CreateExternalTableRequest { + return NewCreateExternalTableRequest( + id, + stageLocation, + NewExternalTableFileFormatRequest().WithFileFormatType(&ExternalTableFileFormatTypeJSON), + ) + } + + createExternalTableWithManualPartitioningReq := func(id AccountObjectIdentifier) *CreateWithManualPartitioningExternalTableRequest { + return NewCreateWithManualPartitioningExternalTableRequest( + id, + stageLocation, + NewExternalTableFileFormatRequest().WithFileFormatType(&ExternalTableFileFormatTypeJSON), + ). + WithOrReplace(Bool(true)). + WithColumns(columnsWithPartition). + WithUserSpecifiedPartitionType(Bool(true)). + WithPartitionBy([]string{"part_date"}). + WithCopyGrants(Bool(true)). + WithComment(String("some_comment")). + WithTag([]*TagAssociationRequest{NewTagAssociationRequest(tag.ID(), "tag-value")}) + } + + t.Run("Create: minimal", func(t *testing.T) { + externalTableID := randomAccountObjectIdentifier(t) + err := client.ExternalTables.Create(ctx, minimalCreateExternalTableReq(externalTableID)) + require.NoError(t, err) + + externalTable, err := client.ExternalTables.ShowByID(ctx, NewShowExternalTableByIDRequest(externalTableID)) + require.NoError(t, err) + assert.Equal(t, externalTableID.Name(), externalTable.Name) + }) + + t.Run("Create: complete", func(t *testing.T) { + externalTableID := randomAccountObjectIdentifier(t) + err := client.ExternalTables.Create( + ctx, + NewCreateExternalTableRequest( + externalTableID, + stageLocation, + NewExternalTableFileFormatRequest().WithFileFormatType(&ExternalTableFileFormatTypeJSON), + ). + WithOrReplace(Bool(true)). + WithColumns(columns). + WithPartitionBy([]string{"filename"}). + WithRefreshOnCreate(Bool(false)). + WithAutoRefresh(Bool(false)). + WithPattern(String("weather-nyc/weather_2_3_0.json.gz")). + WithCopyGrants(Bool(true)). + WithComment(String("some_comment")). + WithTag([]*TagAssociationRequest{NewTagAssociationRequest(tag.ID(), "tag-value")}), + ) + require.NoError(t, err) + + externalTable, err := client.ExternalTables.ShowByID(ctx, NewShowExternalTableByIDRequest(externalTableID)) + require.NoError(t, err) + assert.Equal(t, externalTableID.Name(), externalTable.Name) + }) + + t.Run("Create: infer schema", func(t *testing.T) { + fileFormat, _ := createFileFormat(t, client, schema.ID()) + warehouse, warehouseCleanup := createWarehouse(t, client) + t.Cleanup(warehouseCleanup) + + err = client.Sessions.UseWarehouse(ctx, warehouse.ID()) + require.NoError(t, err) + + id := randomAccountObjectIdentifier(t) + query := fmt.Sprintf(`SELECT ARRAY_AGG(OBJECT_CONSTRUCT(*)) WITHIN GROUP (ORDER BY order_id) FROM TABLE (INFER_SCHEMA(location => '%s', FILE_FORMAT=>'%s', ignore_case => true))`, stageLocation, fileFormat.ID().FullyQualifiedName()) + err = client.ExternalTables.CreateUsingTemplate( + ctx, + NewCreateExternalTableUsingTemplateRequest( + id, + stageLocation, + NewExternalTableFileFormatRequest().WithName(String(fileFormat.ID().FullyQualifiedName())), + ). + WithQuery(query). + WithAutoRefresh(Bool(false))) + require.NoError(t, err) + + _, err = client.ExternalTables.ShowByID(ctx, NewShowExternalTableByIDRequest(id)) + require.NoError(t, err) + }) + + t.Run("Create with manual partitioning: complete", func(t *testing.T) { + externalTableID := randomAccountObjectIdentifier(t) + err := client.ExternalTables.CreateWithManualPartitioning(ctx, createExternalTableWithManualPartitioningReq(externalTableID)) + require.NoError(t, err) + + externalTable, err := client.ExternalTables.ShowByID(ctx, NewShowExternalTableByIDRequest(externalTableID)) + require.NoError(t, err) + assert.Equal(t, externalTableID.Name(), externalTable.Name) + }) + + t.Run("Create delta lake: complete", func(t *testing.T) { + externalTableID := randomAccountObjectIdentifier(t) + err := client.ExternalTables.CreateDeltaLake( + ctx, + NewCreateDeltaLakeExternalTableRequest( + externalTableID, + stageLocation, + NewExternalTableFileFormatRequest().WithFileFormatType(&ExternalTableFileFormatTypeParquet), + ). + WithOrReplace(Bool(true)). + WithColumns(columnsWithPartition). + WithPartitionBy([]string{"filename"}). + WithDeltaTableFormat(Bool(true)). + WithAutoRefresh(Bool(false)). + WithRefreshOnCreate(Bool(false)). + WithCopyGrants(Bool(true)). + WithComment(String("some_comment")). + WithTag([]*TagAssociationRequest{NewTagAssociationRequest(tag.ID(), "tag-value")}), + ) + require.NoError(t, err) + + externalTable, err := client.ExternalTables.ShowByID(ctx, NewShowExternalTableByIDRequest(externalTableID)) + require.NoError(t, err) + assert.Equal(t, externalTableID.Name(), externalTable.Name) + }) + + t.Run("Alter: refresh", func(t *testing.T) { + externalTableID := randomAccountObjectIdentifier(t) + err := client.ExternalTables.Create(ctx, minimalCreateExternalTableReq(externalTableID)) + require.NoError(t, err) + + err = client.ExternalTables.Alter( + ctx, + NewAlterExternalTableRequest(externalTableID). + WithIfExists(Bool(true)). + WithRefresh(NewRefreshExternalTableRequest("weather-nyc")), + ) + require.NoError(t, err) + }) + + t.Run("Alter: add files", func(t *testing.T) { + externalTableID := randomAccountObjectIdentifier(t) + err := client.ExternalTables.Create( + ctx, + minimalCreateExternalTableReq(externalTableID). + WithPattern(String("weather-nyc/weather_2_3_0.json.gz")), + ) + require.NoError(t, err) + + err = client.ExternalTables.Alter( + ctx, + NewAlterExternalTableRequest(externalTableID). + WithIfExists(Bool(true)). + WithAddFiles([]*ExternalTableFileRequest{NewExternalTableFileRequest("weather-nyc/weather_0_0_0.json.gz")}), + ) + require.NoError(t, err) + }) + + t.Run("Alter: remove files", func(t *testing.T) { + externalTableID := randomAccountObjectIdentifier(t) + err := client.ExternalTables.Create( + ctx, + minimalCreateExternalTableReq(externalTableID). + WithPattern(String("weather-nyc/weather_2_3_0.json.gz")), + ) + require.NoError(t, err) + + err = client.ExternalTables.Alter( + ctx, + NewAlterExternalTableRequest(externalTableID). + WithIfExists(Bool(true)). + WithAddFiles([]*ExternalTableFileRequest{NewExternalTableFileRequest("weather-nyc/weather_0_0_0.json.gz")}), + ) + require.NoError(t, err) + + err = client.ExternalTables.Alter( + ctx, + NewAlterExternalTableRequest(externalTableID). + WithIfExists(Bool(true)). + WithRemoveFiles([]*ExternalTableFileRequest{NewExternalTableFileRequest("weather-nyc/weather_0_0_0.json.gz")}), + ) + require.NoError(t, err) + }) + + t.Run("Alter: set auto refresh", func(t *testing.T) { + externalTableID := randomAccountObjectIdentifier(t) + err := client.ExternalTables.Create(ctx, minimalCreateExternalTableReq(externalTableID)) + require.NoError(t, err) + + err = client.ExternalTables.Alter( + ctx, + NewAlterExternalTableRequest(externalTableID). + WithIfExists(Bool(true)). + WithAutoRefresh(Bool(true)), + ) + require.NoError(t, err) + }) + + // TODO: (SNOW-919981) Uncomment when the problem with alter external table set / unset tags is solved + // t.Run("Alter: set tags", func(t *testing.T) { + // externalTableID := randomAccountObjectIdentifier(t) + // err := client.ExternalTables.Create(ctx, minimalCreateExternalTableReq(externalTableID)) + // require.NoError(t, err) + // + // tagValue := "tag-value" + // err = client.ExternalTables.Alter( + // ctx, + // NewAlterExternalTableRequest(externalTableID). + // WithIfExists(Bool(true)). + // WithSetTag([]*TagAssociationRequest{NewTagAssociationRequest(tag.ID(), tagValue)})) + // require.NoError(t, err) + // + // tv, err := client.SystemFunctions.GetTag(ctx, tag.ID(), externalTableID, ObjectTypeExternalTable) + // require.NoError(t, err) + // assert.Equal(t, tagValue, tv) + // }) + // + // t.Run("Alter: unset tags", func(t *testing.T) { + // externalTableID := randomAccountObjectIdentifier(t) + // err := client.ExternalTables.Create( + // ctx, + // minimalCreateExternalTableReq(externalTableID). + // WithTag([]*TagAssociationRequest{NewTagAssociationRequest(tag.ID(), "tag-value")}), + // ) + // require.NoError(t, err) + // tv, err := client.SystemFunctions.GetTag(ctx, tag.ID(), externalTableID, ObjectTypeExternalTable) + // require.NoError(t, err) + // assert.Equal(t, "tag-value", tv) + // + // err = client.ExternalTables.Alter( + // ctx, + // NewAlterExternalTableRequest(externalTableID). + // WithIfExists(Bool(true)). + // WithUnsetTag([]ObjectIdentifier{ + // NewAccountObjectIdentifier(tag.ID().Name()), + // }), + // ) + // require.NoError(t, err) + // + // _, err = client.SystemFunctions.GetTag(ctx, tag.ID(), externalTableID, ObjectTypeExternalTable) + // require.Error(t, err) + // }) + + t.Run("Alter: add partitions", func(t *testing.T) { + externalTableID := randomAccountObjectIdentifier(t) + err := client.ExternalTables.CreateWithManualPartitioning(ctx, createExternalTableWithManualPartitioningReq(externalTableID)) + require.NoError(t, err) + + err = client.ExternalTables.AlterPartitions( + ctx, + NewAlterExternalTablePartitionRequest(externalTableID). + WithIfExists(Bool(true)). + WithAddPartitions([]*PartitionRequest{NewPartitionRequest("part_date", "2019-06-25")}). + WithLocation("2019/06"), + ) + require.NoError(t, err) + }) + + t.Run("Alter: drop partitions", func(t *testing.T) { + externalTableID := randomAccountObjectIdentifier(t) + err := client.ExternalTables.CreateWithManualPartitioning(ctx, createExternalTableWithManualPartitioningReq(externalTableID)) + require.NoError(t, err) + + err = client.ExternalTables.AlterPartitions( + ctx, + NewAlterExternalTablePartitionRequest(externalTableID). + WithIfExists(Bool(true)). + WithAddPartitions([]*PartitionRequest{NewPartitionRequest("part_date", "2019-06-25")}). + WithLocation("2019/06"), + ) + require.NoError(t, err) + + err = client.ExternalTables.AlterPartitions( + ctx, + NewAlterExternalTablePartitionRequest(externalTableID). + WithIfExists(Bool(true)). + WithDropPartition(Bool(true)). + WithLocation("2019/06"), + ) + require.NoError(t, err) + }) + + t.Run("Drop", func(t *testing.T) { + externalTableID := randomAccountObjectIdentifier(t) + err := client.ExternalTables.Create(ctx, minimalCreateExternalTableReq(externalTableID)) + require.NoError(t, err) + + err = client.ExternalTables.Drop( + ctx, + NewDropExternalTableRequest(externalTableID). + WithIfExists(Bool(true)). + WithDropOption(NewExternalTableDropOptionRequest().WithCascade(Bool(true))), + ) + require.NoError(t, err) + + _, err = client.ExternalTables.ShowByID(ctx, NewShowExternalTableByIDRequest(externalTableID)) + require.ErrorIs(t, err, errObjectNotExistOrAuthorized) + }) + + t.Run("Show", func(t *testing.T) { + externalTableID := randomAccountObjectIdentifier(t) + err := client.ExternalTables.Create(ctx, minimalCreateExternalTableReq(externalTableID)) + require.NoError(t, err) + + et, err := client.ExternalTables.Show( + ctx, + NewShowExternalTableRequest(). + WithTerse(Bool(true)). + WithLike(String(externalTableID.Name())). + WithIn(NewShowExternalTableInRequest().WithDatabase(db.ID())). + WithStartsWith(String(externalTableID.Name())). + WithLimitFrom(NewLimitFromRequest().WithRows(Int(1))), + ) + require.NoError(t, err) + assert.Equal(t, 1, len(et)) + assert.Equal(t, externalTableID, et[0].ID()) + }) + + t.Run("Describe: columns", func(t *testing.T) { + externalTableID := randomAccountObjectIdentifier(t) + req := minimalCreateExternalTableReq(externalTableID) + err := client.ExternalTables.Create(ctx, req) + require.NoError(t, err) + + d, err := client.ExternalTables.DescribeColumns(ctx, NewDescribeExternalTableColumnsRequest(externalTableID)) + require.NoError(t, err) + + assert.Equal(t, len(req.columns)+1, len(d)) // +1 because there's underlying Value column + assert.Contains(t, d, ExternalTableColumnDetails{ + Name: "VALUE", + Type: "VARIANT", + Kind: "COLUMN", + IsNullable: true, + Default: nil, + IsPrimary: false, + IsUnique: false, + Check: nil, + Expression: nil, + Comment: String("The value of this row"), + PolicyName: nil, + }) + }) + + t.Run("Describe: stage", func(t *testing.T) { + externalTableID := randomAccountObjectIdentifier(t) + err := client.ExternalTables.Create(ctx, minimalCreateExternalTableReq(externalTableID)) + require.NoError(t, err) + + d, err := client.ExternalTables.DescribeStage(ctx, NewDescribeExternalTableStageRequest(externalTableID)) + require.NoError(t, err) + + assert.Contains(t, d, ExternalTableStageDetails{ + ParentProperty: "STAGE_FILE_FORMAT", + Property: "TIME_FORMAT", + PropertyType: "String", + PropertyValue: "AUTO", + PropertyDefault: "AUTO", + }) + }) +} diff --git a/pkg/sdk/external_tables_test.go b/pkg/sdk/external_tables_test.go new file mode 100644 index 0000000000..c2ca7f1f2c --- /dev/null +++ b/pkg/sdk/external_tables_test.go @@ -0,0 +1,518 @@ +package sdk + +import ( + "testing" +) + +func TestExternalTablesCreate(t *testing.T) { + t.Run("basic options", func(t *testing.T) { + opts := &CreateExternalTableOptions{ + IfNotExists: Bool(true), + name: NewAccountObjectIdentifier("external_table"), + Columns: []ExternalTableColumn{ + { + Name: "column", + Type: "varchar", + AsExpression: []string{"value::column::varchar"}, + InlineConstraint: &ColumnInlineConstraint{ + Name: String("my_constraint"), + NotNull: Bool(true), + Type: &ColumnConstraintTypeUnique, + }, + }, + }, + CloudProviderParams: &CloudProviderParams{ + GoogleCloudStorageIntegration: String("123"), + }, + Location: "@s1/logs/", + FileFormat: []ExternalTableFileFormat{ + { + Type: &ExternalTableFileFormatTypeJSON, + }, + }, + } + assertOptsValidAndSQLEquals(t, opts, `CREATE EXTERNAL TABLE IF NOT EXISTS "external_table" (column varchar AS (value::column::varchar) NOT NULL CONSTRAINT my_constraint UNIQUE) INTEGRATION = '123' LOCATION = @s1/logs/ FILE_FORMAT = (TYPE = JSON)`) + }) + + t.Run("every optional field", func(t *testing.T) { + opts := &CreateExternalTableOptions{ + OrReplace: Bool(true), + name: NewAccountObjectIdentifier("external_table"), + Columns: []ExternalTableColumn{ + { + Name: "column", + Type: "varchar", + AsExpression: []string{"value::column::varchar"}, + InlineConstraint: &ColumnInlineConstraint{ + Name: String("my_constraint"), + NotNull: Bool(true), + Type: &ColumnConstraintTypeUnique, + }, + }, + }, + CloudProviderParams: &CloudProviderParams{ + GoogleCloudStorageIntegration: String("123"), + }, + Location: "@s1/logs/", + FileFormat: []ExternalTableFileFormat{ + { + Type: &ExternalTableFileFormatTypeJSON, + }, + }, + AwsSnsTopic: String("aws_sns_topic"), + CopyGrants: Bool(true), + RowAccessPolicy: &RowAccessPolicy{ + Name: NewSchemaObjectIdentifier("db", "schema", "row_access_policy"), + On: []string{"value1", "value2"}, + }, + Tag: []TagAssociation{ + { + Name: NewAccountObjectIdentifier("tag1"), + Value: "value1", + }, + { + Name: NewAccountObjectIdentifier("tag2"), + Value: "value2", + }, + }, + Comment: String("some_comment"), + } + assertOptsValidAndSQLEquals(t, opts, `CREATE OR REPLACE EXTERNAL TABLE "external_table" (column varchar AS (value::column::varchar) NOT NULL CONSTRAINT my_constraint UNIQUE) INTEGRATION = '123' LOCATION = @s1/logs/ FILE_FORMAT = (TYPE = JSON) AWS_SNS_TOPIC = 'aws_sns_topic' COPY GRANTS COMMENT = 'some_comment' ROW ACCESS POLICY "db"."schema"."row_access_policy" ON (value1, value2) TAG ("tag1" = 'value1', "tag2" = 'value2')`) + }) + + t.Run("invalid options", func(t *testing.T) { + opts := &CreateExternalTableOptions{ + OrReplace: Bool(true), + IfNotExists: Bool(true), + name: NewAccountObjectIdentifier(""), + } + assertOptsInvalidJoinedErrors( + t, opts, + errOneOf("CreateExternalTableOptions", "OrReplace", "IfNotExists"), + errInvalidObjectIdentifier, + errNotSet("CreateExternalTableOptions", "Location"), + errNotSet("CreateExternalTableOptions", "FileFormat"), + ) + }) +} + +func TestExternalTablesCreateWithManualPartitioning(t *testing.T) { + t.Run("valid options", func(t *testing.T) { + opts := &CreateWithManualPartitioningExternalTableOptions{ + OrReplace: Bool(true), + name: NewAccountObjectIdentifier("external_table"), + Columns: []ExternalTableColumn{ + { + Name: "column", + Type: "varchar", + AsExpression: []string{"value::column::varchar"}, + InlineConstraint: &ColumnInlineConstraint{ + Name: String("my_constraint"), + NotNull: Bool(true), + Type: &ColumnConstraintTypeUnique, + }, + }, + }, + CloudProviderParams: &CloudProviderParams{ + GoogleCloudStorageIntegration: String("123"), + }, + Location: "@s1/logs/", + FileFormat: []ExternalTableFileFormat{ + { + Type: &ExternalTableFileFormatTypeJSON, + }, + }, + CopyGrants: Bool(true), + RowAccessPolicy: &RowAccessPolicy{ + Name: NewSchemaObjectIdentifier("db", "schema", "row_access_policy"), + On: []string{"value1", "value2"}, + }, + Tag: []TagAssociation{ + { + Name: NewAccountObjectIdentifier("tag1"), + Value: "value1", + }, + { + Name: NewAccountObjectIdentifier("tag2"), + Value: "value2", + }, + }, + Comment: String("some_comment"), + } + assertOptsValidAndSQLEquals(t, opts, `CREATE OR REPLACE EXTERNAL TABLE "external_table" (column varchar AS (value::column::varchar) NOT NULL CONSTRAINT my_constraint UNIQUE) INTEGRATION = '123' LOCATION = @s1/logs/ FILE_FORMAT = (TYPE = JSON) COPY GRANTS COMMENT = 'some_comment' ROW ACCESS POLICY "db"."schema"."row_access_policy" ON (value1, value2) TAG ("tag1" = 'value1', "tag2" = 'value2')`) + }) + + t.Run("invalid options", func(t *testing.T) { + opts := &CreateWithManualPartitioningExternalTableOptions{ + OrReplace: Bool(true), + IfNotExists: Bool(true), + name: NewAccountObjectIdentifier(""), + } + assertOptsInvalidJoinedErrors( + t, opts, + errOneOf("CreateWithManualPartitioningExternalTableOptions", "OrReplace", "IfNotExists"), + errInvalidObjectIdentifier, + errNotSet("CreateWithManualPartitioningExternalTableOptions", "Location"), + errNotSet("CreateWithManualPartitioningExternalTableOptions", "FileFormat"), + ) + }) +} + +func TestExternalTablesCreateDeltaLake(t *testing.T) { + t.Run("valid options", func(t *testing.T) { + opts := &CreateDeltaLakeExternalTableOptions{ + OrReplace: Bool(true), + name: NewAccountObjectIdentifier("external_table"), + Columns: []ExternalTableColumn{ + { + Name: "column", + Type: "varchar", + AsExpression: []string{"value::column::varchar"}, + InlineConstraint: nil, + }, + }, + CloudProviderParams: &CloudProviderParams{ + MicrosoftAzureIntegration: String("123"), + }, + PartitionBy: []string{"column"}, + Location: "@s1/logs/", + FileFormat: []ExternalTableFileFormat{ + { + Name: String("JSON"), + }, + }, + DeltaTableFormat: Bool(true), + CopyGrants: Bool(true), + RowAccessPolicy: &RowAccessPolicy{ + Name: NewSchemaObjectIdentifier("db", "schema", "row_access_policy"), + On: []string{"value1", "value2"}, + }, + Tag: []TagAssociation{ + { + Name: NewAccountObjectIdentifier("tag1"), + Value: "value1", + }, + { + Name: NewAccountObjectIdentifier("tag2"), + Value: "value2", + }, + }, + Comment: String("some_comment"), + } + assertOptsValidAndSQLEquals(t, opts, `CREATE OR REPLACE EXTERNAL TABLE "external_table" (column varchar AS (value::column::varchar)) INTEGRATION = '123' PARTITION BY (column) LOCATION = @s1/logs/ FILE_FORMAT = (FORMAT_NAME = 'JSON') TABLE_FORMAT = DELTA COPY GRANTS COMMENT = 'some_comment' ROW ACCESS POLICY "db"."schema"."row_access_policy" ON (value1, value2) TAG ("tag1" = 'value1', "tag2" = 'value2')`) + }) + + t.Run("invalid options", func(t *testing.T) { + opts := &CreateDeltaLakeExternalTableOptions{ + OrReplace: Bool(true), + IfNotExists: Bool(true), + name: NewAccountObjectIdentifier(""), + } + assertOptsInvalidJoinedErrors( + t, opts, + errOneOf("CreateDeltaLakeExternalTableOptions", "OrReplace", "IfNotExists"), + errInvalidObjectIdentifier, + errNotSet("CreateDeltaLakeExternalTableOptions", "Location"), + errNotSet("CreateDeltaLakeExternalTableOptions", "FileFormat"), + ) + }) +} + +func TestExternalTableUsingTemplateOpts(t *testing.T) { + t.Run("valid options", func(t *testing.T) { + opts := &CreateExternalTableUsingTemplateOptions{ + OrReplace: Bool(true), + name: NewAccountObjectIdentifier("external_table"), + CopyGrants: Bool(true), + Query: []string{"query statement"}, + CloudProviderParams: &CloudProviderParams{ + MicrosoftAzureIntegration: String("123"), + }, + PartitionBy: []string{"column"}, + Location: "@s1/logs/", + FileFormat: []ExternalTableFileFormat{ + { + Name: String("JSON"), + }, + }, + Comment: String("some_comment"), + RowAccessPolicy: &RowAccessPolicy{ + Name: NewSchemaObjectIdentifier("db", "schema", "row_access_policy"), + On: []string{"value1", "value2"}, + }, + Tag: []TagAssociation{ + { + Name: NewAccountObjectIdentifier("tag1"), + Value: "value1", + }, + { + Name: NewAccountObjectIdentifier("tag2"), + Value: "value2", + }, + }, + } + assertOptsValidAndSQLEquals(t, opts, `CREATE OR REPLACE EXTERNAL TABLE "external_table" COPY GRANTS USING TEMPLATE (query statement) INTEGRATION = '123' PARTITION BY (column) LOCATION = @s1/logs/ FILE_FORMAT = (FORMAT_NAME = 'JSON') COMMENT = 'some_comment' ROW ACCESS POLICY "db"."schema"."row_access_policy" ON (value1, value2) TAG ("tag1" = 'value1', "tag2" = 'value2')`) + }) + + t.Run("invalid options", func(t *testing.T) { + opts := &CreateExternalTableUsingTemplateOptions{ + name: NewAccountObjectIdentifier(""), + } + assertOptsInvalidJoinedErrors( + t, opts, + errInvalidObjectIdentifier, + errNotSet("CreateExternalTableUsingTemplateOptions", "Query"), + errNotSet("CreateExternalTableUsingTemplateOptions", "Location"), + errNotSet("CreateExternalTableUsingTemplateOptions", "FileFormat"), + ) + }) +} + +func TestExternalTablesAlter(t *testing.T) { + t.Run("refresh without path", func(t *testing.T) { + opts := &AlterExternalTableOptions{ + IfExists: Bool(true), + name: NewAccountObjectIdentifier("external_table"), + Refresh: &RefreshExternalTable{}, + } + assertOptsValidAndSQLEquals(t, opts, `ALTER EXTERNAL TABLE IF EXISTS "external_table" REFRESH ''`) + }) + + t.Run("refresh with path", func(t *testing.T) { + opts := &AlterExternalTableOptions{ + IfExists: Bool(true), + name: NewAccountObjectIdentifier("external_table"), + Refresh: &RefreshExternalTable{ + Path: "some/path", + }, + } + assertOptsValidAndSQLEquals(t, opts, `ALTER EXTERNAL TABLE IF EXISTS "external_table" REFRESH 'some/path'`) + }) + + t.Run("add files", func(t *testing.T) { + opts := &AlterExternalTableOptions{ + name: NewAccountObjectIdentifier("external_table"), + AddFiles: []ExternalTableFile{ + {Name: "one/file.txt"}, + {Name: "second/file.txt"}, + }, + } + assertOptsValidAndSQLEquals(t, opts, `ALTER EXTERNAL TABLE "external_table" ADD FILES ('one/file.txt', 'second/file.txt')`) + }) + + t.Run("remove files", func(t *testing.T) { + opts := &AlterExternalTableOptions{ + name: NewAccountObjectIdentifier("external_table"), + RemoveFiles: []ExternalTableFile{ + {Name: "one/file.txt"}, + {Name: "second/file.txt"}, + }, + } + assertOptsValidAndSQLEquals(t, opts, `ALTER EXTERNAL TABLE "external_table" REMOVE FILES ('one/file.txt', 'second/file.txt')`) + }) + + t.Run("set auto refresh", func(t *testing.T) { + opts := &AlterExternalTableOptions{ + name: NewAccountObjectIdentifier("external_table"), + AutoRefresh: Bool(true), + } + assertOptsValidAndSQLEquals(t, opts, `ALTER EXTERNAL TABLE "external_table" SET AUTO_REFRESH = true`) + }) + + t.Run("set tag", func(t *testing.T) { + opts := &AlterExternalTableOptions{ + name: NewAccountObjectIdentifier("external_table"), + SetTag: []TagAssociation{ + { + Name: NewAccountObjectIdentifier("tag1"), + Value: "tag_value1", + }, + { + Name: NewAccountObjectIdentifier("tag2"), + Value: "tag_value2", + }, + }, + } + assertOptsValidAndSQLEquals(t, opts, `ALTER EXTERNAL TABLE "external_table" SET TAG "tag1" = 'tag_value1', "tag2" = 'tag_value2'`) + }) + + t.Run("unset tag", func(t *testing.T) { + opts := &AlterExternalTableOptions{ + name: NewAccountObjectIdentifier("external_table"), + UnsetTag: []ObjectIdentifier{ + NewAccountObjectIdentifier("tag1"), + NewAccountObjectIdentifier("tag2"), + }, + } + assertOptsValidAndSQLEquals(t, opts, `ALTER EXTERNAL TABLE "external_table" UNSET TAG "tag1", "tag2"`) + }) + + t.Run("invalid options", func(t *testing.T) { + opts := &AlterExternalTableOptions{ + name: NewAccountObjectIdentifier(""), + AddFiles: []ExternalTableFile{{Name: "some file"}}, + RemoveFiles: []ExternalTableFile{{Name: "some other file"}}, + } + assertOptsInvalidJoinedErrors( + t, opts, + errInvalidObjectIdentifier, + errOneOf("AlterExternalTableOptions", "Refresh", "AddFiles", "RemoveFiles", "AutoRefresh", "SetTag", "UnsetTag"), + ) + }) +} + +func TestExternalTablesAlterPartitions(t *testing.T) { + t.Run("add partition", func(t *testing.T) { + opts := &AlterExternalTablePartitionOptions{ + name: NewAccountObjectIdentifier("external_table"), + IfExists: Bool(true), + AddPartitions: []Partition{ + { + ColumnName: "one", + Value: "one_value", + }, + { + ColumnName: "two", + Value: "two_value", + }, + }, + Location: "123", + } + assertOptsValidAndSQLEquals(t, opts, `ALTER EXTERNAL TABLE IF EXISTS "external_table" ADD PARTITION (one = 'one_value', two = 'two_value') LOCATION '123'`) + }) + + t.Run("remove partition", func(t *testing.T) { + opts := &AlterExternalTablePartitionOptions{ + name: NewAccountObjectIdentifier("external_table"), + IfExists: Bool(true), + DropPartition: Bool(true), + Location: "partition_location", + } + assertOptsValidAndSQLEquals(t, opts, `ALTER EXTERNAL TABLE IF EXISTS "external_table" DROP PARTITION LOCATION 'partition_location'`) + }) + + t.Run("invalid options", func(t *testing.T) { + opts := &AlterExternalTablePartitionOptions{ + name: NewAccountObjectIdentifier(""), + AddPartitions: []Partition{{ColumnName: "colName", Value: "value"}}, + DropPartition: Bool(true), + } + assertOptsInvalidJoinedErrors( + t, opts, + errInvalidObjectIdentifier, + errOneOf("AlterExternalTablePartitionOptions", "AddPartitions", "DropPartition"), + ) + }) +} + +func TestExternalTablesDrop(t *testing.T) { + t.Run("restrict", func(t *testing.T) { + opts := &DropExternalTableOptions{ + IfExists: Bool(true), + name: NewAccountObjectIdentifier("external_table"), + DropOption: &ExternalTableDropOption{ + Restrict: Bool(true), + }, + } + assertOptsValidAndSQLEquals(t, opts, `DROP EXTERNAL TABLE IF EXISTS "external_table" RESTRICT`) + }) + + t.Run("cascade", func(t *testing.T) { + opts := &DropExternalTableOptions{ + IfExists: Bool(true), + name: NewAccountObjectIdentifier("external_table"), + DropOption: &ExternalTableDropOption{ + Cascade: Bool(true), + }, + } + assertOptsValidAndSQLEquals(t, opts, `DROP EXTERNAL TABLE IF EXISTS "external_table" CASCADE`) + }) + + t.Run("invalid options", func(t *testing.T) { + opts := &DropExternalTableOptions{ + name: NewAccountObjectIdentifier(""), + DropOption: &ExternalTableDropOption{ + Restrict: Bool(true), + Cascade: Bool(true), + }, + } + + assertOptsInvalidJoinedErrors( + t, opts, + errInvalidObjectIdentifier, + errOneOf("ExternalTableDropOption", "Restrict", "Cascade"), + ) + }) +} + +func TestExternalTablesShow(t *testing.T) { + t.Run("all options", func(t *testing.T) { + opts := &ShowExternalTableOptions{ + Terse: Bool(true), + Like: &Like{ + Pattern: String("some_pattern"), + }, + In: &In{ + Account: Bool(true), + }, + StartsWith: String("some_external_table"), + LimitFrom: &LimitFrom{ + Rows: Int(123), + From: String("some_string"), + }, + } + assertOptsValidAndSQLEquals(t, opts, `SHOW TERSE EXTERNAL TABLES LIKE 'some_pattern' IN ACCOUNT STARTS WITH 'some_external_table' LIMIT 123 FROM 'some_string'`) + }) + + t.Run("in database", func(t *testing.T) { + opts := &ShowExternalTableOptions{ + Terse: Bool(true), + In: &In{ + Database: NewAccountObjectIdentifier("database_name"), + }, + } + assertOptsValidAndSQLEquals(t, opts, `SHOW TERSE EXTERNAL TABLES IN DATABASE "database_name"`) + }) + + t.Run("in schema", func(t *testing.T) { + opts := &ShowExternalTableOptions{ + Terse: Bool(true), + In: &In{ + Schema: NewDatabaseObjectIdentifier("database_name", "schema_name"), + }, + } + assertOptsValidAndSQLEquals(t, opts, `SHOW TERSE EXTERNAL TABLES IN SCHEMA "database_name"."schema_name"`) + }) + + t.Run("invalid options", func(t *testing.T) { + opts := &DropExternalTableOptions{ + name: NewAccountObjectIdentifier(""), + DropOption: &ExternalTableDropOption{ + Restrict: Bool(true), + Cascade: Bool(true), + }, + } + + assertOptsInvalidJoinedErrors( + t, opts, + errInvalidObjectIdentifier, + errOneOf("ExternalTableDropOption", "Restrict", "Cascade"), + ) + }) +} + +func TestExternalTablesDescribe(t *testing.T) { + t.Run("type columns", func(t *testing.T) { + opts := &describeExternalTableColumns{ + name: NewAccountObjectIdentifier("external_table"), + } + assertOptsValidAndSQLEquals(t, opts, `DESCRIBE EXTERNAL TABLE "external_table" TYPE = COLUMNS`) + }) + + t.Run("type stage", func(t *testing.T) { + opts := &describeExternalTableStage{ + name: NewAccountObjectIdentifier("external_table"), + } + assertOptsValidAndSQLEquals(t, opts, `DESCRIBE EXTERNAL TABLE "external_table" TYPE = STAGE`) + }) +} diff --git a/pkg/sdk/external_tables_validations.go b/pkg/sdk/external_tables_validations.go new file mode 100644 index 0000000000..1a0d0a2801 --- /dev/null +++ b/pkg/sdk/external_tables_validations.go @@ -0,0 +1,247 @@ +package sdk + +import ( + "errors" + "fmt" +) + +var ( + _ validatable = (*CreateExternalTableOptions)(nil) + _ validatable = (*CreateWithManualPartitioningExternalTableOptions)(nil) + _ validatable = (*CreateDeltaLakeExternalTableOptions)(nil) + _ validatable = (*CreateExternalTableUsingTemplateOptions)(nil) + _ validatable = (*AlterExternalTableOptions)(nil) + _ validatable = (*AlterExternalTablePartitionOptions)(nil) + _ validatable = (*DropExternalTableOptions)(nil) + _ validatable = (*ShowExternalTableOptions)(nil) + _ validatable = (*describeExternalTableColumns)(nil) + _ validatable = (*describeExternalTableStage)(nil) +) + +func (opts *CreateExternalTableOptions) validate() error { + var errs []error + if everyValueSet(opts.OrReplace, opts.IfNotExists) { + errs = append(errs, errOneOf("CreateExternalTableOptions", "OrReplace", "IfNotExists")) + } + if !validObjectidentifier(opts.name) { + errs = append(errs, errInvalidObjectIdentifier) + } + if !valueSet(opts.Location) { + errs = append(errs, errNotSet("CreateExternalTableOptions", "Location")) + } + if !valueSet(opts.FileFormat) { + errs = append(errs, errNotSet("CreateExternalTableOptions", "FileFormat")) + } else { + for i, ff := range opts.FileFormat { + if !valueSet(ff.Name) && !valueSet(ff.Type) { + errs = append(errs, errNotSet(fmt.Sprintf("CreateExternalTableOptions.FileFormat[%d]", i), "Name or Type")) + } + if valueSet(ff.Name) && valueSet(ff.Type) { + errs = append(errs, errOneOf(fmt.Sprintf("CreateExternalTableOptions.FileFormat[%d]", i), "Name or Type")) + } + } + } + return errors.Join(errs...) +} + +func (opts *CreateWithManualPartitioningExternalTableOptions) validate() error { + var errs []error + if everyValueSet(opts.OrReplace, opts.IfNotExists) { + errs = append(errs, errOneOf("CreateWithManualPartitioningExternalTableOptions", "OrReplace", "IfNotExists")) + } + if !validObjectidentifier(opts.name) { + errs = append(errs, errInvalidObjectIdentifier) + } + if !valueSet(opts.Location) { + errs = append(errs, errNotSet("CreateWithManualPartitioningExternalTableOptions", "Location")) + } + if !valueSet(opts.FileFormat) { + errs = append(errs, errNotSet("CreateWithManualPartitioningExternalTableOptions", "FileFormat")) + } else { + for i, ff := range opts.FileFormat { + if !valueSet(ff.Name) && !valueSet(ff.Type) { + errs = append(errs, errNotSet(fmt.Sprintf("CreateWithManualPartitioningExternalTableOptions.FileFormat[%d]", i), "Name or Type")) + } + if valueSet(ff.Name) && valueSet(ff.Type) { + errs = append(errs, errOneOf(fmt.Sprintf("CreateWithManualPartitioningExternalTableOptions.FileFormat[%d]", i), "Name or Type")) + } + } + } + return errors.Join(errs...) +} + +func (opts *CreateDeltaLakeExternalTableOptions) validate() error { + var errs []error + if everyValueSet(opts.OrReplace, opts.IfNotExists) { + errs = append(errs, errOneOf("CreateDeltaLakeExternalTableOptions", "OrReplace", "IfNotExists")) + } + if !validObjectidentifier(opts.name) { + errs = append(errs, errInvalidObjectIdentifier) + } + if !valueSet(opts.Location) { + errs = append(errs, errNotSet("CreateDeltaLakeExternalTableOptions", "Location")) + } + if !valueSet(opts.FileFormat) { + errs = append(errs, errNotSet("CreateDeltaLakeExternalTableOptions", "FileFormat")) + } else { + for i, ff := range opts.FileFormat { + if !valueSet(ff.Name) && !valueSet(ff.Type) { + errs = append(errs, errNotSet(fmt.Sprintf("CreateDeltaLakeExternalTableOptions.FileFormat[%d]", i), "Name or Type")) + } + if valueSet(ff.Name) && valueSet(ff.Type) { + errs = append(errs, errOneOf(fmt.Sprintf("CreateDeltaLakeExternalTableOptions.FileFormat[%d]", i), "Name or Type")) + } + } + } + return errors.Join(errs...) +} + +func (opts *CreateExternalTableUsingTemplateOptions) validate() error { + var errs []error + if !validObjectidentifier(opts.name) { + errs = append(errs, errInvalidObjectIdentifier) + } + if !valueSet(opts.Query) { + errs = append(errs, errNotSet("CreateExternalTableUsingTemplateOptions", "Query")) + } + if !valueSet(opts.Location) { + errs = append(errs, errNotSet("CreateExternalTableUsingTemplateOptions", "Location")) + } + if !valueSet(opts.FileFormat) { + errs = append(errs, errNotSet("CreateExternalTableUsingTemplateOptions", "FileFormat")) + } else { + for i, ff := range opts.FileFormat { + if !valueSet(ff.Name) && !valueSet(ff.Type) { + errs = append(errs, errNotSet(fmt.Sprintf("CreateExternalTableUsingTemplateOptions.FileFormat[%d]", i), "Name or Type")) + } + if valueSet(ff.Name) && valueSet(ff.Type) { + errs = append(errs, errOneOf(fmt.Sprintf("CreateExternalTableUsingTemplateOptions.FileFormat[%d]", i), "Name or Type")) + } + } + } + return errors.Join(errs...) +} + +func (opts *AlterExternalTableOptions) validate() error { + var errs []error + if !validObjectidentifier(opts.name) { + errs = append(errs, errInvalidObjectIdentifier) + } + if anyValueSet(opts.Refresh, opts.AddFiles, opts.RemoveFiles, opts.AutoRefresh, opts.SetTag, opts.UnsetTag) && + !exactlyOneValueSet(opts.Refresh, opts.AddFiles, opts.RemoveFiles, opts.AutoRefresh, opts.SetTag, opts.UnsetTag) { + errs = append(errs, errOneOf("AlterExternalTableOptions", "Refresh", "AddFiles", "RemoveFiles", "AutoRefresh", "SetTag", "UnsetTag")) + } + return errors.Join(errs...) +} + +func (opts *AlterExternalTablePartitionOptions) validate() error { + var errs []error + if !validObjectidentifier(opts.name) { + errs = append(errs, errInvalidObjectIdentifier) + } + if everyValueSet(opts.AddPartitions, opts.DropPartition) { + errs = append(errs, errOneOf("AlterExternalTablePartitionOptions", "AddPartitions", "DropPartition")) + } + return errors.Join(errs...) +} + +func (opts *DropExternalTableOptions) validate() error { + var errs []error + if !validObjectidentifier(opts.name) { + errs = append(errs, errInvalidObjectIdentifier) + } + if valueSet(opts.DropOption) { + if err := opts.DropOption.validate(); err != nil { + errs = append(errs, err) + } + } + return errors.Join(errs...) +} + +func (opts *ShowExternalTableOptions) validate() error { + return nil +} + +func (v *describeExternalTableColumns) validate() error { + if !validObjectidentifier(v.name) { + return errInvalidObjectIdentifier + } + return nil +} + +func (v *describeExternalTableStage) validate() error { + if !validObjectidentifier(v.name) { + return errInvalidObjectIdentifier + } + return nil +} + +func (cpp *CloudProviderParams) validate() error { + if anyValueSet(cpp.GoogleCloudStorageIntegration, cpp.MicrosoftAzureIntegration) && exactlyOneValueSet(cpp.GoogleCloudStorageIntegration, cpp.MicrosoftAzureIntegration) { + return errOneOf("CloudProviderParams", "GoogleCloudStorageIntegration", "MicrosoftAzureIntegration") + } + return nil +} + +func (opts *ExternalTableFileFormat) validate() error { + var errs []error + if everyValueSet(opts.Name, opts.Type) { + errs = append(errs, errOneOf("ExternalTableFileFormat", "Name", "Type")) + } + fields := externalTableFileFormatTypeOptionsFieldsByType(opts.Options) + for formatType := range fields { + if *opts.Type == formatType { + continue + } + if anyValueSet(fields[formatType]...) { + errs = append(errs, fmt.Errorf("cannot set %s fields when TYPE = %s", formatType, *opts.Type)) + } + } + return errors.Join(errs...) +} + +func (opts *ExternalTableDropOption) validate() error { + if anyValueSet(opts.Restrict, opts.Cascade) && !exactlyOneValueSet(opts.Restrict, opts.Cascade) { + return errOneOf("ExternalTableDropOption", "Restrict", "Cascade") + } + return nil +} + +func externalTableFileFormatTypeOptionsFieldsByType(opts *ExternalTableFileFormatTypeOptions) map[ExternalTableFileFormatType][]any { + return map[ExternalTableFileFormatType][]any{ + ExternalTableFileFormatTypeCSV: { + opts.CSVCompression, + opts.CSVRecordDelimiter, + opts.CSVFieldDelimiter, + opts.CSVSkipHeader, + opts.CSVSkipBlankLines, + opts.CSVEscapeUnenclosedField, + opts.CSVTrimSpace, + opts.CSVFieldOptionallyEnclosedBy, + opts.CSVNullIf, + opts.CSVEmptyFieldAsNull, + opts.CSVEncoding, + }, + ExternalTableFileFormatTypeJSON: { + opts.JSONCompression, + opts.JSONAllowDuplicate, + opts.JSONStripOuterArray, + opts.JSONStripNullValues, + opts.JSONReplaceInvalidCharacters, + }, + ExternalTableFileFormatTypeAvro: { + opts.AvroCompression, + opts.AvroReplaceInvalidCharacters, + }, + ExternalTableFileFormatTypeORC: { + opts.ORCTrimSpace, + opts.ORCReplaceInvalidCharacters, + opts.ORCNullIf, + }, + ExternalTableFileFormatTypeParquet: { + opts.ParquetCompression, + opts.ParquetBinaryAsText, + opts.ParquetReplaceInvalidCharacters, + }, + } +} diff --git a/pkg/sdk/helper_test.go b/pkg/sdk/helper_test.go index e55bdfa8f9..f30c7a27ee 100644 --- a/pkg/sdk/helper_test.go +++ b/pkg/sdk/helper_test.go @@ -670,6 +670,21 @@ func createPipe(t *testing.T, client *Client, database *Database, schema *Schema return createdPipe, pipeCleanup } +func createStageWithName(t *testing.T, client *Client, name string) (*string, func()) { + t.Helper() + ctx := context.Background() + stageCleanup := func() { + _, err := client.exec(ctx, fmt.Sprintf("DROP STAGE %s", name)) + require.NoError(t, err) + } + _, err := client.exec(ctx, fmt.Sprintf("CREATE STAGE %s", name)) + if err != nil { + return nil, stageCleanup + } + require.NoError(t, err) + return &name, stageCleanup +} + func createStage(t *testing.T, client *Client, database *Database, schema *Schema, name string) (*Stage, func()) { t.Helper() require.NotNil(t, database, "database has to be created") @@ -695,3 +710,15 @@ func createStage(t *testing.T, client *Client, database *Database, schema *Schem Name: name, }, stageCleanup } + +func createStageWithURL(t *testing.T, client *Client, name AccountObjectIdentifier, url string) (*Stage, func()) { + t.Helper() + ctx := context.Background() + _, err := client.exec(ctx, fmt.Sprintf(`CREATE STAGE "%s" URL = '%s'`, name.Name(), url)) + require.NoError(t, err) + + return nil, func() { + _, err := client.exec(ctx, fmt.Sprintf(`DROP STAGE "%s"`, name.Name())) + require.NoError(t, err) + } +} diff --git a/pkg/sdk/poc/generator/poc.go b/pkg/sdk/poc/generator/poc.go index ddab1e7744..d51b228a65 100644 --- a/pkg/sdk/poc/generator/poc.go +++ b/pkg/sdk/poc/generator/poc.go @@ -187,12 +187,12 @@ func (v *Validation) Condition(field *Field) string { panic("condition for validation unknown") } -func (v *Validation) Error() string { +func (v *Validation) Error(field *Field) string { switch v.Type { case ValidIdentifier: return fmt.Sprintf("ErrInvalidObjectIdentifier") //nolint case ConflictingFields: - return fmt.Sprintf("errOneOf(%s)", strings.Join(v.paramsQuoted(), ",")) + return fmt.Sprintf("errOneOf(%s, %s)", wrapWith(field.Name, `"`), strings.Join(v.paramsQuoted(), ",")) case ExactlyOneValueSet: return fmt.Sprintf("errExactlyOneOf(%s)", strings.Join(v.paramsQuoted(), ",")) case AtLeastOneValueSet: diff --git a/pkg/sdk/poc/generator/templates.go b/pkg/sdk/poc/generator/templates.go index faf03be567..e1e3828f44 100644 --- a/pkg/sdk/poc/generator/templates.go +++ b/pkg/sdk/poc/generator/templates.go @@ -148,7 +148,7 @@ var ValidationsImplTemplate, _ = template.New("validationsImplTemplate").Parse(` {{$field := .}} {{- range .Validations}} if {{.Condition $field}} { - errs = append(errs, {{.Error}}) + errs = append(errs, {{.Error $field}}) } {{- end}} {{- range .Fields}} diff --git a/pkg/sdk/validations.go b/pkg/sdk/validations.go index a7c3598cfc..e3aa3dce28 100644 --- a/pkg/sdk/validations.go +++ b/pkg/sdk/validations.go @@ -79,7 +79,7 @@ func valueSet(value interface{}) bool { reflectedValue = reflectedValue.Elem() } switch reflectedValue.Kind() { - case reflect.Slice: + case reflect.Slice, reflect.String: return reflectedValue.Len() > 0 case reflect.Invalid: return false diff --git a/pkg/sdk/validations_test.go b/pkg/sdk/validations_test.go index e190c744d9..00646b998b 100644 --- a/pkg/sdk/validations_test.go +++ b/pkg/sdk/validations_test.go @@ -168,6 +168,16 @@ func TestValueSet(t *testing.T) { ok := valueSet(s.ot) assert.Equal(t, ok, false) }) + + t.Run("with invalid empty string", func(t *testing.T) { + invalid := "" + assert.False(t, valueSet(invalid)) + }) + + t.Run("with valid non-empty string", func(t *testing.T) { + valid := "non-empty string" + assert.True(t, valueSet(valid)) + }) } func TestValidateIntInRange(t *testing.T) {