From a02968e78b0b60e7ac0bb3dc9f3c452b38330199 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20Cie=C5=9Blak?= Date: Fri, 1 Dec 2023 10:18:54 +0100 Subject: [PATCH] Use external tables from SDK --- pkg/datasources/external_tables.go | 47 ++-- pkg/resources/external_table.go | 229 ++++++----------- pkg/resources/external_table_test.go | 75 ------ pkg/resources/helpers.go | 43 ++++ pkg/resources/schema.go | 24 +- pkg/sdk/external_tables.go | 8 + pkg/sdk/external_tables_dto.go | 44 +++- pkg/sdk/external_tables_dto_builders_gen.go | 48 +++- pkg/sdk/external_tables_test.go | 154 ++++++++++- pkg/sdk/external_tables_validations.go | 28 +- .../external_tables_integration_test.go | 25 +- .../testint/streams_gen_integration_test.go | 2 +- pkg/snowflake/external_table.go | 242 ------------------ pkg/snowflake/external_table_test.go | 42 --- 14 files changed, 422 insertions(+), 589 deletions(-) delete mode 100644 pkg/resources/external_table_test.go delete mode 100644 pkg/snowflake/external_table.go delete mode 100644 pkg/snowflake/external_table_test.go diff --git a/pkg/datasources/external_tables.go b/pkg/datasources/external_tables.go index 47dac65a27..32b2897462 100644 --- a/pkg/datasources/external_tables.go +++ b/pkg/datasources/external_tables.go @@ -1,12 +1,14 @@ package datasources import ( + "context" "database/sql" - "errors" - "fmt" "log" - "github.com/Snowflake-Labs/terraform-provider-snowflake/pkg/snowflake" + "github.com/Snowflake-Labs/terraform-provider-snowflake/pkg/helpers" + + "github.com/Snowflake-Labs/terraform-provider-snowflake/pkg/sdk" + "github.com/hashicorp/terraform-plugin-sdk/v2/helper/schema" ) @@ -58,34 +60,31 @@ func ExternalTables() *schema.Resource { func ReadExternalTables(d *schema.ResourceData, meta interface{}) error { db := meta.(*sql.DB) + ctx := context.Background() + client := sdk.NewClientFromDB(db) databaseName := d.Get("database").(string) schemaName := d.Get("schema").(string) - currentExternalTables, err := snowflake.ListExternalTables(databaseName, schemaName, db) - if errors.Is(err, sql.ErrNoRows) { - // If not found, mark resource to be removed from state file during apply or refresh - log.Printf("[DEBUG] external tables in schema (%s) not found", d.Id()) - d.SetId("") - return nil - } else if err != nil { - log.Printf("[DEBUG] unable to parse external tables in schema (%s)", d.Id()) + schemaId := sdk.NewDatabaseObjectIdentifier(databaseName, schemaName) + showIn := sdk.NewShowExternalTableInRequest().WithSchema(schemaId) + externalTables, err := client.ExternalTables.Show(ctx, sdk.NewShowExternalTableRequest().WithIn(showIn)) + if err != nil { + log.Printf("[DEBUG] failed when searching external tables in schema (%s), err = %s", schemaId.FullyQualifiedName(), err.Error()) d.SetId("") return nil } - externalTables := []map[string]interface{}{} - - for _, externalTable := range currentExternalTables { - externalTableMap := map[string]interface{}{} - - externalTableMap["name"] = externalTable.ExternalTableName.String - externalTableMap["database"] = externalTable.DatabaseName.String - externalTableMap["schema"] = externalTable.SchemaName.String - externalTableMap["comment"] = externalTable.Comment.String - - externalTables = append(externalTables, externalTableMap) + externalTablesObjects := make([]map[string]any, len(externalTables)) + for i, externalTable := range externalTables { + externalTablesObjects[i] = map[string]any{ + "name": externalTable.Name, + "database": externalTable.DatabaseName, + "schema": externalTable.SchemaName, + "comment": externalTable.Comment, + } } - d.SetId(fmt.Sprintf(`%v|%v`, databaseName, schemaName)) - return d.Set("external_tables", externalTables) + d.SetId(helpers.EncodeSnowflakeID(schemaId)) + + return d.Set("external_tables", externalTablesObjects) } diff --git a/pkg/resources/external_table.go b/pkg/resources/external_table.go index b236a0c4cf..660b5ae34e 100644 --- a/pkg/resources/external_table.go +++ b/pkg/resources/external_table.go @@ -1,21 +1,16 @@ package resources import ( - "bytes" + "context" "database/sql" - "encoding/csv" "fmt" "log" - "strings" - "github.com/Snowflake-Labs/terraform-provider-snowflake/pkg/snowflake" + "github.com/Snowflake-Labs/terraform-provider-snowflake/pkg/helpers" + "github.com/Snowflake-Labs/terraform-provider-snowflake/pkg/sdk" "github.com/hashicorp/terraform-plugin-sdk/v2/helper/schema" ) -const ( - externalTableIDDelimiter = '|' -) - var externalTableSchema = map[string]*schema.Schema{ "name": { Type: schema.TypeString, @@ -50,10 +45,11 @@ var externalTableSchema = map[string]*schema.Schema{ ForceNew: true, }, "type": { - Type: schema.TypeString, - Required: true, - Description: "Column type, e.g. VARIANT", - ForceNew: true, + Type: schema.TypeString, + Required: true, + Description: "Column type, e.g. VARIANT", + ForceNew: true, + ValidateFunc: IsDataType(), }, "as": { Type: schema.TypeString, @@ -144,204 +140,137 @@ func ExternalTable() *schema.Resource { } } -type externalTableID struct { - DatabaseName string - SchemaName string - ExternalTableName string -} - -// String() takes in a externalTableID object and returns a pipe-delimited string: -// DatabaseName|SchemaName|ExternalTableName. -func (si *externalTableID) String() (string, error) { - var buf bytes.Buffer - csvWriter := csv.NewWriter(&buf) - csvWriter.Comma = externalTableIDDelimiter - dataIdentifiers := [][]string{{si.DatabaseName, si.SchemaName, si.ExternalTableName}} - if err := csvWriter.WriteAll(dataIdentifiers); err != nil { - return "", err - } - strExternalTableID := strings.TrimSpace(buf.String()) - return strExternalTableID, nil -} - -// externalTableIDFromString() takes in a pipe-delimited string: DatabaseName|SchemaName|ExternalTableName -// and returns a externalTableID object. -func externalTableIDFromString(stringID string) (*externalTableID, error) { - reader := csv.NewReader(strings.NewReader(stringID)) - reader.Comma = externalTableIDDelimiter - lines, err := reader.ReadAll() - if err != nil { - return nil, fmt.Errorf("not CSV compatible") - } - - if len(lines) != 1 { - return nil, fmt.Errorf("1 line at a time") - } - if len(lines[0]) != 3 { - return nil, fmt.Errorf("3 fields allowed") - } - - externalTableResult := &externalTableID{ - DatabaseName: lines[0][0], - SchemaName: lines[0][1], - ExternalTableName: lines[0][2], - } - return externalTableResult, nil -} - // CreateExternalTable implements schema.CreateFunc. -func CreateExternalTable(d *schema.ResourceData, meta interface{}) error { +func CreateExternalTable(d *schema.ResourceData, meta any) error { db := meta.(*sql.DB) + ctx := context.Background() + client := sdk.NewClientFromDB(db) + database := d.Get("database").(string) - dbSchema := d.Get("schema").(string) + schema := d.Get("schema").(string) name := d.Get("name").(string) - - // This type conversion is due to the test framework in the terraform-plugin-sdk having limited support - // for data types in the HCL2ValueFromConfigValue method. - columns := []map[string]string{} - for _, column := range d.Get("column").([]interface{}) { + id := sdk.NewSchemaObjectIdentifier(database, schema, name) + location := d.Get("location").(string) + fileFormat := d.Get("file_format").(string) + req := sdk.NewCreateExternalTableRequest(id, location).WithRawFileFormat(&fileFormat) + + tableColumns := d.Get("column").([]any) + columnRequests := make([]*sdk.ExternalTableColumnRequest, len(tableColumns)) + for i, col := range tableColumns { columnDef := map[string]string{} - for key, val := range column.(map[string]interface{}) { + for key, val := range col.(map[string]any) { columnDef[key] = val.(string) } - columns = append(columns, columnDef) + + name := columnDef["name"] + dataTypeString := columnDef["type"] + dataType, err := sdk.ToDataType(dataTypeString) + if err != nil { + return fmt.Errorf(`failed to parse datatype: %s`, dataTypeString) + } + as := columnDef["as"] + columnRequests[i] = sdk.NewExternalTableColumnRequest(name, dataType, as) } - builder := snowflake.NewExternalTableBuilder(name, database, dbSchema) - builder.WithColumns(columns) - builder.WithFileFormat(d.Get("file_format").(string)) - builder.WithLocation(d.Get("location").(string)) + req.WithColumns(columnRequests) - builder.WithAutoRefresh(d.Get("auto_refresh").(bool)) - builder.WithRefreshOnCreate(d.Get("refresh_on_create").(bool)) - builder.WithCopyGrants(d.Get("copy_grants").(bool)) + req.WithAutoRefresh(sdk.Bool(d.Get("auto_refresh").(bool))) + req.WithRefreshOnCreate(sdk.Bool(d.Get("refresh_on_create").(bool))) + req.WithCopyGrants(sdk.Bool(d.Get("copy_grants").(bool))) - // Set optionals if v, ok := d.GetOk("partition_by"); ok { - partitionBys := expandStringList(v.([]interface{})) - builder.WithPartitionBys(partitionBys) + req.WithPartitionBy(v.([]string)) } if v, ok := d.GetOk("pattern"); ok { - builder.WithPattern(v.(string)) + req.WithPattern(sdk.String(v.(string))) } if v, ok := d.GetOk("aws_sns_topic"); ok { - builder.WithAwsSNSTopic(v.(string)) + req.WithAwsSnsTopic(sdk.String(v.(string))) } if v, ok := d.GetOk("comment"); ok { - builder.WithComment(v.(string)) - } - - if v, ok := d.GetOk("tag"); ok { - tags := getTags(v) - builder.WithTags(tags.toSnowflakeTagValues()) + req.WithComment(sdk.String(v.(string))) } - stmt := builder.Create() - if err := snowflake.Exec(db, stmt); err != nil { - return fmt.Errorf("error creating externalTable %v err = %w", name, err) + if _, ok := d.GetOk("tag"); ok { + tagAssociations := getPropertyTags(d, "tag") + tagAssociationRequests := make([]*sdk.TagAssociationRequest, len(tagAssociations)) + for i, t := range tagAssociations { + tagAssociationRequests[i] = sdk.NewTagAssociationRequest(t.Name, t.Value) + } + req.WithTag(tagAssociationRequests) } - externalTableID := &externalTableID{ - DatabaseName: database, - SchemaName: dbSchema, - ExternalTableName: name, - } - dataIDInput, err := externalTableID.String() - if err != nil { + if err := client.ExternalTables.Create(ctx, req); err != nil { return err } - d.SetId(dataIDInput) + d.SetId(helpers.EncodeSnowflakeID(id)) return ReadExternalTable(d, meta) } // ReadExternalTable implements schema.ReadFunc. -func ReadExternalTable(d *schema.ResourceData, meta interface{}) error { +func ReadExternalTable(d *schema.ResourceData, meta any) error { db := meta.(*sql.DB) - externalTableID, err := externalTableIDFromString(d.Id()) - if err != nil { - return err - } - - dbName := externalTableID.DatabaseName - schema := externalTableID.SchemaName - name := externalTableID.ExternalTableName + ctx := context.Background() + client := sdk.NewClientFromDB(db) + id := helpers.DecodeSnowflakeID(d.Id()).(sdk.SchemaObjectIdentifier) - stmt := snowflake.NewExternalTableBuilder(name, dbName, schema).Show() - row := snowflake.QueryRow(db, stmt) - externalTable, err := snowflake.ScanExternalTable(row) + externalTable, err := client.ExternalTables.ShowByID(ctx, sdk.NewShowExternalTableByIDRequest(id)) if err != nil { - if err.Error() == snowflake.ErrNoRowInRS { - log.Printf("[DEBUG] external table (%s) not found", d.Id()) - d.SetId("") - return nil - } + log.Printf("[DEBUG] external table (%s) not found", d.Id()) + d.SetId("") return err } - if err := d.Set("name", externalTable.ExternalTableName.String); err != nil { + if err := d.Set("owner", externalTable.Owner); err != nil { return err } - if err := d.Set("owner", externalTable.Owner.String); err != nil { - return err - } return nil } // UpdateExternalTable implements schema.UpdateFunc. -func UpdateExternalTable(d *schema.ResourceData, meta interface{}) error { +func UpdateExternalTable(d *schema.ResourceData, meta any) error { db := meta.(*sql.DB) - database := d.Get("database").(string) - dbSchema := d.Get("schema").(string) - name := d.Get("name").(string) - - builder := snowflake.NewExternalTableBuilder(name, database, dbSchema) + ctx := context.Background() + client := sdk.NewClientFromDB(db) + id := helpers.DecodeSnowflakeID(d.Id()).(sdk.SchemaObjectIdentifier) if d.HasChange("tag") { - v := d.Get("tag") - tags := getTags(v) - builder.WithTags(tags.toSnowflakeTagValues()) - } + unsetTags, setTags := getTagsDiff(d, "tag") - stmt := builder.Update() - if err := snowflake.Exec(db, stmt); err != nil { - return fmt.Errorf("error updating externalTable %v err = %w", name, err) - } + err := client.ExternalTables.Alter(ctx, sdk.NewAlterExternalTableRequest(id).WithUnsetTag(unsetTags)) + if err != nil { + return fmt.Errorf("error setting tags on %v, err = %w", d.Id(), err) + } - externalTableID := &externalTableID{ - DatabaseName: database, - SchemaName: dbSchema, - ExternalTableName: name, - } - dataIDInput, err := externalTableID.String() - if err != nil { - return err + tagAssociationRequests := make([]*sdk.TagAssociationRequest, len(setTags)) + for i, t := range setTags { + tagAssociationRequests[i] = sdk.NewTagAssociationRequest(t.Name, t.Value) + } + err = client.ExternalTables.Alter(ctx, sdk.NewAlterExternalTableRequest(id).WithSetTag(tagAssociationRequests)) + if err != nil { + return fmt.Errorf("error setting tags on %v, err = %w", d.Id(), err) + } } - d.SetId(dataIDInput) return ReadExternalTable(d, meta) } // DeleteExternalTable implements schema.DeleteFunc. -func DeleteExternalTable(d *schema.ResourceData, meta interface{}) error { +func DeleteExternalTable(d *schema.ResourceData, meta any) error { db := meta.(*sql.DB) - externalTableID, err := externalTableIDFromString(d.Id()) + ctx := context.Background() + client := sdk.NewClientFromDB(db) + id := helpers.DecodeSnowflakeID(d.Id()).(sdk.SchemaObjectIdentifier) + + err := client.ExternalTables.Drop(ctx, sdk.NewDropExternalTableRequest(id)) if err != nil { return err } - dbName := externalTableID.DatabaseName - schema := externalTableID.SchemaName - externalTableName := externalTableID.ExternalTableName - - q := snowflake.NewExternalTableBuilder(externalTableName, dbName, schema).Drop() - if err := snowflake.Exec(db, q); err != nil { - return fmt.Errorf("error deleting pipe %v err = %w", d.Id(), err) - } - d.SetId("") return nil diff --git a/pkg/resources/external_table_test.go b/pkg/resources/external_table_test.go deleted file mode 100644 index 0cad818fc6..0000000000 --- a/pkg/resources/external_table_test.go +++ /dev/null @@ -1,75 +0,0 @@ -package resources_test - -import ( - "database/sql" - "testing" - - sqlmock "github.com/DATA-DOG/go-sqlmock" - "github.com/Snowflake-Labs/terraform-provider-snowflake/pkg/provider" - "github.com/Snowflake-Labs/terraform-provider-snowflake/pkg/resources" - . "github.com/Snowflake-Labs/terraform-provider-snowflake/pkg/testhelpers" - "github.com/stretchr/testify/require" -) - -func TestExternalTable(t *testing.T) { - r := require.New(t) - err := resources.ExternalTable().InternalValidate(provider.Provider().Schema, true) - r.NoError(err) -} - -func TestExternalTableCreate(t *testing.T) { - r := require.New(t) - - in := map[string]interface{}{ - "name": "good_name", - "database": "database_name", - "schema": "schema_name", - "comment": "great comment", - "column": []interface{}{map[string]interface{}{"name": "column1", "type": "OBJECT", "as": "a"}, map[string]interface{}{"name": "column2", "type": "VARCHAR", "as": "b"}}, - "location": "location", - "file_format": "FORMAT_NAME = 'format'", - "pattern": "pattern", - } - d := externalTable(t, "database_name|schema_name|good_name", in) - - WithMockDb(t, func(db *sql.DB, mock sqlmock.Sqlmock) { - mock.ExpectExec(`CREATE EXTERNAL TABLE "database_name"."schema_name"."good_name" \("column1" OBJECT AS a, "column2" VARCHAR AS b\) WITH LOCATION = location REFRESH_ON_CREATE = true AUTO_REFRESH = true PATTERN = 'pattern' FILE_FORMAT = \( FORMAT_NAME = 'format' \) COMMENT = 'great comment'`).WillReturnResult(sqlmock.NewResult(1, 1)) - - expectExternalTableRead(mock) - err := resources.CreateExternalTable(d, db) - r.NoError(err) - r.Equal("good_name", d.Get("name").(string)) - }) -} - -func expectExternalTableRead(mock sqlmock.Sqlmock) { - rows := sqlmock.NewRows([]string{"name", "type", "kind", "null?", "default", "primary key", "unique key", "check", "expression", "comment"}).AddRow("good_name", "VARCHAR()", "COLUMN", "Y", "NULL", "NULL", "N", "N", "NULL", "mock comment") - mock.ExpectQuery(`SHOW EXTERNAL TABLES LIKE 'good_name' IN SCHEMA "database_name"."schema_name"`).WillReturnRows(rows) -} - -func TestExternalTableRead(t *testing.T) { - r := require.New(t) - - d := externalTable(t, "database_name|schema_name|good_name", map[string]interface{}{"name": "good_name", "comment": "mock comment"}) - - WithMockDb(t, func(db *sql.DB, mock sqlmock.Sqlmock) { - expectExternalTableRead(mock) - - err := resources.ReadExternalTable(d, db) - r.NoError(err) - r.Equal("good_name", d.Get("name").(string)) - r.Equal("mock comment", d.Get("comment").(string)) - }) -} - -func TestExternalTableDelete(t *testing.T) { - r := require.New(t) - - d := externalTable(t, "database_name|schema_name|drop_it", map[string]interface{}{"name": "drop_it"}) - - WithMockDb(t, func(db *sql.DB, mock sqlmock.Sqlmock) { - mock.ExpectExec(`DROP EXTERNAL TABLE "database_name"."schema_name"."drop_it"`).WillReturnResult(sqlmock.NewResult(1, 1)) - err := resources.DeleteExternalTable(d, db) - r.NoError(err) - }) -} diff --git a/pkg/resources/helpers.go b/pkg/resources/helpers.go index 9691ae7106..df6ce63ebe 100644 --- a/pkg/resources/helpers.go +++ b/pkg/resources/helpers.go @@ -88,6 +88,32 @@ func getPropertyTags(d *schema.ResourceData, key string) []sdk.TagAssociation { return nil } +func getTagsDiff(d *schema.ResourceData, key string) (unsetTags []sdk.ObjectIdentifier, setTags []sdk.TagAssociation) { + o, n := d.GetChange(key) + removed, added, changed := getTags(o).diffs(getTags(n)) + + unsetTags = make([]sdk.ObjectIdentifier, len(removed)) + for i, t := range removed { + unsetTags[i] = sdk.NewDatabaseObjectIdentifier(t.database, t.name) + } + + setTags = make([]sdk.TagAssociation, len(added)+len(changed)) + for i, t := range added { + setTags[i] = sdk.TagAssociation{ + Name: sdk.NewSchemaObjectIdentifier(t.database, t.schema, t.name), + Value: t.value, + } + } + for i, t := range changed { + setTags[len(added)+i] = sdk.TagAssociation{ + Name: sdk.NewSchemaObjectIdentifier(t.database, t.schema, t.name), + Value: t.value, + } + } + + return unsetTags, setTags +} + func GetPropertyAsPointer[T any](d *schema.ResourceData, property string) *T { value, ok := d.GetOk(property) if !ok { @@ -99,3 +125,20 @@ func GetPropertyAsPointer[T any](d *schema.ResourceData, property string) *T { } return &typedValue } + +func IsDataType() schema.SchemaValidateFunc { + return func(value any, key string) (warnings []string, errors []error) { + stringValue, ok := value.(string) + if !ok { + errors = append(errors, fmt.Errorf("expected type of %s to be string, got %T", key, value)) + return warnings, errors + } + + _, err := sdk.ToDataType(stringValue) + if err != nil { + errors = append(errors, fmt.Errorf("expected %s to be one of %T values, got %s", key, sdk.DataTypeString, stringValue)) + } + + return warnings, errors + } +} diff --git a/pkg/resources/schema.go b/pkg/resources/schema.go index 83b07d31ee..0eb6958eaf 100644 --- a/pkg/resources/schema.go +++ b/pkg/resources/schema.go @@ -229,38 +229,20 @@ func UpdateSchema(d *schema.ResourceData, meta interface{}) error { } if d.HasChange("tag") { - o, n := d.GetChange("tag") - removed, added, changed := getTags(o).diffs(getTags(n)) + unsetTags, setTags := getTagsDiff(d, "tag") - unsetTags := make([]sdk.ObjectIdentifier, len(removed)) - for i, t := range removed { - unsetTags[i] = sdk.NewDatabaseObjectIdentifier(t.database, t.name) - } err := client.Schemas.Alter(ctx, id, &sdk.AlterSchemaOptions{ UnsetTag: unsetTags, }) if err != nil { - return fmt.Errorf("error dropping tags on %v", d.Id()) + return fmt.Errorf("error occurred when dropping tags on %v, err = %w", d.Id(), err) } - setTags := make([]sdk.TagAssociation, len(added)+len(changed)) - for i, t := range added { - setTags[i] = sdk.TagAssociation{ - Name: sdk.NewSchemaObjectIdentifier(t.database, t.schema, t.name), - Value: t.value, - } - } - for i, t := range changed { - setTags[i] = sdk.TagAssociation{ - Name: sdk.NewSchemaObjectIdentifier(t.database, t.schema, t.name), - Value: t.value, - } - } err = client.Schemas.Alter(ctx, id, &sdk.AlterSchemaOptions{ SetTag: setTags, }) if err != nil { - return fmt.Errorf("error setting tags on %v", d.Id()) + return fmt.Errorf("error occurred when setting tags on %v, err = %w", d.Id(), err) } } diff --git a/pkg/sdk/external_tables.go b/pkg/sdk/external_tables.go index eea89fc9bd..87e057a199 100644 --- a/pkg/sdk/external_tables.go +++ b/pkg/sdk/external_tables.go @@ -56,6 +56,10 @@ func (v *ExternalTable) ObjectType() ObjectType { return ObjectTypeExternalTable } +type RawFileFormat struct { + Format string `ddl:"keyword"` +} + // CreateExternalTableOptions based on https://docs.snowflake.com/en/sql-reference/sql/create-external-table type CreateExternalTableOptions struct { create bool `ddl:"static" sql:"CREATE"` @@ -71,6 +75,7 @@ type CreateExternalTableOptions struct { AutoRefresh *bool `ddl:"parameter" sql:"AUTO_REFRESH"` Pattern *string `ddl:"parameter,single_quotes" sql:"PATTERN"` FileFormat []ExternalTableFileFormat `ddl:"parameter,parentheses" sql:"FILE_FORMAT"` + RawFileFormat *RawFileFormat `ddl:"list,parentheses" sql:"FILE_FORMAT ="` AwsSnsTopic *string `ddl:"parameter,single_quotes" sql:"AWS_SNS_TOPIC"` CopyGrants *bool `ddl:"keyword" sql:"COPY GRANTS"` Comment *string `ddl:"parameter,single_quotes" sql:"COMMENT"` @@ -203,6 +208,7 @@ type CreateWithManualPartitioningExternalTableOptions struct { Location string `ddl:"parameter" sql:"LOCATION"` UserSpecifiedPartitionType *bool `ddl:"keyword" sql:"PARTITION_TYPE = USER_SPECIFIED"` FileFormat []ExternalTableFileFormat `ddl:"parameter,parentheses" sql:"FILE_FORMAT"` + RawFileFormat *RawFileFormat `ddl:"list,parentheses" sql:"FILE_FORMAT ="` CopyGrants *bool `ddl:"keyword" sql:"COPY GRANTS"` Comment *string `ddl:"parameter,single_quotes" sql:"COMMENT"` RowAccessPolicy *RowAccessPolicy `ddl:"keyword"` @@ -224,6 +230,7 @@ type CreateDeltaLakeExternalTableOptions struct { AutoRefresh *bool `ddl:"parameter" sql:"AUTO_REFRESH"` UserSpecifiedPartitionType *bool `ddl:"keyword" sql:"PARTITION_TYPE = USER_SPECIFIED"` FileFormat []ExternalTableFileFormat `ddl:"parameter,parentheses" sql:"FILE_FORMAT"` + RawFileFormat *RawFileFormat `ddl:"list,parentheses" sql:"FILE_FORMAT ="` DeltaTableFormat *bool `ddl:"keyword" sql:"TABLE_FORMAT = DELTA"` CopyGrants *bool `ddl:"keyword" sql:"COPY GRANTS"` Comment *string `ddl:"parameter,single_quotes" sql:"COMMENT"` @@ -246,6 +253,7 @@ type CreateExternalTableUsingTemplateOptions struct { AutoRefresh *bool `ddl:"parameter" sql:"AUTO_REFRESH"` Pattern *string `ddl:"parameter,single_quotes" sql:"PATTERN"` FileFormat []ExternalTableFileFormat `ddl:"parameter,parentheses" sql:"FILE_FORMAT"` + RawFileFormat *RawFileFormat `ddl:"list,parentheses" sql:"FILE_FORMAT ="` AwsSnsTopic *string `ddl:"parameter,single_quotes" sql:"AWS_SNS_TOPIC"` Comment *string `ddl:"parameter,single_quotes" sql:"COMMENT"` RowAccessPolicy *RowAccessPolicy `ddl:"keyword"` diff --git a/pkg/sdk/external_tables_dto.go b/pkg/sdk/external_tables_dto.go index 3bcc01103f..44e3a95c1a 100644 --- a/pkg/sdk/external_tables_dto.go +++ b/pkg/sdk/external_tables_dto.go @@ -26,7 +26,8 @@ type CreateExternalTableRequest struct { refreshOnCreate *bool autoRefresh *bool pattern *string - fileFormat *ExternalTableFileFormatRequest // required + rawFileFormat *string + fileFormat *ExternalTableFileFormatRequest awsSnsTopic *string copyGrants *bool comment *string @@ -267,6 +268,13 @@ func (s *CreateExternalTableRequest) toOpts() *CreateExternalTableOptions { fileFormat = []ExternalTableFileFormat{s.fileFormat.toOpts()} } + var rawFileFormat *RawFileFormat + if s.rawFileFormat != nil { + rawFileFormat = &RawFileFormat{ + Format: *s.rawFileFormat, + } + } + var cloudProviderParams *CloudProviderParams if s.cloudProviderParams != nil { cloudProviderParams = s.cloudProviderParams.toOpts() @@ -294,6 +302,7 @@ func (s *CreateExternalTableRequest) toOpts() *CreateExternalTableOptions { RefreshOnCreate: s.refreshOnCreate, AutoRefresh: s.autoRefresh, Pattern: s.pattern, + RawFileFormat: rawFileFormat, FileFormat: fileFormat, AwsSnsTopic: s.awsSnsTopic, CopyGrants: s.copyGrants, @@ -312,7 +321,8 @@ type CreateWithManualPartitioningExternalTableRequest struct { partitionBy []string location string // required userSpecifiedPartitionType *bool - fileFormat *ExternalTableFileFormatRequest // required + rawFileFormat *string + fileFormat *ExternalTableFileFormatRequest copyGrants *bool comment *string rowAccessPolicy *RowAccessPolicyRequest @@ -337,6 +347,13 @@ func (v *CreateWithManualPartitioningExternalTableRequest) toOpts() *CreateWithM fileFormat = []ExternalTableFileFormat{v.fileFormat.toOpts()} } + var rawFileFormat *RawFileFormat + if v.rawFileFormat != nil { + rawFileFormat = &RawFileFormat{ + Format: *v.rawFileFormat, + } + } + var rowAccessPolicy *RowAccessPolicy if v.rowAccessPolicy != nil { rowAccessPolicy = v.rowAccessPolicy.toOpts() @@ -358,6 +375,7 @@ func (v *CreateWithManualPartitioningExternalTableRequest) toOpts() *CreateWithM PartitionBy: v.partitionBy, Location: v.location, UserSpecifiedPartitionType: v.userSpecifiedPartitionType, + RawFileFormat: rawFileFormat, FileFormat: fileFormat, CopyGrants: v.copyGrants, Comment: v.comment, @@ -377,7 +395,8 @@ type CreateDeltaLakeExternalTableRequest struct { userSpecifiedPartitionType *bool refreshOnCreate *bool autoRefresh *bool - fileFormat *ExternalTableFileFormatRequest // required + rawFileFormat *string + fileFormat *ExternalTableFileFormatRequest deltaTableFormat *bool copyGrants *bool comment *string @@ -403,6 +422,13 @@ func (v *CreateDeltaLakeExternalTableRequest) toOpts() *CreateDeltaLakeExternalT fileFormat = []ExternalTableFileFormat{v.fileFormat.toOpts()} } + var rawFileFormat *RawFileFormat + if v.rawFileFormat != nil { + rawFileFormat = &RawFileFormat{ + Format: *v.rawFileFormat, + } + } + var rowAccessPolicy *RowAccessPolicy if v.rowAccessPolicy != nil { rowAccessPolicy = v.rowAccessPolicy.toOpts() @@ -426,6 +452,7 @@ func (v *CreateDeltaLakeExternalTableRequest) toOpts() *CreateDeltaLakeExternalT UserSpecifiedPartitionType: v.userSpecifiedPartitionType, RefreshOnCreate: v.refreshOnCreate, AutoRefresh: v.autoRefresh, + RawFileFormat: rawFileFormat, FileFormat: fileFormat, DeltaTableFormat: v.deltaTableFormat, CopyGrants: v.copyGrants, @@ -446,7 +473,8 @@ type CreateExternalTableUsingTemplateRequest struct { refreshOnCreate *bool autoRefresh *bool pattern *string - fileFormat *ExternalTableFileFormatRequest // required + rawFileFormat *string + fileFormat *ExternalTableFileFormatRequest awsSnsTopic *string comment *string rowAccessPolicy *RowAccessPolicyRequest @@ -464,6 +492,13 @@ func (v *CreateExternalTableUsingTemplateRequest) toOpts() *CreateExternalTableU fileFormat = []ExternalTableFileFormat{v.fileFormat.toOpts()} } + var rawFileFormat *RawFileFormat + if v.rawFileFormat != nil { + rawFileFormat = &RawFileFormat{ + Format: *v.rawFileFormat, + } + } + var rowAccessPolicy *RowAccessPolicy if v.rowAccessPolicy != nil { rowAccessPolicy = v.rowAccessPolicy.toOpts() @@ -487,6 +522,7 @@ func (v *CreateExternalTableUsingTemplateRequest) toOpts() *CreateExternalTableU RefreshOnCreate: v.refreshOnCreate, AutoRefresh: v.autoRefresh, Pattern: v.pattern, + RawFileFormat: rawFileFormat, FileFormat: fileFormat, AwsSnsTopic: v.awsSnsTopic, Comment: v.comment, diff --git a/pkg/sdk/external_tables_dto_builders_gen.go b/pkg/sdk/external_tables_dto_builders_gen.go index cad2cf1097..a85c940272 100644 --- a/pkg/sdk/external_tables_dto_builders_gen.go +++ b/pkg/sdk/external_tables_dto_builders_gen.go @@ -5,12 +5,10 @@ package sdk func NewCreateExternalTableRequest( name SchemaObjectIdentifier, location string, - fileFormat *ExternalTableFileFormatRequest, ) *CreateExternalTableRequest { s := CreateExternalTableRequest{} s.name = name s.location = location - s.fileFormat = fileFormat return &s } @@ -54,6 +52,16 @@ func (s *CreateExternalTableRequest) WithPattern(pattern *string) *CreateExterna return s } +func (s *CreateExternalTableRequest) WithRawFileFormat(rawFileFormat *string) *CreateExternalTableRequest { + s.rawFileFormat = rawFileFormat + return s +} + +func (s *CreateExternalTableRequest) WithFileFormat(fileFormat *ExternalTableFileFormatRequest) *CreateExternalTableRequest { + s.fileFormat = fileFormat + return s +} + func (s *CreateExternalTableRequest) WithAwsSnsTopic(awsSnsTopic *string) *CreateExternalTableRequest { s.awsSnsTopic = awsSnsTopic return s @@ -402,12 +410,10 @@ func NewTagAssociationRequest( func NewCreateWithManualPartitioningExternalTableRequest( name SchemaObjectIdentifier, location string, - fileFormat *ExternalTableFileFormatRequest, ) *CreateWithManualPartitioningExternalTableRequest { s := CreateWithManualPartitioningExternalTableRequest{} s.name = name s.location = location - s.fileFormat = fileFormat return &s } @@ -441,6 +447,16 @@ func (s *CreateWithManualPartitioningExternalTableRequest) WithUserSpecifiedPart return s } +func (s *CreateWithManualPartitioningExternalTableRequest) WithRawFileFormat(rawFileFormat *string) *CreateWithManualPartitioningExternalTableRequest { + s.rawFileFormat = rawFileFormat + return s +} + +func (s *CreateWithManualPartitioningExternalTableRequest) WithFileFormat(fileFormat *ExternalTableFileFormatRequest) *CreateWithManualPartitioningExternalTableRequest { + s.fileFormat = fileFormat + return s +} + func (s *CreateWithManualPartitioningExternalTableRequest) WithCopyGrants(copyGrants *bool) *CreateWithManualPartitioningExternalTableRequest { s.copyGrants = copyGrants return s @@ -464,12 +480,10 @@ func (s *CreateWithManualPartitioningExternalTableRequest) WithTag(tag []*TagAss func NewCreateDeltaLakeExternalTableRequest( name SchemaObjectIdentifier, location string, - fileFormat *ExternalTableFileFormatRequest, ) *CreateDeltaLakeExternalTableRequest { s := CreateDeltaLakeExternalTableRequest{} s.name = name s.location = location - s.fileFormat = fileFormat return &s } @@ -513,6 +527,16 @@ func (s *CreateDeltaLakeExternalTableRequest) WithAutoRefresh(autoRefresh *bool) return s } +func (s *CreateDeltaLakeExternalTableRequest) WithRawFileFormat(rawFileFormat *string) *CreateDeltaLakeExternalTableRequest { + s.rawFileFormat = rawFileFormat + return s +} + +func (s *CreateDeltaLakeExternalTableRequest) WithFileFormat(fileFormat *ExternalTableFileFormatRequest) *CreateDeltaLakeExternalTableRequest { + s.fileFormat = fileFormat + return s +} + func (s *CreateDeltaLakeExternalTableRequest) WithDeltaTableFormat(deltaTableFormat *bool) *CreateDeltaLakeExternalTableRequest { s.deltaTableFormat = deltaTableFormat return s @@ -541,12 +565,10 @@ func (s *CreateDeltaLakeExternalTableRequest) WithTag(tag []*TagAssociationReque func NewCreateExternalTableUsingTemplateRequest( name SchemaObjectIdentifier, location string, - fileFormat *ExternalTableFileFormatRequest, ) *CreateExternalTableUsingTemplateRequest { s := CreateExternalTableUsingTemplateRequest{} s.name = name s.location = location - s.fileFormat = fileFormat return &s } @@ -590,6 +612,16 @@ func (s *CreateExternalTableUsingTemplateRequest) WithPattern(pattern *string) * return s } +func (s *CreateExternalTableUsingTemplateRequest) WithRawFileFormat(rawFileFormat *string) *CreateExternalTableUsingTemplateRequest { + s.rawFileFormat = rawFileFormat + return s +} + +func (s *CreateExternalTableUsingTemplateRequest) WithFileFormat(fileFormat *ExternalTableFileFormatRequest) *CreateExternalTableUsingTemplateRequest { + s.fileFormat = fileFormat + return s +} + func (s *CreateExternalTableUsingTemplateRequest) WithAwsSnsTopic(awsSnsTopic *string) *CreateExternalTableUsingTemplateRequest { s.awsSnsTopic = awsSnsTopic return s diff --git a/pkg/sdk/external_tables_test.go b/pkg/sdk/external_tables_test.go index e6ed3fb29a..0955f9f89d 100644 --- a/pkg/sdk/external_tables_test.go +++ b/pkg/sdk/external_tables_test.go @@ -91,9 +91,50 @@ func TestExternalTablesCreate(t *testing.T) { errOneOf("CreateExternalTableOptions", "OrReplace", "IfNotExists"), ErrInvalidObjectIdentifier, errNotSet("CreateExternalTableOptions", "Location"), - errNotSet("CreateExternalTableOptions", "FileFormat"), + errExactlyOneOf("CreateExternalTableOptions", "RawFileFormat", "FileFormat"), ) }) + + t.Run("raw file format", func(t *testing.T) { + opts := &CreateExternalTableOptions{ + name: NewSchemaObjectIdentifier("db", "schema", "external_table"), + Columns: []ExternalTableColumn{ + { + Name: "column", + Type: "varchar", + AsExpression: []string{"value::column::varchar"}, + InlineConstraint: &ColumnInlineConstraint{ + Name: String("my_constraint"), + NotNull: Bool(true), + Type: &ColumnConstraintTypeUnique, + }, + }, + }, + Location: "@s1/logs/", + RawFileFormat: &RawFileFormat{Format: "TYPE = JSON"}, + } + assertOptsValidAndSQLEquals(t, opts, `CREATE EXTERNAL TABLE "db"."schema"."external_table" (column varchar AS (value::column::varchar) NOT NULL CONSTRAINT my_constraint UNIQUE) LOCATION = @s1/logs/ FILE_FORMAT = (TYPE = JSON)`) + }) + + t.Run("validation: neither raw file format is set, nor file format", func(t *testing.T) { + opts := &CreateExternalTableOptions{ + name: NewSchemaObjectIdentifier("db", "schema", "external_table"), + Columns: []ExternalTableColumn{ + { + Name: "column", + Type: "varchar", + AsExpression: []string{"value::column::varchar"}, + InlineConstraint: &ColumnInlineConstraint{ + Name: String("my_constraint"), + NotNull: Bool(true), + Type: &ColumnConstraintTypeUnique, + }, + }, + }, + Location: "@s1/logs/", + } + assertOptsInvalid(t, opts, errExactlyOneOf("CreateExternalTableOptions", "RawFileFormat", "FileFormat")) + }) } func TestExternalTablesCreateWithManualPartitioning(t *testing.T) { @@ -153,9 +194,50 @@ func TestExternalTablesCreateWithManualPartitioning(t *testing.T) { errOneOf("CreateWithManualPartitioningExternalTableOptions", "OrReplace", "IfNotExists"), ErrInvalidObjectIdentifier, errNotSet("CreateWithManualPartitioningExternalTableOptions", "Location"), - errNotSet("CreateWithManualPartitioningExternalTableOptions", "FileFormat"), + errExactlyOneOf("CreateWithManualPartitioningExternalTableOptions", "RawFileFormat", "FileFormat"), ) }) + + t.Run("raw file format", func(t *testing.T) { + opts := &CreateWithManualPartitioningExternalTableOptions{ + name: NewSchemaObjectIdentifier("db", "schema", "external_table"), + Columns: []ExternalTableColumn{ + { + Name: "column", + Type: "varchar", + AsExpression: []string{"value::column::varchar"}, + InlineConstraint: &ColumnInlineConstraint{ + Name: String("my_constraint"), + NotNull: Bool(true), + Type: &ColumnConstraintTypeUnique, + }, + }, + }, + Location: "@s1/logs/", + RawFileFormat: &RawFileFormat{Format: "TYPE = JSON"}, + } + assertOptsValidAndSQLEquals(t, opts, `CREATE EXTERNAL TABLE "db"."schema"."external_table" (column varchar AS (value::column::varchar) NOT NULL CONSTRAINT my_constraint UNIQUE) LOCATION = @s1/logs/ FILE_FORMAT = (TYPE = JSON)`) + }) + + t.Run("validation: neither raw file format is set, nor file format", func(t *testing.T) { + opts := &CreateWithManualPartitioningExternalTableOptions{ + name: NewSchemaObjectIdentifier("db", "schema", "external_table"), + Columns: []ExternalTableColumn{ + { + Name: "column", + Type: "varchar", + AsExpression: []string{"value::column::varchar"}, + InlineConstraint: &ColumnInlineConstraint{ + Name: String("my_constraint"), + NotNull: Bool(true), + Type: &ColumnConstraintTypeUnique, + }, + }, + }, + Location: "@s1/logs/", + } + assertOptsInvalid(t, opts, errExactlyOneOf("CreateWithManualPartitioningExternalTableOptions", "RawFileFormat", "FileFormat")) + }) } func TestExternalTablesCreateDeltaLake(t *testing.T) { @@ -213,9 +295,50 @@ func TestExternalTablesCreateDeltaLake(t *testing.T) { errOneOf("CreateDeltaLakeExternalTableOptions", "OrReplace", "IfNotExists"), ErrInvalidObjectIdentifier, errNotSet("CreateDeltaLakeExternalTableOptions", "Location"), - errNotSet("CreateDeltaLakeExternalTableOptions", "FileFormat"), + errExactlyOneOf("CreateDeltaLakeExternalTableOptions", "RawFileFormat", "FileFormat"), ) }) + + t.Run("raw file format", func(t *testing.T) { + opts := &CreateDeltaLakeExternalTableOptions{ + name: NewSchemaObjectIdentifier("db", "schema", "external_table"), + Columns: []ExternalTableColumn{ + { + Name: "column", + Type: "varchar", + AsExpression: []string{"value::column::varchar"}, + InlineConstraint: &ColumnInlineConstraint{ + Name: String("my_constraint"), + NotNull: Bool(true), + Type: &ColumnConstraintTypeUnique, + }, + }, + }, + Location: "@s1/logs/", + RawFileFormat: &RawFileFormat{Format: "TYPE = JSON"}, + } + assertOptsValidAndSQLEquals(t, opts, `CREATE EXTERNAL TABLE "db"."schema"."external_table" (column varchar AS (value::column::varchar) NOT NULL CONSTRAINT my_constraint UNIQUE) LOCATION = @s1/logs/ FILE_FORMAT = (TYPE = JSON)`) + }) + + t.Run("validation: neither raw file format is set, nor file format", func(t *testing.T) { + opts := &CreateDeltaLakeExternalTableOptions{ + name: NewSchemaObjectIdentifier("db", "schema", "external_table"), + Columns: []ExternalTableColumn{ + { + Name: "column", + Type: "varchar", + AsExpression: []string{"value::column::varchar"}, + InlineConstraint: &ColumnInlineConstraint{ + Name: String("my_constraint"), + NotNull: Bool(true), + Type: &ColumnConstraintTypeUnique, + }, + }, + }, + Location: "@s1/logs/", + } + assertOptsInvalid(t, opts, errExactlyOneOf("CreateDeltaLakeExternalTableOptions", "RawFileFormat", "FileFormat")) + }) } func TestExternalTableUsingTemplateOpts(t *testing.T) { @@ -263,9 +386,32 @@ func TestExternalTableUsingTemplateOpts(t *testing.T) { ErrInvalidObjectIdentifier, errNotSet("CreateExternalTableUsingTemplateOptions", "Query"), errNotSet("CreateExternalTableUsingTemplateOptions", "Location"), - errNotSet("CreateExternalTableUsingTemplateOptions", "FileFormat"), + errExactlyOneOf("CreateExternalTableUsingTemplateOptions", "RawFileFormat", "FileFormat"), ) }) + + t.Run("raw file format", func(t *testing.T) { + opts := &CreateExternalTableUsingTemplateOptions{ + name: NewSchemaObjectIdentifier("db", "schema", "external_table"), + Location: "@s1/logs/", + Query: []string{ + "query statement", + }, + RawFileFormat: &RawFileFormat{Format: "TYPE = JSON"}, + } + assertOptsValidAndSQLEquals(t, opts, `CREATE EXTERNAL TABLE "db"."schema"."external_table" USING TEMPLATE (query statement) LOCATION = @s1/logs/ FILE_FORMAT = (TYPE = JSON)`) + }) + + t.Run("validation: neither raw file format is set, nor file format", func(t *testing.T) { + opts := &CreateExternalTableUsingTemplateOptions{ + name: NewSchemaObjectIdentifier("db", "schema", "external_table"), + Location: "@s1/logs/", + Query: []string{ + "query statement", + }, + } + assertOptsInvalid(t, opts, errExactlyOneOf("CreateExternalTableUsingTemplateOptions", "RawFileFormat", "FileFormat")) + }) } func TestExternalTablesAlter(t *testing.T) { diff --git a/pkg/sdk/external_tables_validations.go b/pkg/sdk/external_tables_validations.go index a34264ce0c..4bdd96c50e 100644 --- a/pkg/sdk/external_tables_validations.go +++ b/pkg/sdk/external_tables_validations.go @@ -32,9 +32,10 @@ func (opts *CreateExternalTableOptions) validate() error { if !valueSet(opts.Location) { errs = append(errs, errNotSet("CreateExternalTableOptions", "Location")) } - if !valueSet(opts.FileFormat) { - errs = append(errs, errNotSet("CreateExternalTableOptions", "FileFormat")) - } else { + if !exactlyOneValueSet(opts.RawFileFormat, opts.FileFormat) { + errs = append(errs, errExactlyOneOf("CreateExternalTableOptions", "RawFileFormat", "FileFormat")) + } + if valueSet(opts.FileFormat) { for i, ff := range opts.FileFormat { if !valueSet(ff.Name) && !valueSet(ff.Type) { errs = append(errs, errNotSet(fmt.Sprintf("CreateExternalTableOptions.FileFormat[%d]", i), "Name or Type")) @@ -61,9 +62,10 @@ func (opts *CreateWithManualPartitioningExternalTableOptions) validate() error { if !valueSet(opts.Location) { errs = append(errs, errNotSet("CreateWithManualPartitioningExternalTableOptions", "Location")) } - if !valueSet(opts.FileFormat) { - errs = append(errs, errNotSet("CreateWithManualPartitioningExternalTableOptions", "FileFormat")) - } else { + if !exactlyOneValueSet(opts.RawFileFormat, opts.FileFormat) { + errs = append(errs, errExactlyOneOf("CreateWithManualPartitioningExternalTableOptions", "RawFileFormat", "FileFormat")) + } + if valueSet(opts.FileFormat) { for i, ff := range opts.FileFormat { if !valueSet(ff.Name) && !valueSet(ff.Type) { errs = append(errs, errNotSet(fmt.Sprintf("CreateWithManualPartitioningExternalTableOptions.FileFormat[%d]", i), "Name or Type")) @@ -90,9 +92,10 @@ func (opts *CreateDeltaLakeExternalTableOptions) validate() error { if !valueSet(opts.Location) { errs = append(errs, errNotSet("CreateDeltaLakeExternalTableOptions", "Location")) } - if !valueSet(opts.FileFormat) { - errs = append(errs, errNotSet("CreateDeltaLakeExternalTableOptions", "FileFormat")) - } else { + if !exactlyOneValueSet(opts.RawFileFormat, opts.FileFormat) { + errs = append(errs, errExactlyOneOf("CreateDeltaLakeExternalTableOptions", "RawFileFormat", "FileFormat")) + } + if valueSet(opts.FileFormat) { for i, ff := range opts.FileFormat { if !valueSet(ff.Name) && !valueSet(ff.Type) { errs = append(errs, errNotSet(fmt.Sprintf("CreateDeltaLakeExternalTableOptions.FileFormat[%d]", i), "Name or Type")) @@ -119,9 +122,10 @@ func (opts *CreateExternalTableUsingTemplateOptions) validate() error { if !valueSet(opts.Location) { errs = append(errs, errNotSet("CreateExternalTableUsingTemplateOptions", "Location")) } - if !valueSet(opts.FileFormat) { - errs = append(errs, errNotSet("CreateExternalTableUsingTemplateOptions", "FileFormat")) - } else { + if !exactlyOneValueSet(opts.RawFileFormat, opts.FileFormat) { + errs = append(errs, errExactlyOneOf("CreateExternalTableUsingTemplateOptions", "RawFileFormat", "FileFormat")) + } + if valueSet(opts.FileFormat) { for i, ff := range opts.FileFormat { if !valueSet(ff.Name) && !valueSet(ff.Type) { errs = append(errs, errNotSet(fmt.Sprintf("CreateExternalTableUsingTemplateOptions.FileFormat[%d]", i), "Name or Type")) diff --git a/pkg/sdk/testint/external_tables_integration_test.go b/pkg/sdk/testint/external_tables_integration_test.go index a3eb9e639b..cbe21326d4 100644 --- a/pkg/sdk/testint/external_tables_integration_test.go +++ b/pkg/sdk/testint/external_tables_integration_test.go @@ -40,16 +40,15 @@ func TestInt_ExternalTables(t *testing.T) { return sdk.NewCreateExternalTableRequest( sdk.NewSchemaObjectIdentifier(testDb(t).Name, testSchema(t).Name, name), stageLocation, - sdk.NewExternalTableFileFormatRequest().WithFileFormatType(&sdk.ExternalTableFileFormatTypeJSON), - ) + ).WithFileFormat(sdk.NewExternalTableFileFormatRequest().WithFileFormatType(&sdk.ExternalTableFileFormatTypeJSON)) } createExternalTableWithManualPartitioningReq := func(name string) *sdk.CreateWithManualPartitioningExternalTableRequest { return sdk.NewCreateWithManualPartitioningExternalTableRequest( sdk.NewSchemaObjectIdentifier(testDb(t).Name, testSchema(t).Name, name), stageLocation, - sdk.NewExternalTableFileFormatRequest().WithFileFormatType(&sdk.ExternalTableFileFormatTypeJSON), ). + WithFileFormat(sdk.NewExternalTableFileFormatRequest().WithFileFormatType(&sdk.ExternalTableFileFormatTypeJSON)). WithOrReplace(sdk.Bool(true)). WithColumns(columnsWithPartition). WithUserSpecifiedPartitionType(sdk.Bool(true)). @@ -70,6 +69,20 @@ func TestInt_ExternalTables(t *testing.T) { assert.Equal(t, name, externalTable.Name) }) + t.Run("Create: with raw file format", func(t *testing.T) { + name := random.AlphanumericN(32) + externalTableID := sdk.NewSchemaObjectIdentifier(testDb(t).Name, testSchema(t).Name, name) + err := client.ExternalTables.Create(ctx, minimalCreateExternalTableReq(name). + WithFileFormat(nil). + WithRawFileFormat(sdk.String("TYPE = JSON")), + ) + require.NoError(t, err) + + externalTable, err := client.ExternalTables.ShowByID(ctx, sdk.NewShowExternalTableByIDRequest(externalTableID)) + require.NoError(t, err) + assert.Equal(t, name, externalTable.Name) + }) + t.Run("Create: complete", func(t *testing.T) { name := random.AlphanumericN(32) externalTableID := sdk.NewSchemaObjectIdentifier(testDb(t).Name, testSchema(t).Name, name) @@ -78,8 +91,8 @@ func TestInt_ExternalTables(t *testing.T) { sdk.NewCreateExternalTableRequest( externalTableID, stageLocation, - sdk.NewExternalTableFileFormatRequest().WithFileFormatType(&sdk.ExternalTableFileFormatTypeJSON), ). + WithFileFormat(sdk.NewExternalTableFileFormatRequest().WithFileFormatType(&sdk.ExternalTableFileFormatTypeJSON)). WithOrReplace(sdk.Bool(true)). WithColumns(columns). WithPartitionBy([]string{"filename"}). @@ -111,8 +124,8 @@ func TestInt_ExternalTables(t *testing.T) { sdk.NewCreateExternalTableUsingTemplateRequest( id, stageLocation, - sdk.NewExternalTableFileFormatRequest().WithName(sdk.String(fileFormat.ID().FullyQualifiedName())), ). + WithFileFormat(sdk.NewExternalTableFileFormatRequest().WithName(sdk.String(fileFormat.ID().FullyQualifiedName()))). WithQuery(query). WithAutoRefresh(sdk.Bool(false))) require.NoError(t, err) @@ -140,8 +153,8 @@ func TestInt_ExternalTables(t *testing.T) { sdk.NewCreateDeltaLakeExternalTableRequest( externalTableID, stageLocation, - sdk.NewExternalTableFileFormatRequest().WithFileFormatType(&sdk.ExternalTableFileFormatTypeParquet), ). + WithFileFormat(sdk.NewExternalTableFileFormatRequest().WithFileFormatType(&sdk.ExternalTableFileFormatTypeParquet)). WithOrReplace(sdk.Bool(true)). WithColumns(columnsWithPartition). WithPartitionBy([]string{"filename"}). diff --git a/pkg/sdk/testint/streams_gen_integration_test.go b/pkg/sdk/testint/streams_gen_integration_test.go index a6d1aba37f..93a03bd49a 100644 --- a/pkg/sdk/testint/streams_gen_integration_test.go +++ b/pkg/sdk/testint/streams_gen_integration_test.go @@ -60,7 +60,7 @@ func TestInt_Streams(t *testing.T) { _, _ = createStageWithURL(t, client, stageID, nycWeatherDataURL) externalTableId := sdk.NewSchemaObjectIdentifier(db.Name, schema.Name, random.AlphanumericN(32)) - err := client.ExternalTables.Create(ctx, sdk.NewCreateExternalTableRequest(externalTableId, stageLocation, sdk.NewExternalTableFileFormatRequest().WithFileFormatType(&sdk.ExternalTableFileFormatTypeJSON))) + err := client.ExternalTables.Create(ctx, sdk.NewCreateExternalTableRequest(externalTableId, stageLocation).WithFileFormat(sdk.NewExternalTableFileFormatRequest().WithFileFormatType(&sdk.ExternalTableFileFormatTypeJSON))) require.NoError(t, err) t.Cleanup(func() { err := client.ExternalTables.Drop(ctx, sdk.NewDropExternalTableRequest(externalTableId)) diff --git a/pkg/snowflake/external_table.go b/pkg/snowflake/external_table.go deleted file mode 100644 index 9eceebf25b..0000000000 --- a/pkg/snowflake/external_table.go +++ /dev/null @@ -1,242 +0,0 @@ -package snowflake - -import ( - "database/sql" - "errors" - "fmt" - "log" - "strings" - - "github.com/jmoiron/sqlx" -) - -// externalTableBuilder abstracts the creation of SQL queries for a Snowflake schema. -type ExternalTableBuilder struct { - name string - db string - schema string - columns []map[string]string - partitionBys []string - location string - refreshOnCreate bool - autoRefresh bool - pattern string - fileFormat string - copyGrants bool - awsSNSTopic string - comment string - tags []TagValue -} - -// QualifiedName prepends the db and schema if set and escapes everything nicely. -func (tb *ExternalTableBuilder) QualifiedName() string { - var n strings.Builder - - if tb.db != "" && tb.schema != "" { - n.WriteString(fmt.Sprintf(`"%v"."%v".`, tb.db, tb.schema)) - } - - if tb.db != "" && tb.schema == "" { - n.WriteString(fmt.Sprintf(`"%v"..`, tb.db)) - } - - if tb.db == "" && tb.schema != "" { - n.WriteString(fmt.Sprintf(`"%v".`, tb.schema)) - } - - n.WriteString(fmt.Sprintf(`"%v"`, tb.name)) - - return n.String() -} - -// WithComment adds a comment to the ExternalTableBuilder. -func (tb *ExternalTableBuilder) WithComment(c string) *ExternalTableBuilder { - tb.comment = c - return tb -} - -// WithColumns sets the column definitions on the ExternalTableBuilder. -func (tb *ExternalTableBuilder) WithColumns(c []map[string]string) *ExternalTableBuilder { - tb.columns = c - return tb -} - -func (tb *ExternalTableBuilder) WithPartitionBys(c []string) *ExternalTableBuilder { - tb.partitionBys = c - return tb -} - -func (tb *ExternalTableBuilder) WithLocation(c string) *ExternalTableBuilder { - tb.location = c - return tb -} - -func (tb *ExternalTableBuilder) WithRefreshOnCreate(c bool) *ExternalTableBuilder { - tb.refreshOnCreate = c - return tb -} - -func (tb *ExternalTableBuilder) WithAutoRefresh(c bool) *ExternalTableBuilder { - tb.autoRefresh = c - return tb -} - -func (tb *ExternalTableBuilder) WithPattern(c string) *ExternalTableBuilder { - tb.pattern = c - return tb -} - -func (tb *ExternalTableBuilder) WithFileFormat(c string) *ExternalTableBuilder { - tb.fileFormat = c - return tb -} - -func (tb *ExternalTableBuilder) WithCopyGrants(c bool) *ExternalTableBuilder { - tb.copyGrants = c - return tb -} - -func (tb *ExternalTableBuilder) WithAwsSNSTopic(c string) *ExternalTableBuilder { - tb.awsSNSTopic = c - return tb -} - -// WithTags sets the tags on the ExternalTableBuilder. -func (tb *ExternalTableBuilder) WithTags(tags []TagValue) *ExternalTableBuilder { - tb.tags = tags - return tb -} - -// ExternalexternalTable returns a pointer to a Builder that abstracts the DDL operations for a externalTable. -// -// Supported DDL operations are: -// - CREATE externalTable -// -// [Snowflake Reference](https://docs.snowflake.com/en/sql-reference/sql/create-external-table.html) - -func NewExternalTableBuilder(name, db, schema string) *ExternalTableBuilder { - return &ExternalTableBuilder{ - name: name, - db: db, - schema: schema, - } -} - -// Create returns the SQL statement required to create a externalTable. -func (tb *ExternalTableBuilder) Create() string { - q := strings.Builder{} - q.WriteString(fmt.Sprintf(`CREATE EXTERNAL TABLE %v`, tb.QualifiedName())) - - q.WriteString(` (`) - columnDefinitions := []string{} - for _, columnDefinition := range tb.columns { - columnDefinitions = append(columnDefinitions, fmt.Sprintf(`"%v" %v AS %v`, EscapeString(columnDefinition["name"]), EscapeString(columnDefinition["type"]), columnDefinition["as"])) - } - q.WriteString(strings.Join(columnDefinitions, ", ")) - q.WriteString(`)`) - - if len(tb.partitionBys) > 0 { - q.WriteString(` PARTITION BY ( `) - q.WriteString(EscapeString(strings.Join(tb.partitionBys, ", "))) - q.WriteString(` )`) - } - - q.WriteString(` WITH LOCATION = ` + EscapeString(tb.location)) - q.WriteString(fmt.Sprintf(` REFRESH_ON_CREATE = %t`, tb.refreshOnCreate)) - q.WriteString(fmt.Sprintf(` AUTO_REFRESH = %t`, tb.autoRefresh)) - - if tb.pattern != "" { - q.WriteString(fmt.Sprintf(` PATTERN = '%v'`, EscapeString(tb.pattern))) - } - - q.WriteString(fmt.Sprintf(` FILE_FORMAT = ( %v )`, tb.fileFormat)) - - if tb.awsSNSTopic != "" { - q.WriteString(fmt.Sprintf(` AWS_SNS_TOPIC = '%v'`, EscapeString(tb.awsSNSTopic))) - } - - if tb.copyGrants { - q.WriteString(" COPY GRANTS") - } - - if tb.comment != "" { - q.WriteString(fmt.Sprintf(` COMMENT = '%v'`, EscapeString(tb.comment))) - } - - if len(tb.tags) > 0 { - q.WriteString(fmt.Sprintf(` WITH TAG (%s)`, tb.GetTagValueString())) - } - - return q.String() -} - -// Update returns the SQL statement required to update an externalTable. -func (tb *ExternalTableBuilder) Update() string { - q := strings.Builder{} - q.WriteString(fmt.Sprintf(`ALTER EXTERNAL TABLE %v`, tb.QualifiedName())) - - if len(tb.tags) > 0 { - q.WriteString(fmt.Sprintf(` TAG %s`, tb.GetTagValueString())) - } - - return q.String() -} - -// Drop returns the SQL query that will drop a externalTable. -func (tb *ExternalTableBuilder) Drop() string { - return fmt.Sprintf(`DROP EXTERNAL TABLE %v`, tb.QualifiedName()) -} - -// Show returns the SQL query that will show a externalTable. -func (tb *ExternalTableBuilder) Show() string { - return fmt.Sprintf(`SHOW EXTERNAL TABLES LIKE '%v' IN SCHEMA "%v"."%v"`, tb.name, tb.db, tb.schema) -} - -func (tb *ExternalTableBuilder) GetTagValueString() string { - var q strings.Builder - for _, v := range tb.tags { - fmt.Println(v) - if v.Schema != "" { - if v.Database != "" { - q.WriteString(fmt.Sprintf(`"%v".`, v.Database)) - } - q.WriteString(fmt.Sprintf(`"%v".`, v.Schema)) - } - q.WriteString(fmt.Sprintf(`"%v" = "%v", `, v.Name, v.Value)) - } - return strings.TrimSuffix(q.String(), ", ") -} - -type ExternalTable struct { - CreatedOn sql.NullString `db:"created_on"` - ExternalTableName sql.NullString `db:"name"` - DatabaseName sql.NullString `db:"database_name"` - SchemaName sql.NullString `db:"schema_name"` - Comment sql.NullString `db:"comment"` - Owner sql.NullString `db:"owner"` -} - -func ScanExternalTable(row *sqlx.Row) (*ExternalTable, error) { - t := &ExternalTable{} - e := row.StructScan(t) - return t, e -} - -func ListExternalTables(databaseName string, schemaName string, db *sql.DB) ([]ExternalTable, error) { - stmt := fmt.Sprintf(`SHOW EXTERNAL TABLES IN SCHEMA "%s"."%v"`, databaseName, schemaName) - rows, err := Query(db, stmt) - if err != nil { - return nil, err - } - defer rows.Close() - - dbs := []ExternalTable{} - if err := sqlx.StructScan(rows, &dbs); err != nil { - if errors.Is(err, sql.ErrNoRows) { - log.Println("[DEBUG] no external tables found") - return nil, nil - } - return nil, fmt.Errorf("unable to scan row for %s err = %w", stmt, err) - } - return dbs, nil -} diff --git a/pkg/snowflake/external_table_test.go b/pkg/snowflake/external_table_test.go deleted file mode 100644 index f55ef97e30..0000000000 --- a/pkg/snowflake/external_table_test.go +++ /dev/null @@ -1,42 +0,0 @@ -package snowflake - -import ( - "testing" - - "github.com/stretchr/testify/require" -) - -func TestExternalTableCreate(t *testing.T) { - r := require.New(t) - s := NewExternalTableBuilder("test_table", "test_db", "test_schema") - s.WithColumns([]map[string]string{{"name": "column1", "type": "OBJECT", "as": "expression1"}, {"name": "column2", "type": "VARCHAR", "as": "expression2"}}) - s.WithLocation("location") - s.WithPattern("pattern") - s.WithFileFormat("TYPE = CSV FIELD_DELIMITER = '|'") - r.Equal(`"test_db"."test_schema"."test_table"`, s.QualifiedName()) - - r.Equal(`CREATE EXTERNAL TABLE "test_db"."test_schema"."test_table" ("column1" OBJECT AS expression1, "column2" VARCHAR AS expression2) WITH LOCATION = location REFRESH_ON_CREATE = false AUTO_REFRESH = false PATTERN = 'pattern' FILE_FORMAT = ( TYPE = CSV FIELD_DELIMITER = '|' )`, s.Create()) - - s.WithComment("Test Comment") - r.Equal(`CREATE EXTERNAL TABLE "test_db"."test_schema"."test_table" ("column1" OBJECT AS expression1, "column2" VARCHAR AS expression2) WITH LOCATION = location REFRESH_ON_CREATE = false AUTO_REFRESH = false PATTERN = 'pattern' FILE_FORMAT = ( TYPE = CSV FIELD_DELIMITER = '|' ) COMMENT = 'Test Comment'`, s.Create()) -} - -func TestExternalTableUpdate(t *testing.T) { - r := require.New(t) - s := NewExternalTableBuilder("test_table", "test_db", "test_schema") - s.WithTags([]TagValue{{Name: "tag1", Value: "value1", Schema: "test_schema", Database: "test_db"}}) - expected := `ALTER EXTERNAL TABLE "test_db"."test_schema"."test_table" TAG "test_db"."test_schema"."tag1" = "value1"` - r.Equal(expected, s.Update()) -} - -func TestExternalTableDrop(t *testing.T) { - r := require.New(t) - s := NewExternalTableBuilder("test_table", "test_db", "test_schema") - r.Equal(`DROP EXTERNAL TABLE "test_db"."test_schema"."test_table"`, s.Drop()) -} - -func TestExternalTableShow(t *testing.T) { - r := require.New(t) - s := NewExternalTableBuilder("test_table", "test_db", "test_schema") - r.Equal(`SHOW EXTERNAL TABLES LIKE 'test_table' IN SCHEMA "test_db"."test_schema"`, s.Show()) -}