From 48153ae5eee2546d414a08fe40dd4eaca009793a Mon Sep 17 00:00:00 2001 From: = Date: Thu, 5 Dec 2024 10:21:58 +0100 Subject: [PATCH] feat: get datasource tables on par with views --- pkg/datasources/tables.go | 159 +++++++++++++++++++++++--------- pkg/schemas/table.go | 83 +++++++++++++++++ pkg/sdk/tables.go | 16 ++-- pkg/sdk/tables_dto.go | 28 +++--- pkg/sdk/tables_dto_generated.go | 20 ++-- pkg/sdk/tables_impl.go | 3 +- 6 files changed, 230 insertions(+), 79 deletions(-) create mode 100644 pkg/schemas/table.go diff --git a/pkg/datasources/tables.go b/pkg/datasources/tables.go index fac59da574..8920b620bd 100644 --- a/pkg/datasources/tables.go +++ b/pkg/datasources/tables.go @@ -2,51 +2,84 @@ package datasources import ( "context" - "log" "github.com/Snowflake-Labs/terraform-provider-snowflake/pkg/provider/datasources" - "github.com/hashicorp/terraform-plugin-sdk/v2/diag" "github.com/Snowflake-Labs/terraform-provider-snowflake/pkg/internal/provider" + "github.com/Snowflake-Labs/terraform-provider-snowflake/pkg/resources" + "github.com/Snowflake-Labs/terraform-provider-snowflake/pkg/schemas" + "github.com/hashicorp/terraform-plugin-sdk/v2/diag" - "github.com/Snowflake-Labs/terraform-provider-snowflake/pkg/helpers" "github.com/Snowflake-Labs/terraform-provider-snowflake/pkg/sdk" "github.com/hashicorp/terraform-plugin-sdk/v2/helper/schema" ) var tablesSchema = map[string]*schema.Schema{ - "database": { + "with_describe": { + Type: schema.TypeBool, + Optional: true, + Default: true, + Description: "Runs DESC TABLE for each table returned by SHOW TABLES. The output of describe is saved to the description field. By default this value is set to true.", + }, + "in": { + Type: schema.TypeList, + Optional: true, + Description: "IN clause to filter the list of tables", + MaxItems: 1, + Elem: &schema.Resource{ + Schema: map[string]*schema.Schema{ + "account": { + Type: schema.TypeBool, + Optional: true, + Description: "Returns records for the entire account.", + ExactlyOneOf: []string{"in.0.account", "in.0.database", "in.0.schema"}, + }, + "database": { + Type: schema.TypeString, + Optional: true, + Description: "Returns records for the current database in use or for a specified database.", + ExactlyOneOf: []string{"in.0.account", "in.0.database", "in.0.schema"}, + }, + "schema": { + Type: schema.TypeString, + Optional: true, + Description: "Returns records for the current schema in use or a specified schema. Use fully qualified name.", + ExactlyOneOf: []string{"in.0.account", "in.0.database", "in.0.schema"}, + }, + }, + }, + }, + "like": { Type: schema.TypeString, - Required: true, - Description: "The database from which to return the schemas from.", + Optional: true, + Description: "Filters the output with **case-insensitive** pattern, with support for SQL wildcard characters (`%` and `_`).", }, - "schema": { + "starts_with": { Type: schema.TypeString, - Required: true, - Description: "The schema from which to return the tables from.", + Optional: true, + Description: "Filters the output with **case-sensitive** characters indicating the beginning of the object name.", }, "tables": { Type: schema.TypeList, Computed: true, - Description: "The tables in the schema", + Description: "Holds the aggregated output of all tables details queries.", Elem: &schema.Resource{ Schema: map[string]*schema.Schema{ - "name": { - Type: schema.TypeString, - Computed: true, + resources.ShowOutputAttributeName: { + Type: schema.TypeList, + Computed: true, + Description: "Holds the output of SHOW TABLES.", + Elem: &schema.Resource{ + Schema: schemas.ShowTableSchema, + }, }, - "database": { - Type: schema.TypeString, - Computed: true, - }, - "schema": { - Type: schema.TypeString, - Computed: true, - }, - "comment": { - Type: schema.TypeString, - Optional: true, - Computed: true, + resources.DescribeOutputAttributeName: { + Type: schema.TypeList, + Computed: true, + Description: "Holds the output of DESCRIBE TABLES.", + Elem: &schema.Resource{ + Schema: schemas.TableDescribeSchema, + }, }, }, }, @@ -57,41 +90,75 @@ func Tables() *schema.Resource { return &schema.Resource{ ReadContext: TrackingReadWrapper(datasources.Tables, ReadTables), Schema: tablesSchema, + Description: "Datasource used to get details of filtered tables. Filtering is aligned with the current possibilities for [SHOW VIEWS](https://docs.snowflake.com/en/sql-reference/sql/show-tables) query (only `like` is supported). The results of SHOW and DESCRIBE are encapsulated in one output collection `tables`.", } } func ReadTables(ctx context.Context, d *schema.ResourceData, meta any) diag.Diagnostics { client := meta.(*provider.Context).Client - databaseName := d.Get("database").(string) - schemaName := d.Get("schema").(string) + req := sdk.NewShowTableRequest() + + if v, ok := d.GetOk("in"); ok { + in := v.([]any)[0].(map[string]any) + if v, ok := in["account"]; ok && v.(bool) { + req.WithIn(sdk.ExtendedIn{In: sdk.In{Account: sdk.Bool(true)}}) + } + if v, ok := in["database"]; ok { + database := v.(string) + if database != "" { + req.WithIn(sdk.ExtendedIn{In: sdk.In{Database: sdk.NewAccountObjectIdentifier(database)}}) + } + } + if v, ok := in["schema"]; ok { + schema := v.(string) + if schema != "" { + schemaId, err := sdk.ParseDatabaseObjectIdentifier(schema) + if err != nil { + return diag.FromErr(err) + } + req.WithIn(sdk.ExtendedIn{In: sdk.In{Schema: schemaId}}) + } + } + } + + if likePattern, ok := d.GetOk("like"); ok { + req.WithLike(sdk.Like{ + Pattern: sdk.String(likePattern.(string)), + }) + } - schemaId := sdk.NewDatabaseObjectIdentifier(databaseName, schemaName) - extractedTables, err := client.Tables.Show(ctx, sdk.NewShowTableRequest().WithIn( - &sdk.In{Schema: schemaId}, - )) + if v, ok := d.GetOk("starts_with"); ok { + req.WithStartsWith(v.(string)) + } + + tables, err := client.Tables.Show(ctx, req) if err != nil { - log.Printf("[DEBUG] failed when searching tables in schema (%s), err = %s", schemaId.FullyQualifiedName(), err.Error()) - d.SetId("") - return nil + return diag.FromErr(err) } - tables := make([]map[string]any, 0) + d.SetId("tables_read") - for _, extractedTable := range extractedTables { - if extractedTable.IsExternal { - continue + flattenedTables := make([]map[string]any, len(tables)) + for i, table := range tables { + table := table + var tableDescriptions []map[string]any + if d.Get("with_describe").(bool) { + describeOutput, err := client.Tables.DescribeColumns(ctx, sdk.NewDescribeTableColumnsRequest(table.ID())) + if err != nil { + return diag.FromErr(err) + } + tableDescriptions = schemas.TableDescriptionToSchema(describeOutput) } - table := map[string]any{ - "name": extractedTable.Name, - "database": extractedTable.DatabaseName, - "schema": extractedTable.SchemaName, - "comment": extractedTable.Comment, + flattenedTables[i] = map[string]any{ + resources.ShowOutputAttributeName: []map[string]any{schemas.TableToSchema(&table)}, + resources.DescribeOutputAttributeName: tableDescriptions, } + } - tables = append(tables, table) + if err := d.Set("tables", flattenedTables); err != nil { + return diag.FromErr(err) } - d.SetId(helpers.EncodeSnowflakeID(databaseName, schemaName)) - return diag.FromErr(d.Set("tables", tables)) + return nil } diff --git a/pkg/schemas/table.go b/pkg/schemas/table.go new file mode 100644 index 0000000000..e8fe9fddb0 --- /dev/null +++ b/pkg/schemas/table.go @@ -0,0 +1,83 @@ +package schemas + +import ( + "github.com/Snowflake-Labs/terraform-provider-snowflake/pkg/sdk" + "github.com/hashicorp/terraform-plugin-sdk/v2/helper/schema" +) + +var TableDescribeSchema = map[string]*schema.Schema{ + "name": { + Type: schema.TypeString, + Computed: true, + }, + "type": { + Type: schema.TypeString, + Computed: true, + }, + "kind": { + Type: schema.TypeString, + Computed: true, + }, + "is_nullable": { + Type: schema.TypeBool, + Computed: true, + }, + "default": { + Type: schema.TypeString, + Computed: true, + }, + "is_primary": { + Type: schema.TypeBool, + Computed: true, + }, + "is_unique": { + Type: schema.TypeBool, + Computed: true, + }, + "check": { + Type: schema.TypeString, + Computed: true, + }, + "expression": { + Type: schema.TypeString, + Computed: true, + }, + "comment": { + Type: schema.TypeString, + Computed: true, + }, + "policy_name": { + Type: schema.TypeString, + Computed: true, + }, + "collation": { + Type: schema.TypeString, + Computed: true, + }, + "schema_evolution_record": { + Type: schema.TypeString, + Computed: true, + }, +} + +func TableDescriptionToSchema(description []sdk.TableColumnDetails) []map[string]any { + result := make([]map[string]any, len(description)) + for i, row := range description { + result[i] = map[string]any{ + "name": row.Name, + "type": row.Type, + "kind": row.Kind, + "is_nullable": row.IsNullable, + "default": row.Default, + "is_primary": row.IsPrimary, + "is_unique": row.IsUnique, + "check": row.Check, + "expression": row.Expression, + "comment": row.Comment, + "policy_name": row.PolicyName, + "collation": row.Collation, + "schema_evolution_record": row.SchemaEvolutionRecord, + } + } + return result +} diff --git a/pkg/sdk/tables.go b/pkg/sdk/tables.go index bba157ba8f..f5bdb7a3da 100644 --- a/pkg/sdk/tables.go +++ b/pkg/sdk/tables.go @@ -497,14 +497,14 @@ type dropTableOptions struct { } type showTableOptions struct { - show bool `ddl:"static" sql:"SHOW"` - Terse *bool `ddl:"keyword" sql:"TERSE"` - tables bool `ddl:"static" sql:"TABLES"` - History *bool `ddl:"keyword" sql:"HISTORY"` - Like *Like `ddl:"keyword" sql:"LIKE"` - In *In `ddl:"keyword" sql:"IN"` - StartsWith *string `ddl:"parameter,single_quotes,no_equals" sql:"STARTS WITH"` - LimitFrom *LimitFrom `ddl:"keyword" sql:"LIMIT"` + show bool `ddl:"static" sql:"SHOW"` + Terse *bool `ddl:"keyword" sql:"TERSE"` + tables bool `ddl:"static" sql:"TABLES"` + History *bool `ddl:"keyword" sql:"HISTORY"` + Like *Like `ddl:"keyword" sql:"LIKE"` + In *ExtendedIn `ddl:"keyword" sql:"IN"` + StartsWith *string `ddl:"parameter,single_quotes,no_equals" sql:"STARTS WITH"` + LimitFrom *LimitFrom `ddl:"keyword" sql:"LIMIT"` } type tableDBRow struct { diff --git a/pkg/sdk/tables_dto.go b/pkg/sdk/tables_dto.go index c11e0d65cd..3cd193ee4e 100644 --- a/pkg/sdk/tables_dto.go +++ b/pkg/sdk/tables_dto.go @@ -209,25 +209,25 @@ func (s *DropTableRequest) toOpts() *dropTableOptions { func (s *ShowTableRequest) toOpts() *showTableOptions { var like *Like - if s.likePattern != "" { + if s.Like != nil { like = &Like{ - Pattern: &s.likePattern, + Pattern: s.Like.Pattern, } } var limitFrom *LimitFrom - if s.limitFrom != nil { + if s.Limit != nil { limitFrom = &LimitFrom{ - Rows: s.limitFrom.Rows, - From: s.limitFrom.From, + Rows: s.Limit.Rows, + From: s.Limit.From, } } return &showTableOptions{ - Terse: s.terse, + Terse: s.Terse, History: s.history, Like: like, - StartsWith: s.startsWith, + StartsWith: s.StartsWith, LimitFrom: limitFrom, - In: s.in, + In: s.In, } } @@ -519,12 +519,12 @@ type TableExternalTableColumnDropActionRequest struct { } type ShowTableRequest struct { - terse *bool - history *bool - likePattern string - in *In - startsWith *string - limitFrom *LimitFrom + Terse *bool + history *bool + Like *Like + In *ExtendedIn + StartsWith *string + Limit *LimitFrom } type ShowTableInRequest struct { diff --git a/pkg/sdk/tables_dto_generated.go b/pkg/sdk/tables_dto_generated.go index 5efceac847..0755d664b5 100644 --- a/pkg/sdk/tables_dto_generated.go +++ b/pkg/sdk/tables_dto_generated.go @@ -1618,8 +1618,8 @@ func NewShowTableRequest() *ShowTableRequest { return &ShowTableRequest{} } -func (s *ShowTableRequest) WithTerse(terse *bool) *ShowTableRequest { - s.terse = terse +func (s *ShowTableRequest) WithTerse(Terse bool) *ShowTableRequest { + s.Terse = &Terse return s } @@ -1628,23 +1628,23 @@ func (s *ShowTableRequest) WithHistory(history *bool) *ShowTableRequest { return s } -func (s *ShowTableRequest) WithLikePattern(likePattern string) *ShowTableRequest { - s.likePattern = likePattern +func (s *ShowTableRequest) WithLike(Like Like) *ShowTableRequest { + s.Like = &Like return s } -func (s *ShowTableRequest) WithIn(in *In) *ShowTableRequest { - s.in = in +func (s *ShowTableRequest) WithIn(In ExtendedIn) *ShowTableRequest { + s.In = &In return s } -func (s *ShowTableRequest) WithStartsWith(startsWith *string) *ShowTableRequest { - s.startsWith = startsWith +func (s *ShowTableRequest) WithStartsWith(StartsWith string) *ShowTableRequest { + s.StartsWith = &StartsWith return s } -func (s *ShowTableRequest) WithLimitFrom(limitFrom *LimitFrom) *ShowTableRequest { - s.limitFrom = limitFrom +func (s *ShowTableRequest) WithLimitFrom(Limit LimitFrom) *ShowTableRequest { + s.Limit = &Limit return s } diff --git a/pkg/sdk/tables_impl.go b/pkg/sdk/tables_impl.go index f26ea5363d..8f0a8e2a09 100644 --- a/pkg/sdk/tables_impl.go +++ b/pkg/sdk/tables_impl.go @@ -78,7 +78,8 @@ func (v *tables) Show(ctx context.Context, request *ShowTableRequest) ([]Table, } func (v *tables) ShowByID(ctx context.Context, id SchemaObjectIdentifier) (*Table, error) { - request := NewShowTableRequest().WithIn(&In{Schema: id.SchemaId()}).WithLikePattern(id.Name()) + request := NewShowTableRequest().WithIn(ExtendedIn{In: In{Schema: id.SchemaId()}}). + WithLike(Like{Pattern: String(id.Name())}) returnedTables, err := v.Show(ctx, request) if err != nil { return nil, err