From fd7aad460dbe19c629e0f3bcb473e4aefbb04661 Mon Sep 17 00:00:00 2001 From: Norbert Dopjera Date: Sun, 18 Feb 2024 20:12:15 +0100 Subject: [PATCH] fix(warehouse): BQ credentials drift changes (#84) * feat(warehouse): credentials drift changes * feat(warehouse): BQ credentials drift detection finished * fix(warehouse): BQ tests with new credentials structure --- client/monte_carlo_client.go | 16 +- internal/warehouse/bigquery_warehouse.go | 149 ++++++++++++++---- internal/warehouse/bigquery_warehouse_test.go | 8 +- .../create.tf | 2 +- .../update.tf | 2 +- 5 files changed, 138 insertions(+), 39 deletions(-) diff --git a/client/monte_carlo_client.go b/client/monte_carlo_client.go index 32788bf..6c7b159 100644 --- a/client/monte_carlo_client.go +++ b/client/monte_carlo_client.go @@ -83,6 +83,8 @@ type AddConnection struct { AddConnection struct { Connection struct { Uuid string + CreatedOn string + UpdatedOn string Warehouse struct { Name string Uuid string @@ -95,8 +97,10 @@ type GetWarehouse struct { GetWarehouse *struct { Name string `json:"name"` Connections []struct { - Uuid string `json:"uuid"` - Type string `json:"type"` + Uuid string `json:"uuid"` + Type string `json:"type"` + CreatedOn string `json:"createdOn"` + UpdatedOn string `json:"updatedOn"` } `json:"connections"` DataCollector struct { Uuid string `json:"uuid"` @@ -108,7 +112,7 @@ const BigQueryConnectionType = "bigquery" const BigQueryConnectionTypeResponse = "BIGQUERY" const TransactionalConnectionType = "transactional-db" const TransactionalConnectionTypeResponse = "TRANSACTIONAL_DB" -const GetWarehouseQuery string = "query getWarehouse($uuid: UUID) { getWarehouse(uuid: $uuid) { name,connections{uuid,type},dataCollector{uuid} } }" +const GetWarehouseQuery string = "query getWarehouse($uuid: UUID) { getWarehouse(uuid: $uuid) { name,connections{uuid,type,createdOn,updatedOn},dataCollector{uuid} } }" type RemoveConnection struct { RemoveConnection struct { @@ -127,7 +131,8 @@ type SetWarehouseName struct { type UpdateCredentials struct { UpdateCredentials struct { - Success bool + Success bool + UpdatedAt string } `graphql:"updateCredentials(changes: $changes, connectionId: $connectionId, shouldReplace: $shouldReplace, shouldValidate: $shouldValidate)"` } @@ -299,7 +304,7 @@ type CreateOrUpdateComparisonRule struct { Severity string RuleType string WarehouseUuid string - Comparisons []struct { + Comparisons []struct { ComparisonType string FullTableId string FullTableIds []string @@ -307,7 +312,6 @@ type CreateOrUpdateComparisonRule struct { Metric string Operator string Threshold float64 - } } } `graphql:"createOrUpdateComparisonRule(comparisons: $comparisons, customRuleUuid: $customRuleUuid, description: $description, queryResultType: $queryResultType, scheduleConfig: $scheduleConfig, sourceConnectionId: $sourceConnectionId, sourceDwId: $sourceDwId, sourceSqlQuery: $sourceSqlQuery, targetConnectionId: $targetConnectionId, targetDwId: $targetDwId, targetSqlQuery: $targetSqlQuery)"` diff --git a/internal/warehouse/bigquery_warehouse.go b/internal/warehouse/bigquery_warehouse.go index 95c17cb..045a75f 100644 --- a/internal/warehouse/bigquery_warehouse.go +++ b/internal/warehouse/bigquery_warehouse.go @@ -38,6 +38,20 @@ type BigQueryWarehouseResource struct { // BigQueryWarehouseResourceModel describes the resource data model according to its Schema. type BigQueryWarehouseResourceModel struct { + Uuid types.String `tfsdk:"uuid"` + Credentials Credentials `tfsdk:"credentials"` + Name types.String `tfsdk:"name"` + CollectorUuid types.String `tfsdk:"collector_uuid"` + DeletionProtection types.Bool `tfsdk:"deletion_protection"` +} + +type Credentials struct { + ConnectionUuid types.String `tfsdk:"connection_uuid"` + ServiceAccountKey types.String `tfsdk:"service_account_key"` + UpdatedAt types.String `tfsdk:"updated_at"` +} + +type BigQueryWarehouseResourceModelV1 struct { Uuid types.String `tfsdk:"uuid"` ConnectionUuid types.String `tfsdk:"connection_uuid"` Name types.String `tfsdk:"name"` @@ -70,11 +84,24 @@ func (r *BigQueryWarehouseResource) Schema(ctx context.Context, req resource.Sch stringplanmodifier.UseStateForUnknown(), }, }, - "connection_uuid": schema.StringAttribute{ - Computed: true, - Optional: false, - PlanModifiers: []planmodifier.String{ - stringplanmodifier.UseStateForUnknown(), + "credentials": schema.SingleNestedAttribute{ + Required: true, + Attributes: map[string]schema.Attribute{ + "connection_uuid": schema.StringAttribute{ + Computed: true, + Optional: false, + PlanModifiers: []planmodifier.String{ + stringplanmodifier.UseStateForUnknown(), + }, + }, + "service_account_key": schema.StringAttribute{ + Required: true, + Sensitive: true, + }, + "updated_at": schema.StringAttribute{ + Computed: true, + Optional: false, + }, }, }, "name": schema.StringAttribute{ @@ -87,10 +114,6 @@ func (r *BigQueryWarehouseResource) Schema(ctx context.Context, req resource.Sch stringplanmodifier.RequiresReplaceIfConfigured(), }, }, - "service_account_key": schema.StringAttribute{ - Required: true, - Sensitive: true, - }, "deletion_protection": schema.BoolAttribute{ Optional: true, Computed: true, @@ -120,8 +143,8 @@ func (r *BigQueryWarehouseResource) Create(ctx context.Context, req resource.Cre } data.Uuid = result.Uuid - data.ConnectionUuid = result.ConnectionUuid - data.Name = result.Name + data.Credentials.UpdatedAt = result.Credentials.UpdatedAt + data.Credentials.ConnectionUuid = result.Credentials.ConnectionUuid resp.Diagnostics.Append(resp.State.Set(ctx, &data)...) } @@ -167,9 +190,11 @@ func (r *BigQueryWarehouseResource) Read(ctx context.Context, req resource.ReadR } readConnectionUuid := types.StringNull() - readServiceAccountKey := types.StringNull() + readConnectionSAKey := types.StringNull() + readConnectionUpdatedAt := types.StringNull() + for _, connection := range getResult.GetWarehouse.Connections { - if connection.Uuid == data.ConnectionUuid.ValueString() { + if connection.Uuid == data.Credentials.ConnectionUuid.ValueString() { if connection.Type != client.BigQueryConnectionTypeResponse { resp.Diagnostics.AddError( fmt.Sprintf("Obtained Warehouse [uuid: %s, connection_uuid: %s] but got unexpected connection "+ @@ -177,13 +202,23 @@ func (r *BigQueryWarehouseResource) Read(ctx context.Context, req resource.ReadR "Users can manually fix remote state or delete this resource from the Terraform configuration.") return } - readConnectionUuid = data.ConnectionUuid - readServiceAccountKey = data.ServiceAccountKey + + readConnectionUuid = data.Credentials.ConnectionUuid + readConnectionSAKey = data.Credentials.ServiceAccountKey + readConnectionUpdatedAt = types.StringValue(connection.UpdatedOn) + if connection.UpdatedOn == "" { + readConnectionUpdatedAt = types.StringValue(connection.CreatedOn) + } } } - data.ConnectionUuid = readConnectionUuid - data.ServiceAccountKey = readServiceAccountKey + if !readConnectionSAKey.IsNull() && !readConnectionUpdatedAt.Equal(data.Credentials.UpdatedAt) { + readConnectionSAKey = types.StringValue("(unknown external value)") + } + + data.Credentials.UpdatedAt = readConnectionUpdatedAt + data.Credentials.ConnectionUuid = readConnectionUuid + data.Credentials.ServiceAccountKey = readConnectionSAKey data.Name = types.StringValue(getResult.GetWarehouse.Name) resp.Diagnostics.Append(resp.State.Set(ctx, &data)...) } @@ -207,10 +242,11 @@ func (r *BigQueryWarehouseResource) Update(ctx context.Context, req resource.Upd return } - if data.ConnectionUuid.IsUnknown() || data.ConnectionUuid.IsNull() { + if data.Credentials.ConnectionUuid.IsUnknown() || data.Credentials.ConnectionUuid.IsNull() { if result, diags := r.addConnection(ctx, data); result != nil { resp.Diagnostics.Append(diags...) - data.ConnectionUuid = result.ConnectionUuid + data.Credentials.UpdatedAt = result.Credentials.UpdatedAt + data.Credentials.ConnectionUuid = result.Credentials.ConnectionUuid } else { resp.Diagnostics.Append(diags...) return @@ -219,8 +255,8 @@ func (r *BigQueryWarehouseResource) Update(ctx context.Context, req resource.Upd updateResult := client.UpdateCredentials{} variables = map[string]interface{}{ - "changes": client.JSONString(data.ServiceAccountKey.ValueString()), - "connectionId": client.UUID(data.ConnectionUuid.ValueString()), + "changes": client.JSONString(data.Credentials.ServiceAccountKey.ValueString()), + "connectionId": client.UUID(data.Credentials.ConnectionUuid.ValueString()), "shouldReplace": true, "shouldValidate": true, } @@ -235,6 +271,8 @@ func (r *BigQueryWarehouseResource) Update(ctx context.Context, req resource.Upd resp.Diagnostics.AddError(toPrint, "") return } + + data.Credentials.UpdatedAt = types.StringValue(updateResult.UpdateCredentials.UpdatedAt) resp.Diagnostics.Append(resp.State.Set(ctx, &data)...) } @@ -256,7 +294,7 @@ func (r *BigQueryWarehouseResource) Delete(ctx context.Context, req resource.Del } removeResult := client.RemoveConnection{} - variables := map[string]interface{}{"connectionId": client.UUID(data.ConnectionUuid.ValueString())} + variables := map[string]interface{}{"connectionId": client.UUID(data.Credentials.ConnectionUuid.ValueString())} if err := r.client.Mutate(ctx, &removeResult, variables); err != nil { toPrint := fmt.Sprintf("MC client 'RemoveConnection' mutation result - %s", err.Error()) resp.Diagnostics.AddError(toPrint, "") @@ -272,7 +310,7 @@ func (r *BigQueryWarehouseResource) ImportState(ctx context.Context, req resourc idsImported := strings.Split(req.ID, ",") if len(idsImported) == 3 && idsImported[0] != "" && idsImported[1] != "" && idsImported[2] != "" { resp.Diagnostics.Append(resp.State.SetAttribute(ctx, path.Root("uuid"), idsImported[0])...) - resp.Diagnostics.Append(resp.State.SetAttribute(ctx, path.Root("connection_uuid"), idsImported[1])...) + resp.Diagnostics.Append(resp.State.SetAttribute(ctx, path.Root("credentials").AtName("connection_uuid"), idsImported[1])...) resp.Diagnostics.Append(resp.State.SetAttribute(ctx, path.Root("collector_uuid"), idsImported[2])...) } else { resp.Diagnostics.AddError("Unexpected Import Identifier", fmt.Sprintf( @@ -289,7 +327,7 @@ func (r *BigQueryWarehouseResource) addConnection(ctx context.Context, data BigQ "validationName": "save_credentials", "connectionDetails": BqConnectionDetails{ "serviceJson": b64.StdEncoding.EncodeToString( - []byte(data.ServiceAccountKey.ValueString()), + []byte(data.Credentials.ServiceAccountKey.ValueString()), ), }, } @@ -334,8 +372,8 @@ func (r *BigQueryWarehouseResource) addConnection(ctx context.Context, data BigQ } data.Uuid = types.StringValue(addResult.AddConnection.Connection.Warehouse.Uuid) - data.ConnectionUuid = types.StringValue(addResult.AddConnection.Connection.Uuid) - data.Name = types.StringValue(addResult.AddConnection.Connection.Warehouse.Name) + data.Credentials.UpdatedAt = types.StringValue(addResult.AddConnection.Connection.CreatedOn) + data.Credentials.ConnectionUuid = types.StringValue(addResult.AddConnection.Connection.Uuid) return &data, diagsResult } @@ -398,7 +436,7 @@ func (r *BigQueryWarehouseResource) UpgradeState(ctx context.Context) map[int64] var priorStateData BigQueryWarehouseResourceModelV0 resp.Diagnostics.Append(req.State.Get(ctx, &priorStateData)...) if !resp.Diagnostics.HasError() { - upgradedStateData := BigQueryWarehouseResourceModel{ + upgradedStateData := BigQueryWarehouseResourceModelV1{ Uuid: priorStateData.Uuid, ConnectionUuid: priorStateData.ConnectionUuid, CollectorUuid: priorStateData.DataCollectorUuid, @@ -410,5 +448,62 @@ func (r *BigQueryWarehouseResource) UpgradeState(ctx context.Context) map[int64] } }, }, + 1: { + PriorSchema: &schema.Schema{ + Attributes: map[string]schema.Attribute{ + "uuid": schema.StringAttribute{ + Computed: true, + Optional: false, + PlanModifiers: []planmodifier.String{ + stringplanmodifier.UseStateForUnknown(), + }, + }, + "connection_uuid": schema.StringAttribute{ + Computed: true, + Optional: false, + PlanModifiers: []planmodifier.String{ + stringplanmodifier.UseStateForUnknown(), + }, + }, + "name": schema.StringAttribute{ + Required: true, + Validators: []validator.String{stringvalidator.LengthAtLeast(1)}, + }, + "collector_uuid": schema.StringAttribute{ + Required: true, + PlanModifiers: []planmodifier.String{ + stringplanmodifier.RequiresReplaceIfConfigured(), + }, + }, + "service_account_key": schema.StringAttribute{ + Required: true, + Sensitive: true, + }, + "deletion_protection": schema.BoolAttribute{ + Optional: true, + Computed: true, + Default: booldefault.StaticBool(true), + }, + }, + }, + StateUpgrader: func(ctx context.Context, req resource.UpgradeStateRequest, resp *resource.UpgradeStateResponse) { + var priorStateData BigQueryWarehouseResourceModelV1 + resp.Diagnostics.Append(req.State.Get(ctx, &priorStateData)...) + if !resp.Diagnostics.HasError() { + upgradedStateData := BigQueryWarehouseResourceModel{ + Uuid: priorStateData.Uuid, + CollectorUuid: priorStateData.CollectorUuid, + Name: priorStateData.Name, + DeletionProtection: priorStateData.DeletionProtection, + Credentials: Credentials{ + ConnectionUuid: priorStateData.ConnectionUuid, + ServiceAccountKey: priorStateData.ServiceAccountKey, + UpdatedAt: types.StringNull(), + }, + } + resp.Diagnostics.Append(resp.State.Set(ctx, upgradedStateData)...) + } + }, + }, } } diff --git a/internal/warehouse/bigquery_warehouse_test.go b/internal/warehouse/bigquery_warehouse_test.go index 835076f..6949632 100644 --- a/internal/warehouse/bigquery_warehouse_test.go +++ b/internal/warehouse/bigquery_warehouse_test.go @@ -37,7 +37,7 @@ func TestAccBigQueryWarehouseResource(t *testing.T) { Check: resource.ComposeAggregateTestCheckFunc( resource.TestCheckResourceAttr("montecarlo_bigquery_warehouse.test", "name", "test-warehouse"), resource.TestCheckResourceAttr("montecarlo_bigquery_warehouse.test", "collector_uuid", collectorUuid), - resource.TestCheckResourceAttr("montecarlo_bigquery_warehouse.test", "service_account_key", serviceAccount), + resource.TestCheckResourceAttr("montecarlo_bigquery_warehouse.test", "credentials.service_account_key", serviceAccount), resource.TestCheckResourceAttr("montecarlo_bigquery_warehouse.test", "deletion_protection", "false"), ), }, @@ -53,11 +53,11 @@ func TestAccBigQueryWarehouseResource(t *testing.T) { ImportStateVerify: true, ImportStateIdFunc: func(s *terraform.State) (string, error) { uuid := s.RootModule().Resources["montecarlo_bigquery_warehouse.test"].Primary.Attributes["uuid"] - connectionUuid := s.RootModule().Resources["montecarlo_bigquery_warehouse.test"].Primary.Attributes["connection_uuid"] + connectionUuid := s.RootModule().Resources["montecarlo_bigquery_warehouse.test"].Primary.Attributes["credentials.connection_uuid"] return fmt.Sprintf("%[1]s,%[2]s,%[3]s", uuid, connectionUuid, collectorUuid), nil }, ImportStateVerifyIdentifierAttribute: "uuid", - ImportStateVerifyIgnore: []string{"deletion_protection", "service_account_key"}, + ImportStateVerifyIgnore: []string{"deletion_protection", "credentials.service_account_key"}, }, { // Update and Read testing ProtoV6ProviderFactories: acctest.TestAccProviderFactories, @@ -70,7 +70,7 @@ func TestAccBigQueryWarehouseResource(t *testing.T) { Check: resource.ComposeAggregateTestCheckFunc( resource.TestCheckResourceAttr("montecarlo_bigquery_warehouse.test", "name", "test-warehouse-updated"), resource.TestCheckResourceAttr("montecarlo_bigquery_warehouse.test", "collector_uuid", collectorUuid), - resource.TestCheckResourceAttr("montecarlo_bigquery_warehouse.test", "service_account_key", serviceAccount), + resource.TestCheckResourceAttr("montecarlo_bigquery_warehouse.test", "credentials.service_account_key", serviceAccount), resource.TestCheckResourceAttr("montecarlo_bigquery_warehouse.test", "deletion_protection", "false"), ), }, diff --git a/internal/warehouse/testdata/TestAccBigQueryWarehouseResource/create.tf b/internal/warehouse/testdata/TestAccBigQueryWarehouseResource/create.tf index 2569f14..a640009 100644 --- a/internal/warehouse/testdata/TestAccBigQueryWarehouseResource/create.tf +++ b/internal/warehouse/testdata/TestAccBigQueryWarehouseResource/create.tf @@ -20,6 +20,6 @@ variable "bq_service_account" { resource "montecarlo_bigquery_warehouse" "test" { name = "test-warehouse" collector_uuid = "a08d23fc-00a0-4c36-b568-82e9d0e67ad8" - service_account_key = var.bq_service_account + credentials = { service_account_key = var.bq_service_account } deletion_protection = false } diff --git a/internal/warehouse/testdata/TestAccBigQueryWarehouseResource/update.tf b/internal/warehouse/testdata/TestAccBigQueryWarehouseResource/update.tf index 2938164..1576393 100644 --- a/internal/warehouse/testdata/TestAccBigQueryWarehouseResource/update.tf +++ b/internal/warehouse/testdata/TestAccBigQueryWarehouseResource/update.tf @@ -20,6 +20,6 @@ variable "bq_service_account" { resource "montecarlo_bigquery_warehouse" "test" { name = "test-warehouse-updated" collector_uuid = "a08d23fc-00a0-4c36-b568-82e9d0e67ad8" - service_account_key = var.bq_service_account + credentials = { service_account_key = var.bq_service_account } deletion_protection = false }