Skip to content

Commit

Permalink
fix(warehouse): BQ credentials drift changes (#84)
Browse files Browse the repository at this point in the history 
* feat(warehouse): credentials drift changes

* feat(warehouse): BQ credentials drift detection finished

* fix(warehouse): BQ tests with new credentials structure
  • Loading branch information
@ndopj
ndopj authored Feb 18, 2024
1 parent 0488b11 commit fd7aad4
Show file tree
Hide file tree
Showing 5 changed files with 138 additions and 39 deletions.
16 changes: 10 additions & 6 deletions client/monte_carlo_client.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -83,6 +83,8 @@ type AddConnection struct {
AddConnection struct {
Connection struct {
Uuid string
CreatedOn string
UpdatedOn string
Warehouse struct {
Name string
Uuid string
Expand All @@ -95,8 +97,10 @@ type GetWarehouse struct {
GetWarehouse *struct {
Name string `json:"name"`
Connections []struct {
Uuid string `json:"uuid"`
Type string `json:"type"`
Uuid string `json:"uuid"`
Type string `json:"type"`
CreatedOn string `json:"createdOn"`
UpdatedOn string `json:"updatedOn"`
} `json:"connections"`
DataCollector struct {
Uuid string `json:"uuid"`
Expand All @@ -108,7 +112,7 @@ const BigQueryConnectionType = "bigquery"
const BigQueryConnectionTypeResponse = "BIGQUERY"
const TransactionalConnectionType = "transactional-db"
const TransactionalConnectionTypeResponse = "TRANSACTIONAL_DB"
const GetWarehouseQuery string = "query getWarehouse($uuid: UUID) { getWarehouse(uuid: $uuid) { name,connections{uuid,type},dataCollector{uuid} } }"
const GetWarehouseQuery string = "query getWarehouse($uuid: UUID) { getWarehouse(uuid: $uuid) { name,connections{uuid,type,createdOn,updatedOn},dataCollector{uuid} } }"

type RemoveConnection struct {
RemoveConnection struct {
Expand All @@ -127,7 +131,8 @@ type SetWarehouseName struct {

type UpdateCredentials struct {
UpdateCredentials struct {
Success bool
Success bool
UpdatedAt string
} `graphql:"updateCredentials(changes: $changes, connectionId: $connectionId, shouldReplace: $shouldReplace, shouldValidate: $shouldValidate)"`
}

Expand Down Expand Up @@ -299,15 +304,14 @@ type CreateOrUpdateComparisonRule struct {
Severity string
RuleType string
WarehouseUuid string
Comparisons []struct {
Comparisons []struct {
ComparisonType string
FullTableId string
FullTableIds []string
Field string
Metric string
Operator string
Threshold float64

}
}
} `graphql:"createOrUpdateComparisonRule(comparisons: $comparisons, customRuleUuid: $customRuleUuid, description: $description, queryResultType: $queryResultType, scheduleConfig: $scheduleConfig, sourceConnectionId: $sourceConnectionId, sourceDwId: $sourceDwId, sourceSqlQuery: $sourceSqlQuery, targetConnectionId: $targetConnectionId, targetDwId: $targetDwId, targetSqlQuery: $targetSqlQuery)"`
Expand Down
149 changes: 122 additions & 27 deletions internal/warehouse/bigquery_warehouse.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -38,6 +38,20 @@ type BigQueryWarehouseResource struct {

// BigQueryWarehouseResourceModel describes the resource data model according to its Schema.
type BigQueryWarehouseResourceModel struct {
Uuid types.String `tfsdk:"uuid"`
Credentials Credentials `tfsdk:"credentials"`
Name types.String `tfsdk:"name"`
CollectorUuid types.String `tfsdk:"collector_uuid"`
DeletionProtection types.Bool `tfsdk:"deletion_protection"`
}

type Credentials struct {
ConnectionUuid types.String `tfsdk:"connection_uuid"`
ServiceAccountKey types.String `tfsdk:"service_account_key"`
UpdatedAt types.String `tfsdk:"updated_at"`
}

type BigQueryWarehouseResourceModelV1 struct {
Uuid types.String `tfsdk:"uuid"`
ConnectionUuid types.String `tfsdk:"connection_uuid"`
Name types.String `tfsdk:"name"`
Expand Down Expand Up @@ -70,11 +84,24 @@ func (r *BigQueryWarehouseResource) Schema(ctx context.Context, req resource.Sch
stringplanmodifier.UseStateForUnknown(),
},
},
"connection_uuid": schema.StringAttribute{
Computed: true,
Optional: false,
PlanModifiers: []planmodifier.String{
stringplanmodifier.UseStateForUnknown(),
"credentials": schema.SingleNestedAttribute{
Required: true,
Attributes: map[string]schema.Attribute{
"connection_uuid": schema.StringAttribute{
Computed: true,
Optional: false,
PlanModifiers: []planmodifier.String{
stringplanmodifier.UseStateForUnknown(),
},
},
"service_account_key": schema.StringAttribute{
Required: true,
Sensitive: true,
},
"updated_at": schema.StringAttribute{
Computed: true,
Optional: false,
},
},
},
"name": schema.StringAttribute{
Expand All @@ -87,10 +114,6 @@ func (r *BigQueryWarehouseResource) Schema(ctx context.Context, req resource.Sch
stringplanmodifier.RequiresReplaceIfConfigured(),
},
},
"service_account_key": schema.StringAttribute{
Required: true,
Sensitive: true,
},
"deletion_protection": schema.BoolAttribute{
Optional: true,
Computed: true,
Expand Down Expand Up @@ -120,8 +143,8 @@ func (r *BigQueryWarehouseResource) Create(ctx context.Context, req resource.Cre
}

data.Uuid = result.Uuid
data.ConnectionUuid = result.ConnectionUuid
data.Name = result.Name
data.Credentials.UpdatedAt = result.Credentials.UpdatedAt
data.Credentials.ConnectionUuid = result.Credentials.ConnectionUuid
resp.Diagnostics.Append(resp.State.Set(ctx, &data)...)
}

Expand Down Expand Up @@ -167,23 +190,35 @@ func (r *BigQueryWarehouseResource) Read(ctx context.Context, req resource.ReadR
}

readConnectionUuid := types.StringNull()
readServiceAccountKey := types.StringNull()
readConnectionSAKey := types.StringNull()
readConnectionUpdatedAt := types.StringNull()

for _, connection := range getResult.GetWarehouse.Connections {
if connection.Uuid == data.ConnectionUuid.ValueString() {
if connection.Uuid == data.Credentials.ConnectionUuid.ValueString() {
if connection.Type != client.BigQueryConnectionTypeResponse {
resp.Diagnostics.AddError(
fmt.Sprintf("Obtained Warehouse [uuid: %s, connection_uuid: %s] but got unexpected connection "+
"type '%s'.", data.Uuid.ValueString(), connection.Uuid, connection.Type),
"Users can manually fix remote state or delete this resource from the Terraform configuration.")
return
}
readConnectionUuid = data.ConnectionUuid
readServiceAccountKey = data.ServiceAccountKey

readConnectionUuid = data.Credentials.ConnectionUuid
readConnectionSAKey = data.Credentials.ServiceAccountKey
readConnectionUpdatedAt = types.StringValue(connection.UpdatedOn)
if connection.UpdatedOn == "" {
readConnectionUpdatedAt = types.StringValue(connection.CreatedOn)
}
}
}

data.ConnectionUuid = readConnectionUuid
data.ServiceAccountKey = readServiceAccountKey
if !readConnectionSAKey.IsNull() && !readConnectionUpdatedAt.Equal(data.Credentials.UpdatedAt) {
readConnectionSAKey = types.StringValue("(unknown external value)")
}

data.Credentials.UpdatedAt = readConnectionUpdatedAt
data.Credentials.ConnectionUuid = readConnectionUuid
data.Credentials.ServiceAccountKey = readConnectionSAKey
data.Name = types.StringValue(getResult.GetWarehouse.Name)
resp.Diagnostics.Append(resp.State.Set(ctx, &data)...)
}
Expand All @@ -207,10 +242,11 @@ func (r *BigQueryWarehouseResource) Update(ctx context.Context, req resource.Upd
return
}

if data.ConnectionUuid.IsUnknown() || data.ConnectionUuid.IsNull() {
if data.Credentials.ConnectionUuid.IsUnknown() || data.Credentials.ConnectionUuid.IsNull() {
if result, diags := r.addConnection(ctx, data); result != nil {
resp.Diagnostics.Append(diags...)
data.ConnectionUuid = result.ConnectionUuid
data.Credentials.UpdatedAt = result.Credentials.UpdatedAt
data.Credentials.ConnectionUuid = result.Credentials.ConnectionUuid
} else {
resp.Diagnostics.Append(diags...)
return
Expand All @@ -219,8 +255,8 @@ func (r *BigQueryWarehouseResource) Update(ctx context.Context, req resource.Upd

updateResult := client.UpdateCredentials{}
variables = map[string]interface{}{
"changes": client.JSONString(data.ServiceAccountKey.ValueString()),
"connectionId": client.UUID(data.ConnectionUuid.ValueString()),
"changes": client.JSONString(data.Credentials.ServiceAccountKey.ValueString()),
"connectionId": client.UUID(data.Credentials.ConnectionUuid.ValueString()),
"shouldReplace": true,
"shouldValidate": true,
}
Expand All @@ -235,6 +271,8 @@ func (r *BigQueryWarehouseResource) Update(ctx context.Context, req resource.Upd
resp.Diagnostics.AddError(toPrint, "")
return
}

data.Credentials.UpdatedAt = types.StringValue(updateResult.UpdateCredentials.UpdatedAt)
resp.Diagnostics.Append(resp.State.Set(ctx, &data)...)
}

Expand All @@ -256,7 +294,7 @@ func (r *BigQueryWarehouseResource) Delete(ctx context.Context, req resource.Del
}

removeResult := client.RemoveConnection{}
variables := map[string]interface{}{"connectionId": client.UUID(data.ConnectionUuid.ValueString())}
variables := map[string]interface{}{"connectionId": client.UUID(data.Credentials.ConnectionUuid.ValueString())}
if err := r.client.Mutate(ctx, &removeResult, variables); err != nil {
toPrint := fmt.Sprintf("MC client 'RemoveConnection' mutation result - %s", err.Error())
resp.Diagnostics.AddError(toPrint, "")
Expand All @@ -272,7 +310,7 @@ func (r *BigQueryWarehouseResource) ImportState(ctx context.Context, req resourc
idsImported := strings.Split(req.ID, ",")
if len(idsImported) == 3 && idsImported[0] != "" && idsImported[1] != "" && idsImported[2] != "" {
resp.Diagnostics.Append(resp.State.SetAttribute(ctx, path.Root("uuid"), idsImported[0])...)
resp.Diagnostics.Append(resp.State.SetAttribute(ctx, path.Root("connection_uuid"), idsImported[1])...)
resp.Diagnostics.Append(resp.State.SetAttribute(ctx, path.Root("credentials").AtName("connection_uuid"), idsImported[1])...)
resp.Diagnostics.Append(resp.State.SetAttribute(ctx, path.Root("collector_uuid"), idsImported[2])...)
} else {
resp.Diagnostics.AddError("Unexpected Import Identifier", fmt.Sprintf(
Expand All @@ -289,7 +327,7 @@ func (r *BigQueryWarehouseResource) addConnection(ctx context.Context, data BigQ
"validationName": "save_credentials",
"connectionDetails": BqConnectionDetails{
"serviceJson": b64.StdEncoding.EncodeToString(
[]byte(data.ServiceAccountKey.ValueString()),
[]byte(data.Credentials.ServiceAccountKey.ValueString()),
),
},
}
Expand Down Expand Up @@ -334,8 +372,8 @@ func (r *BigQueryWarehouseResource) addConnection(ctx context.Context, data BigQ
}

data.Uuid = types.StringValue(addResult.AddConnection.Connection.Warehouse.Uuid)
data.ConnectionUuid = types.StringValue(addResult.AddConnection.Connection.Uuid)
data.Name = types.StringValue(addResult.AddConnection.Connection.Warehouse.Name)
data.Credentials.UpdatedAt = types.StringValue(addResult.AddConnection.Connection.CreatedOn)
data.Credentials.ConnectionUuid = types.StringValue(addResult.AddConnection.Connection.Uuid)
return &data, diagsResult
}

Expand Down Expand Up @@ -398,7 +436,7 @@ func (r *BigQueryWarehouseResource) UpgradeState(ctx context.Context) map[int64]
var priorStateData BigQueryWarehouseResourceModelV0
resp.Diagnostics.Append(req.State.Get(ctx, &priorStateData)...)
if !resp.Diagnostics.HasError() {
upgradedStateData := BigQueryWarehouseResourceModel{
upgradedStateData := BigQueryWarehouseResourceModelV1{
Uuid: priorStateData.Uuid,
ConnectionUuid: priorStateData.ConnectionUuid,
CollectorUuid: priorStateData.DataCollectorUuid,
Expand All @@ -410,5 +448,62 @@ func (r *BigQueryWarehouseResource) UpgradeState(ctx context.Context) map[int64]
}
},
},
1: {
PriorSchema: &schema.Schema{
Attributes: map[string]schema.Attribute{
"uuid": schema.StringAttribute{
Computed: true,
Optional: false,
PlanModifiers: []planmodifier.String{
stringplanmodifier.UseStateForUnknown(),
},
},
"connection_uuid": schema.StringAttribute{
Computed: true,
Optional: false,
PlanModifiers: []planmodifier.String{
stringplanmodifier.UseStateForUnknown(),
},
},
"name": schema.StringAttribute{
Required: true,
Validators: []validator.String{stringvalidator.LengthAtLeast(1)},
},
"collector_uuid": schema.StringAttribute{
Required: true,
PlanModifiers: []planmodifier.String{
stringplanmodifier.RequiresReplaceIfConfigured(),
},
},
"service_account_key": schema.StringAttribute{
Required: true,
Sensitive: true,
},
"deletion_protection": schema.BoolAttribute{
Optional: true,
Computed: true,
Default: booldefault.StaticBool(true),
},
},
},
StateUpgrader: func(ctx context.Context, req resource.UpgradeStateRequest, resp *resource.UpgradeStateResponse) {
var priorStateData BigQueryWarehouseResourceModelV1
resp.Diagnostics.Append(req.State.Get(ctx, &priorStateData)...)
if !resp.Diagnostics.HasError() {
upgradedStateData := BigQueryWarehouseResourceModel{
Uuid: priorStateData.Uuid,
CollectorUuid: priorStateData.CollectorUuid,
Name: priorStateData.Name,
DeletionProtection: priorStateData.DeletionProtection,
Credentials: Credentials{
ConnectionUuid: priorStateData.ConnectionUuid,
ServiceAccountKey: priorStateData.ServiceAccountKey,
UpdatedAt: types.StringNull(),
},
}
resp.Diagnostics.Append(resp.State.Set(ctx, upgradedStateData)...)
}
},
},
}
}
8 changes: 4 additions & 4 deletions internal/warehouse/bigquery_warehouse_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -37,7 +37,7 @@ func TestAccBigQueryWarehouseResource(t *testing.T) {
Check: resource.ComposeAggregateTestCheckFunc(
resource.TestCheckResourceAttr("montecarlo_bigquery_warehouse.test", "name", "test-warehouse"),
resource.TestCheckResourceAttr("montecarlo_bigquery_warehouse.test", "collector_uuid", collectorUuid),
resource.TestCheckResourceAttr("montecarlo_bigquery_warehouse.test", "service_account_key", serviceAccount),
resource.TestCheckResourceAttr("montecarlo_bigquery_warehouse.test", "credentials.service_account_key", serviceAccount),
resource.TestCheckResourceAttr("montecarlo_bigquery_warehouse.test", "deletion_protection", "false"),
),
},
Expand All @@ -53,11 +53,11 @@ func TestAccBigQueryWarehouseResource(t *testing.T) {
ImportStateVerify: true,
ImportStateIdFunc: func(s *terraform.State) (string, error) {
uuid := s.RootModule().Resources["montecarlo_bigquery_warehouse.test"].Primary.Attributes["uuid"]
connectionUuid := s.RootModule().Resources["montecarlo_bigquery_warehouse.test"].Primary.Attributes["connection_uuid"]
connectionUuid := s.RootModule().Resources["montecarlo_bigquery_warehouse.test"].Primary.Attributes["credentials.connection_uuid"]
return fmt.Sprintf("%[1]s,%[2]s,%[3]s", uuid, connectionUuid, collectorUuid), nil
},
ImportStateVerifyIdentifierAttribute: "uuid",
ImportStateVerifyIgnore: []string{"deletion_protection", "service_account_key"},
ImportStateVerifyIgnore: []string{"deletion_protection", "credentials.service_account_key"},
},
{ // Update and Read testing
ProtoV6ProviderFactories: acctest.TestAccProviderFactories,
Expand All @@ -70,7 +70,7 @@ func TestAccBigQueryWarehouseResource(t *testing.T) {
Check: resource.ComposeAggregateTestCheckFunc(
resource.TestCheckResourceAttr("montecarlo_bigquery_warehouse.test", "name", "test-warehouse-updated"),
resource.TestCheckResourceAttr("montecarlo_bigquery_warehouse.test", "collector_uuid", collectorUuid),
resource.TestCheckResourceAttr("montecarlo_bigquery_warehouse.test", "service_account_key", serviceAccount),
resource.TestCheckResourceAttr("montecarlo_bigquery_warehouse.test", "credentials.service_account_key", serviceAccount),
resource.TestCheckResourceAttr("montecarlo_bigquery_warehouse.test", "deletion_protection", "false"),
),
},
Expand Down
2 changes: 1 addition & 1 deletion internal/warehouse/testdata/TestAccBigQueryWarehouseResource/create.tf
View file
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,6 @@ variable "bq_service_account" {
resource "montecarlo_bigquery_warehouse" "test" {
name = "test-warehouse"
collector_uuid = "a08d23fc-00a0-4c36-b568-82e9d0e67ad8"
service_account_key = var.bq_service_account
credentials = { service_account_key = var.bq_service_account }
deletion_protection = false
}
2 changes: 1 addition & 1 deletion internal/warehouse/testdata/TestAccBigQueryWarehouseResource/update.tf
View file
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,6 @@ variable "bq_service_account" {
resource "montecarlo_bigquery_warehouse" "test" {
name = "test-warehouse-updated"
collector_uuid = "a08d23fc-00a0-4c36-b568-82e9d0e67ad8"
service_account_key = var.bq_service_account
credentials = { service_account_key = var.bq_service_account }
deletion_protection = false
}

0 comments on commit fd7aad4

Please sign in to comment.