From d8a27fa0c4a5fb8abf9d01aaf628a4d55faf4030 Mon Sep 17 00:00:00 2001 From: Alex Ott Date: Sat, 14 Dec 2024 03:15:31 -0500 Subject: [PATCH] [Exporter] add support for `databricks_credential` (#4292) ## Changes ## Tests - [x] `make test` run locally - [x] relevant change in `docs/` folder - [ ] covered with integration tests in `internal/acceptance` - [ ] relevant acceptance tests are passing - [x] using Go SDK --- docs/guides/experimental-exporter.md | 2 ++ exporter/context.go | 24 ++++++------- exporter/exporter_test.go | 15 +++++++-- exporter/importables.go | 50 ++++++++++++++++++++++------ exporter/importables_test.go | 38 +++++++++++++++++++++ exporter/util.go | 7 ++++ 6 files changed, 111 insertions(+), 25 deletions(-) diff --git a/docs/guides/experimental-exporter.md b/docs/guides/experimental-exporter.md index b3ea7048a..b013bc7d9 100644 --- a/docs/guides/experimental-exporter.md +++ b/docs/guides/experimental-exporter.md @@ -137,6 +137,7 @@ Services are just logical groups of resources used for filtering and organizatio * `uc-artifact-allowlist` - **listing** exports [databricks_artifact_allowlist](../resources/artifact_allowlist.md) resources for Unity Catalog Allow Lists attached to the current metastore. * `uc-catalogs` - **listing** [databricks_catalog](../resources/catalog.md) and [databricks_workspace_binding](../resources/workspace_binding.md) * `uc-connections` - **listing** [databricks_connection](../resources/connection.md). *Please note that because API doesn't return sensitive fields, such as, passwords, tokens, ..., the generated `options` block could be incomplete!* +* `uc-credentials` - **listing** exports [databricks_credential](../resources/credential.md) resources on workspace or account level. *Please note that it will skip storage credentials! Use the `uc-storage-credentials` service for them* * `uc-external-locations` - **listing** exports [databricks_external_location](../resources/external_location.md) resource. * `uc-grants` - [databricks_grants](../resources/grants.md). *Please note that during export the list of grants is expanded to include the identity that does the export! This is done to allow to creation of objects in case when catalogs/schemas have different owners than the current identity.*. * `uc-metastores` - **listing** [databricks_metastore](../resources/metastore.md) and [databricks_metastore_assignment](../resource/metastore_assignment.md) (only on account-level). *Please note that when using workspace-level configuration, only the metastores from the workspace's region are listed!* @@ -179,6 +180,7 @@ Exporter aims to generate HCL code for most of the resources within the Databric | [databricks_cluster](../resources/cluster.md) | Yes | No | Yes | No | | [databricks_cluster_policy](../resources/cluster_policy.md) | Yes | No | Yes | No | | [databricks_connection](../resources/connection.md) | Yes | Yes | Yes | No | +| [databricks_credential](../resources/credential.md) | Yes | Yes | Yes | No | | [databricks_dashboard](../resources/dashboard.md) | Yes | No | Yes | No | | [databricks_dbfs_file](../resources/dbfs_file.md) | Yes | No | Yes | No | | [databricks_external_location](../resources/external_location.md) | Yes | Yes | Yes | No | diff --git a/exporter/context.go b/exporter/context.go index 86a8a080b..833090afc 100644 --- a/exporter/context.go +++ b/exporter/context.go @@ -494,20 +494,20 @@ func (ic *importContext) Run() error { // nolint dcfile.WriteString( `terraform { - required_providers { - databricks = { - source = "databricks/databricks" - version = "` + common.Version() + `" - } - } - } + required_providers { + databricks = { + source = "databricks/databricks" + version = "` + common.Version() + `" + } + } +} - provider "databricks" { - `) +provider "databricks" { +`) if ic.accountLevel { - dcfile.WriteString(fmt.Sprintf(` host = "%s" - account_id = "%s" - `, ic.Client.Config.Host, ic.Client.Config.AccountID)) + dcfile.WriteString(fmt.Sprintf(` host = "%s" + account_id = "%s" +`, ic.Client.Config.Host, ic.Client.Config.AccountID)) } dcfile.WriteString(`}`) dcfile.Close() diff --git a/exporter/exporter_test.go b/exporter/exporter_test.go index b8cbd53ac..a3bc50b27 100644 --- a/exporter/exporter_test.go +++ b/exporter/exporter_test.go @@ -273,13 +273,20 @@ var emptyExternalLocations = qa.HTTPFixture{ Response: &catalog.ListExternalLocationsResponse{}, } -var emptyStorageCrdentials = qa.HTTPFixture{ +var emptyStorageCredentials = qa.HTTPFixture{ Method: "GET", Resource: "/api/2.1/unity-catalog/storage-credentials?", Status: 200, Response: &catalog.ListStorageCredentialsResponse{}, } +var emptyUcCredentials = qa.HTTPFixture{ + Method: "GET", + Resource: "/api/2.1/unity-catalog/credentials?", + Status: 200, + Response: &catalog.ListCredentialsResponse{}, +} + var emptyConnections = qa.HTTPFixture{ Method: "GET", Resource: "/api/2.1/unity-catalog/connections?", @@ -495,7 +502,8 @@ func TestImportingUsersGroupsSecretScopes(t *testing.T) { emptyInstancePools, emptyModelServing, emptyExternalLocations, - emptyStorageCrdentials, + emptyStorageCredentials, + emptyUcCredentials, emptyMlflowWebhooks, emptySqlDashboards, emptySqlEndpoints, @@ -763,7 +771,8 @@ func TestImportingNoResourcesError(t *testing.T) { emptyMetastoreList, emptyRepos, emptyExternalLocations, - emptyStorageCrdentials, + emptyStorageCredentials, + emptyUcCredentials, emptyShares, emptyConnections, emptyRecipients, diff --git a/exporter/importables.go b/exporter/importables.go index 5465e1afa..058cb8629 100644 --- a/exporter/importables.go +++ b/exporter/importables.go @@ -2987,12 +2987,47 @@ var resourcesMap map[string]importable = map[string]importable{ } return nil }, - ShouldOmitField: func(ic *importContext, pathString string, as *schema.Schema, d *schema.ResourceData) bool { - if pathString == "isolation_mode" { - return d.Get(pathString).(string) != "ISOLATION_MODE_ISOLATED" + ShouldOmitField: shouldOmitWithIsolationMode, + Depends: []reference{ + {Path: "azure_service_principal.client_secret", Variable: true}, + }, + }, + "databricks_credential": { + WorkspaceLevel: true, + Service: "uc-credentials", + Import: func(ic *importContext, r *resource) error { + ic.emitUCGrantsWithOwner("credential/"+r.ID, r) + if r.Data != nil { + isolationMode := r.Data.Get("isolation_mode").(string) + if isolationMode == "ISOLATION_MODE_ISOLATED" { + purpose := r.Data.Get("purpose").(string) + if purpose == "SERVICE" { + ic.emitWorkspaceBindings("credential", r.ID) + } else if purpose == "STORAGE" { + ic.emitWorkspaceBindings("storage_credential", r.ID) + } + } } - return shouldOmitForUnityCatalog(ic, pathString, as, d) + return nil + }, + List: func(ic *importContext) error { + it := ic.workspaceClient.Credentials.ListCredentials(ic.Context, catalog.ListCredentialsRequest{}) + for it.HasNext(ic.Context) { + v, err := it.Next(ic.Context) + if err != nil { + return err + } + if v.Purpose == catalog.CredentialPurposeStorage { + continue // we're handling storage credentials separately + } + ic.EmitIfUpdatedAfterMillisAndNameMatches(&resource{ + Resource: "databricks_credential", + ID: v.Name, + }, v.Name, v.UpdatedAt, fmt.Sprintf("credential %s", v.Name)) + } + return nil }, + ShouldOmitField: shouldOmitWithIsolationMode, Depends: []reference{ {Path: "azure_service_principal.client_secret", Variable: true}, }, @@ -3032,12 +3067,7 @@ var resourcesMap map[string]importable = map[string]importable{ } return nil }, - ShouldOmitField: func(ic *importContext, pathString string, as *schema.Schema, d *schema.ResourceData) bool { - if pathString == "isolation_mode" { - return d.Get(pathString).(string) != "ISOLATION_MODE_ISOLATED" - } - return shouldOmitForUnityCatalog(ic, pathString, as, d) - }, + ShouldOmitField: shouldOmitWithIsolationMode, // This external location is automatically created when metastore is created with the `storage_root` Ignore: func(ic *importContext, r *resource) bool { return r.ID == "metastore_default_location" diff --git a/exporter/importables_test.go b/exporter/importables_test.go index e735da5d4..6306f859b 100644 --- a/exporter/importables_test.go +++ b/exporter/importables_test.go @@ -2017,6 +2017,44 @@ func TestListExternalLocations(t *testing.T) { }) } +func TestServiceCredentials(t *testing.T) { + qa.HTTPFixturesApply(t, []qa.HTTPFixture{ + { + ReuseRequest: true, + Method: "GET", + Resource: "/api/2.1/unity-catalog/credentials?", + Response: catalog.ListCredentialsResponse{ + Credentials: []catalog.CredentialInfo{ + { + Name: "test-storage", + Purpose: catalog.CredentialPurposeStorage, + }, + { + Name: "test-service", + Purpose: catalog.CredentialPurposeService, + }, + }, + }, + }, + }, func(ctx context.Context, client *common.DatabricksClient) { + ic := importContextForTestWithClient(ctx, client) + ic.enableServices("uc-credentials,uc-grants") + ic.currentMetastore = currentMetastoreResponse + // Test listing + err := resourcesMap["databricks_credential"].List(ic) + assert.NoError(t, err) + require.Equal(t, 1, len(ic.testEmits)) + assert.True(t, ic.testEmits["databricks_credential[] (id: test-service)"]) + // Test import + err = resourcesMap["databricks_credential"].Import(ic, &resource{ + ID: "1234", + }) + assert.NoError(t, err) + require.Equal(t, 2, len(ic.testEmits)) + assert.True(t, ic.testEmits["databricks_grants[] (id: credential/1234)"]) + }) +} + func TestStorageCredentials(t *testing.T) { qa.HTTPFixturesApply(t, []qa.HTTPFixture{ { diff --git a/exporter/util.go b/exporter/util.go index 44e7d523a..ccc5862ca 100644 --- a/exporter/util.go +++ b/exporter/util.go @@ -436,6 +436,13 @@ func shouldOmitForUnityCatalog(ic *importContext, pathString string, as *schema. return defaultShouldOmitFieldFunc(ic, pathString, as, d) } +func shouldOmitWithIsolationMode(ic *importContext, pathString string, as *schema.Schema, d *schema.ResourceData) bool { + if pathString == "isolation_mode" { + return d.Get(pathString).(string) != "ISOLATION_MODE_ISOLATED" + } + return shouldOmitForUnityCatalog(ic, pathString, as, d) +} + func appendEndingSlashToDirName(dir string) string { if dir == "" || dir[len(dir)-1] == '/' { return dir