Skip to content

Commit

Permalink
[Exporter] add support for databricks_credential (#4292)
Browse files Browse the repository at this point in the history
## Changes
<!-- Summary of your changes that are easy to understand -->

## Tests
<!-- 
How is this tested? Please see the checklist below and also describe any
other relevant tests
-->

- [x] `make test` run locally
- [x] relevant change in `docs/` folder
- [ ] covered with integration tests in `internal/acceptance`
- [ ] relevant acceptance tests are passing
- [x] using Go SDK
  • Loading branch information
alexott authored Dec 14, 2024
1 parent 18d13bf commit d8a27fa
Show file tree
Hide file tree
Showing 6 changed files with 111 additions and 25 deletions.
2 changes: 2 additions & 0 deletions docs/guides/experimental-exporter.md
Original file line number Diff line number Diff line change
Expand Up @@ -137,6 +137,7 @@ Services are just logical groups of resources used for filtering and organizatio
* `uc-artifact-allowlist` - **listing** exports [databricks_artifact_allowlist](../resources/artifact_allowlist.md) resources for Unity Catalog Allow Lists attached to the current metastore.
* `uc-catalogs` - **listing** [databricks_catalog](../resources/catalog.md) and [databricks_workspace_binding](../resources/workspace_binding.md)
* `uc-connections` - **listing** [databricks_connection](../resources/connection.md). *Please note that because API doesn't return sensitive fields, such as, passwords, tokens, ..., the generated `options` block could be incomplete!*
* `uc-credentials` - **listing** exports [databricks_credential](../resources/credential.md) resources on workspace or account level. *Please note that it will skip storage credentials! Use the `uc-storage-credentials` service for them*
* `uc-external-locations` - **listing** exports [databricks_external_location](../resources/external_location.md) resource.
* `uc-grants` - [databricks_grants](../resources/grants.md). *Please note that during export the list of grants is expanded to include the identity that does the export! This is done to allow to creation of objects in case when catalogs/schemas have different owners than the current identity.*.
* `uc-metastores` - **listing** [databricks_metastore](../resources/metastore.md) and [databricks_metastore_assignment](../resource/metastore_assignment.md) (only on account-level). *Please note that when using workspace-level configuration, only the metastores from the workspace's region are listed!*
Expand Down Expand Up @@ -179,6 +180,7 @@ Exporter aims to generate HCL code for most of the resources within the Databric
| [databricks_cluster](../resources/cluster.md) | Yes | No | Yes | No |
| [databricks_cluster_policy](../resources/cluster_policy.md) | Yes | No | Yes | No |
| [databricks_connection](../resources/connection.md) | Yes | Yes | Yes | No |
| [databricks_credential](../resources/credential.md) | Yes | Yes | Yes | No |
| [databricks_dashboard](../resources/dashboard.md) | Yes | No | Yes | No |
| [databricks_dbfs_file](../resources/dbfs_file.md) | Yes | No | Yes | No |
| [databricks_external_location](../resources/external_location.md) | Yes | Yes | Yes | No |
Expand Down
24 changes: 12 additions & 12 deletions exporter/context.go
Original file line number Diff line number Diff line change
Expand Up @@ -494,20 +494,20 @@ func (ic *importContext) Run() error {
// nolint
dcfile.WriteString(
`terraform {
required_providers {
databricks = {
source = "databricks/databricks"
version = "` + common.Version() + `"
}
}
}
required_providers {
databricks = {
source = "databricks/databricks"
version = "` + common.Version() + `"
}
}
}
provider "databricks" {
`)
provider "databricks" {
`)
if ic.accountLevel {
dcfile.WriteString(fmt.Sprintf(` host = "%s"
account_id = "%s"
`, ic.Client.Config.Host, ic.Client.Config.AccountID))
dcfile.WriteString(fmt.Sprintf(` host = "%s"
account_id = "%s"
`, ic.Client.Config.Host, ic.Client.Config.AccountID))
}
dcfile.WriteString(`}`)
dcfile.Close()
Expand Down
15 changes: 12 additions & 3 deletions exporter/exporter_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -273,13 +273,20 @@ var emptyExternalLocations = qa.HTTPFixture{
Response: &catalog.ListExternalLocationsResponse{},
}

var emptyStorageCrdentials = qa.HTTPFixture{
var emptyStorageCredentials = qa.HTTPFixture{
Method: "GET",
Resource: "/api/2.1/unity-catalog/storage-credentials?",
Status: 200,
Response: &catalog.ListStorageCredentialsResponse{},
}

var emptyUcCredentials = qa.HTTPFixture{
Method: "GET",
Resource: "/api/2.1/unity-catalog/credentials?",
Status: 200,
Response: &catalog.ListCredentialsResponse{},
}

var emptyConnections = qa.HTTPFixture{
Method: "GET",
Resource: "/api/2.1/unity-catalog/connections?",
Expand Down Expand Up @@ -495,7 +502,8 @@ func TestImportingUsersGroupsSecretScopes(t *testing.T) {
emptyInstancePools,
emptyModelServing,
emptyExternalLocations,
emptyStorageCrdentials,
emptyStorageCredentials,
emptyUcCredentials,
emptyMlflowWebhooks,
emptySqlDashboards,
emptySqlEndpoints,
Expand Down Expand Up @@ -763,7 +771,8 @@ func TestImportingNoResourcesError(t *testing.T) {
emptyMetastoreList,
emptyRepos,
emptyExternalLocations,
emptyStorageCrdentials,
emptyStorageCredentials,
emptyUcCredentials,
emptyShares,
emptyConnections,
emptyRecipients,
Expand Down
50 changes: 40 additions & 10 deletions exporter/importables.go
Original file line number Diff line number Diff line change
Expand Up @@ -2987,12 +2987,47 @@ var resourcesMap map[string]importable = map[string]importable{
}
return nil
},
ShouldOmitField: func(ic *importContext, pathString string, as *schema.Schema, d *schema.ResourceData) bool {
if pathString == "isolation_mode" {
return d.Get(pathString).(string) != "ISOLATION_MODE_ISOLATED"
ShouldOmitField: shouldOmitWithIsolationMode,
Depends: []reference{
{Path: "azure_service_principal.client_secret", Variable: true},
},
},
"databricks_credential": {
WorkspaceLevel: true,
Service: "uc-credentials",
Import: func(ic *importContext, r *resource) error {
ic.emitUCGrantsWithOwner("credential/"+r.ID, r)
if r.Data != nil {
isolationMode := r.Data.Get("isolation_mode").(string)
if isolationMode == "ISOLATION_MODE_ISOLATED" {
purpose := r.Data.Get("purpose").(string)
if purpose == "SERVICE" {
ic.emitWorkspaceBindings("credential", r.ID)
} else if purpose == "STORAGE" {
ic.emitWorkspaceBindings("storage_credential", r.ID)
}
}
}
return shouldOmitForUnityCatalog(ic, pathString, as, d)
return nil
},
List: func(ic *importContext) error {
it := ic.workspaceClient.Credentials.ListCredentials(ic.Context, catalog.ListCredentialsRequest{})
for it.HasNext(ic.Context) {
v, err := it.Next(ic.Context)
if err != nil {
return err
}
if v.Purpose == catalog.CredentialPurposeStorage {
continue // we're handling storage credentials separately
}
ic.EmitIfUpdatedAfterMillisAndNameMatches(&resource{
Resource: "databricks_credential",
ID: v.Name,
}, v.Name, v.UpdatedAt, fmt.Sprintf("credential %s", v.Name))
}
return nil
},
ShouldOmitField: shouldOmitWithIsolationMode,
Depends: []reference{
{Path: "azure_service_principal.client_secret", Variable: true},
},
Expand Down Expand Up @@ -3032,12 +3067,7 @@ var resourcesMap map[string]importable = map[string]importable{
}
return nil
},
ShouldOmitField: func(ic *importContext, pathString string, as *schema.Schema, d *schema.ResourceData) bool {
if pathString == "isolation_mode" {
return d.Get(pathString).(string) != "ISOLATION_MODE_ISOLATED"
}
return shouldOmitForUnityCatalog(ic, pathString, as, d)
},
ShouldOmitField: shouldOmitWithIsolationMode,
// This external location is automatically created when metastore is created with the `storage_root`
Ignore: func(ic *importContext, r *resource) bool {
return r.ID == "metastore_default_location"
Expand Down
38 changes: 38 additions & 0 deletions exporter/importables_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -2017,6 +2017,44 @@ func TestListExternalLocations(t *testing.T) {
})
}

func TestServiceCredentials(t *testing.T) {
qa.HTTPFixturesApply(t, []qa.HTTPFixture{
{
ReuseRequest: true,
Method: "GET",
Resource: "/api/2.1/unity-catalog/credentials?",
Response: catalog.ListCredentialsResponse{
Credentials: []catalog.CredentialInfo{
{
Name: "test-storage",
Purpose: catalog.CredentialPurposeStorage,
},
{
Name: "test-service",
Purpose: catalog.CredentialPurposeService,
},
},
},
},
}, func(ctx context.Context, client *common.DatabricksClient) {
ic := importContextForTestWithClient(ctx, client)
ic.enableServices("uc-credentials,uc-grants")
ic.currentMetastore = currentMetastoreResponse
// Test listing
err := resourcesMap["databricks_credential"].List(ic)
assert.NoError(t, err)
require.Equal(t, 1, len(ic.testEmits))
assert.True(t, ic.testEmits["databricks_credential[<unknown>] (id: test-service)"])
// Test import
err = resourcesMap["databricks_credential"].Import(ic, &resource{
ID: "1234",
})
assert.NoError(t, err)
require.Equal(t, 2, len(ic.testEmits))
assert.True(t, ic.testEmits["databricks_grants[<unknown>] (id: credential/1234)"])
})
}

func TestStorageCredentials(t *testing.T) {
qa.HTTPFixturesApply(t, []qa.HTTPFixture{
{
Expand Down
7 changes: 7 additions & 0 deletions exporter/util.go
Original file line number Diff line number Diff line change
Expand Up @@ -436,6 +436,13 @@ func shouldOmitForUnityCatalog(ic *importContext, pathString string, as *schema.
return defaultShouldOmitFieldFunc(ic, pathString, as, d)
}

func shouldOmitWithIsolationMode(ic *importContext, pathString string, as *schema.Schema, d *schema.ResourceData) bool {
if pathString == "isolation_mode" {
return d.Get(pathString).(string) != "ISOLATION_MODE_ISOLATED"
}
return shouldOmitForUnityCatalog(ic, pathString, as, d)
}

func appendEndingSlashToDirName(dir string) string {
if dir == "" || dir[len(dir)-1] == '/' {
return dir
Expand Down

0 comments on commit d8a27fa

Please sign in to comment.