Merge branch 'master' into generic-settings-resource

.github/ISSUE_TEMPLATE/provider-issue.md

@@ -37,3 +37,6 @@ Thank you for opening an issue. Please note that we try to keep the Databricks P
 
 ### Important Factoids
 <!-- Are there anything atypical about your accounts that we should know? -->
+
+### Would you like to implement a fix?
+<!-- If you plan to implement a fix for this, let the maintainers and community know -->

.github/workflows/push.yml

@@ -19,7 +19,7 @@ jobs:
       - name: Set up Go
         uses: actions/setup-go@v1
         with:
-          go-version: 1.20.x
+          go-version: 1.21.x
 
       - name: Cache packages
         uses: actions/cache@v2
@@ -57,7 +57,7 @@ jobs:
       - name: Setup Go
         uses: actions/setup-go@v3
         with:
-          go-version: 1.20.x
+          go-version: 1.21.x
 
           # No need to download cached dependencies when running gofmt.
           cache: false

.github/workflows/release.yml

@@ -19,7 +19,7 @@ jobs:
         name: Set up Go
         uses: actions/setup-go@v1
         with:
-          go-version: 1.20.x
+          go-version: 1.21.x
       - 
         name: Import GPG key
         id: import_gpg

.github/workflows/schema.yml

@@ -7,139 +7,54 @@ on:
   workflow_dispatch:
     inputs:
       base:
-        description: 'Base ref'
-        default: 'master'
+        description: "Base ref"
+        default: "master"
         required: true
       head:
-        description: 'Head ref'
-        default: 'master'
+        description: "Head ref"
+        default: "master"
         required: true
 
 jobs:
-  compute_current:
-    name: "Generate current"
+  compute_diff:
     runs-on: ubuntu-latest
 
     steps:
+      # Check out the base commit first and the head commit second. diff-schema
+      # uses the current commit as the head commit.
       - if: github.event_name == 'pull_request'
-        name: Checkout
+        name: Checkout PR base commit
         uses: actions/checkout@v4
         with:
-          # Checkout main branch to generate schema for current release
-          ref: master
+          ref: ${{ github.base_ref }}
+      - if: github.event_name == 'pull_request'
+        name: Checkout PR merge commit
+        uses: actions/checkout@v4
 
       - if: github.event_name == 'workflow_dispatch'
-        name: Checkout
+        name: Checkout base commit
         uses: actions/checkout@v4
         with:
           ref: ${{ github.event.inputs.base }}
-
-      - name: 'Setup Go'
-        uses: actions/setup-go@v4
-        with:
-          go-version: 1.21.x
-
-      - name: 'Setup Terraform'
-        uses: hashicorp/setup-terraform@v2
-        with:
-          terraform_wrapper: false
-
-      - run: make install
-
-      - name: Generate provider schema
-        shell: bash
-        run: |
-          set -ex
-          cd /tmp
-          cat > main.tf <<EOF
-          terraform {
-            required_providers {
-              databricks = {
-                source = "databricks/databricks"
-              }
-            }
-          }
-          EOF
-          terraform init
-          terraform providers schema -json > provider.json
-
-      - name: 'Upload provider schema'
-        uses: actions/upload-artifact@v3
-        with:
-          name: schema-current
-          path: /tmp/provider.json
-          retention-days: 1
-
-  compute_new:
-    name: "Generate new"
-    runs-on: ubuntu-latest
-
-    steps:
-      - if: github.event_name == 'pull_request'
-        name: Checkout
-        uses: actions/checkout@v4
-
       - if: github.event_name == 'workflow_dispatch'
-        name: Checkout
+        name: Checkout head commit
         uses: actions/checkout@v4
         with:
           ref: ${{ github.event.inputs.head }}
 
-      - name: 'Setup Go'
+      - name: "Setup Go"
         uses: actions/setup-go@v4
         with:
           go-version: 1.21.x
 
-      - name: 'Setup Terraform'
+      - name: "Setup Terraform"
         uses: hashicorp/setup-terraform@v2
         with:
           terraform_wrapper: false
 
-      - run: make install
-
-      - name: Generate provider schema
-        shell: bash
-        run: |
-          set -ex
-          cd /tmp
-          cat > main.tf <<EOF
-          terraform {
-            required_providers {
-              databricks = {
-                source = "databricks/databricks"
-              }
-            }
-          }
-          EOF
-          terraform init
-          terraform providers schema -json > provider.json
-
-      - name: 'Upload provider schema'
-        uses: actions/upload-artifact@v3
-        with:
-          name: schema-new
-          path: /tmp/provider.json
-          retention-days: 1
-
-  diff:
-    needs: [compute_current, compute_new]
-
-    name: "Compute diff"
-    runs-on: ubuntu-latest
-
-    steps:
-      - name: 'Setup Go'
-        uses: actions/setup-go@v4
-        with:
-          go-version: 1.21.x
-          cache: false
-
-      - run: go install github.com/josephburnett/jd@latest
-
-      - name: 'Download provider schemas'
-        uses: actions/download-artifact@v3
-
-      - run: ls -l schema*/*
+      - name: "Install jd"
+        run: go install github.com/josephburnett/jd@latest
 
-      - run: jd -color schema-current/provider.json schema-new/provider.json
-        continue-on-error: true
+      - run: make diff-schema
+        env:
+          BASE_COMMIT: ${{ github.event_name == 'pull_request' && github.base_ref || github.event.inputs.base }}

CHANGELOG.md

@@ -1,5 +1,89 @@
 # Version changelog
 
+## 1.34.0
+
+### New Features and Improvements:
+ * Added `workspace_path` to the [databricks_directory](https://registry.terraform.io/providers/databricks/databricks/latest/docs/data-sources/directory) data source ([#3051](https://github.com/databricks/terraform-provider-databricks/pull/3051)).
+ * Added resource [databricks_grant](https://registry.terraform.io/providers/databricks/databricks/latest/docs/resources/grant) for managing singular principal ([#3024](https://github.com/databricks/terraform-provider-databricks/pull/3024)).
+ * Added the [databricks_current_metastore](https://registry.terraform.io/providers/databricks/databricks/latest/docs/data-sources/current_metastore) data source to retrieve information about a UC metastore attached to the current workspace ([#3012](https://github.com/databricks/terraform-provider-databricks/pull/3012)).
+ * Added search by `display_name` to [databricks_service_principal](https://registry.terraform.io/providers/databricks/databricks/latest/docs/data-sources/service_principal) data source ([#2963](https://github.com/databricks/terraform-provider-databricks/pull/2963)).
+ * Fixed Read operation of [databricks_permission_assignment](https://registry.terraform.io/providers/databricks/databricks/latest/docs/resources/permission_assignment) resource ([#3066](https://github.com/databricks/terraform-provider-databricks/pull/3066)).
+ * Sort based on the Task Key specified in the 'Depends On' field ([#3000](https://github.com/databricks/terraform-provider-databricks/pull/3000)).
+ * Stop using `/api/2.0/preview/accounts/` API in [databricks_mws_permission_assignment](https://registry.terraform.io/providers/databricks/databricks/latest/docs/resources/mws_permission_assignment) ([#3062](https://github.com/databricks/terraform-provider-databricks/pull/3062)).
+ * Added `skip_validation` to [databricks_storage_credential](https://registry.terraform.io/providers/databricks/databricks/latest/docs/resources/storage_credential) ([#3087](https://github.com/databricks/terraform-provider-databricks/pull/3087)).
+
+### Documentation Changes:
+ * Expand troubleshooting guide with documentation about private link problems ([#3064](https://github.com/databricks/terraform-provider-databricks/pull/3064)).
+ * Add a note about workspace or account-level provider usage ([#3074](https://github.com/databricks/terraform-provider-databricks/pull/3074)).
+ * Use tasks in the [databricks_job](https://registry.terraform.io/providers/databricks/databricks/latest/docs/resources/job) examples ([#3097](https://github.com/databricks/terraform-provider-databricks/pull/3097)).
+
+### Exporter:
+ * Generate zero values for required attributes & attributes with non-zero defaults ([#3068](https://github.com/databricks/terraform-provider-databricks/pull/3068)).
+ * Improve handling of referenced objects & dependencies in [databricks_job](https://registry.terraform.io/providers/databricks/databricks/latest/docs/resources/job) ([#3082](https://github.com/databricks/terraform-provider-databricks/pull/3082)).
+ * Add emitting of libraries that are workspace files ([#3006](https://github.com/databricks/terraform-provider-databricks/pull/3006)).
+ * Add support for [databricks_artifact_allowlist](https://registry.terraform.io/providers/databricks/databricks/latest/docs/resources/artifact_allowlist) ([#3083](https://github.com/databricks/terraform-provider-databricks/pull/3083)).
+ * Added export of [databricks_system_schema](https://registry.terraform.io/providers/databricks/databricks/latest/docs/resources/system_schema) resources ([#3072](https://github.com/databricks/terraform-provider-databricks/pull/3072)).
+ * Generate cluster libraries blocks directly in [databricks_cluster](https://registry.terraform.io/providers/databricks/databricks/latest/docs/resources/cluster), don't generate [databricks_library](https://registry.terraform.io/providers/databricks/databricks/latest/docs/resources/library) resources ([#2983](https://github.com/databricks/terraform-provider-databricks/pull/2983)).
+ * Increase coverage for [databricks_mount](https://registry.terraform.io/providers/databricks/databricks/latest/docs/resources/mount) resource generation ([#2982](https://github.com/databricks/terraform-provider-databricks/pull/2982)).
+ * Fixed some broken links ([#3061](https://github.com/databricks/terraform-provider-databricks/pull/3061)).
+
+### Internal Changes:
+ * Added an item to check if the ticket opener wants to do a bug fix ([#3020](https://github.com/databricks/terraform-provider-databricks/pull/3020)).
+ * Updated Go SDK to v0.29.0 ([#3098](https://github.com/databricks/terraform-provider-databricks/pull/3098)).
+ * Migrated SQL Warehouse to Go SDK ([#3044](https://github.com/databricks/terraform-provider-databricks/pull/3044)).
+ * Fixed `TestClustersDataSourceErrorsOut` test that run too long ([#3073](https://github.com/databricks/terraform-provider-databricks/pull/3073)).
+ * Fixed TestAccServicePrinicpalHomeDeleteNotDeleted ([#3075](https://github.com/databricks/terraform-provider-databricks/pull/3075)).
+ * Fixed diff schema on PRs ([#3071](https://github.com/databricks/terraform-provider-databricks/pull/3071)).
+ * Used diffSuppressor instead of makeEmptyBlockSuppressFunc ([#3099](https://github.com/databricks/terraform-provider-databricks/pull/3099)).
+ * Force creation of home folder if not created ([#3052](https://github.com/databricks/terraform-provider-databricks/pull/3052)).
+
+
+## 1.33.0
+New Features and Improvements:
+ * Added support for ownership changes for unity catalog resources ([#3029](https://github.com/databricks/terraform-provider-databricks/pull/3029)).
+
+Exporter:
+ * Ignore workspace assets of deleted users and service principals ([#2980](https://github.com/databricks/terraform-provider-databricks/pull/2980)).
+
+## 1.32.0
+New Features and Improvements:
+ * Improved debugging of errors ([#744](https://github.com/databricks/databricks-sdk-go/pull/744))
+ * Removed requirement to provide accountId in mws_credentials resource ([#3028](https://github.com/databricks/terraform-provider-databricks/pull/3028)).
+ * Validated metastore_id field in worspace UC resources except metastore resource ([#3035](https://github.com/databricks/terraform-provider-databricks/pull/3035)), ([#3039](https://github.com/databricks/terraform-provider-databricks/pull/3039)).
+ * Wait for resource to be updated during grants resource modification ([#3026](https://github.com/databricks/terraform-provider-databricks/pull/3026)).
+ * Deprecated top level task attributes ([#2993](https://github.com/databricks/terraform-provider-databricks/pull/2993)).
+
+Documentation Changes:
+ * Added BQ example for [databricks_connection](https://registry.terraform.io/providers/databricks/databricks/latest/docs/resources/connection) ([#3040](https://github.com/databricks/terraform-provider-databricks/pull/3040)).
+ * Consistent header levels and same doc structure ([#3007](https://github.com/databricks/terraform-provider-databricks/pull/3007)).
+ * Fixed two minor docs typos in node_type and cluster_policy ([#3038](https://github.com/databricks/terraform-provider-databricks/pull/3038)).
+
+Exporter:
+ * Removed dead code that reflects old Jobs API 2.0 structures ([#2981](https://github.com/databricks/terraform-provider-databricks/pull/2981)).
+
+ Internal Changes:
+ * Bumped Go SDK to 0.28.1 ([#3043](https://github.com/databricks/terraform-provider-databricks/pull/3043)).
+ * Bumped github.com/hashicorp/terraform-plugin-sdk/v2 from 2.30.0 to 2.31.0 ([#3033](https://github.com/databricks/terraform-provider-databricks/pull/3033)).
+ * Added acceptance tests for [databricks_grants](https://registry.terraform.io/providers/databricks/databricks/latest/docs/resources/grants) ([#3034](https://github.com/databricks/terraform-provider-databricks/pull/3034)).
+ * Migrated library create to use go-sdk ([#2998](https://github.com/databricks/terraform-provider-databricks/pull/2998)).
+
+## 1.31.1
+
+New Features and Improvements:
+ * Added `workspace_path` attribute to [databricks_workspace_file](https://registry.terraform.io/providers/databricks/databricks/latest/docs/resources/workspace_file) resource ([#2984](https://github.com/databricks/terraform-provider-databricks/pull/2984)).
+
+Bugfixes:
+ * Bugfix for azure authentication ([#732](https://github.com/databricks/databricks-sdk-go/pull/732))
+
+Exporter:
+ * Exporter: fix suppression of empty `webhook_notifications` blocks in tasks ([#2954](https://github.com/databricks/terraform-provider-databricks/pull/2954)).
+
+Internal Changes:
+ * Migrated library read to use Go SDK ([#2985](https://github.com/databricks/terraform-provider-databricks/pull/2985)).
+ * Migrated [databricks_mws_credentials](https://registry.terraform.io/providers/databricks/databricks/latest/docs/resources/mws_credentials) to Go SDK ([#2962](https://github.com/databricks/terraform-provider-databricks/pull/2962)).
+ * Update Go SDK to v0.26.2
+
+
 ## 1.31.0
 
 New Features and Improvements:

Makefile

@@ -81,4 +81,10 @@ test-preview: install
 docker-it:
 	docker build -t databricks-terrafrom/test -f scripts/Dockerfile .
 
+schema:
+	@/bin/bash scripts/print-schema.sh
+
+diff-schema:
+	@/bin/bash scripts/diff-schema.sh
+
 .PHONY: build fmt python-setup docs vendor build fmt coverage test lint

README.md

@@ -24,6 +24,7 @@
 | [databricks_external_location](docs/resources/external_location.md)
 | [databricks_git_credential](docs/resources/git_credential.md)
 | [databricks_global_init_script](docs/resources/global_init_script.md)
+| [databricks_grant](docs/resources/grant.md)
 | [databricks_grants](docs/resources/grants.md)
 | [databricks_group](docs/resources/group.md)
 | [databricks_group](docs/data-sources/group.md) data
@@ -156,6 +157,10 @@ output "job_url" {
 
 Then run `terraform init` then `terraform apply` to apply the hcl code to your Databricks workspace.
 
+## OpenTofu Support
+
+[OpenTofu](https://opentofu.org/) is an open-source fork of Terraform with the MPL 2.0 license. The Databricks Terraform provider should be compatible with OpenTofu, but this integration is not actively tested and should be considered experimental. Please raise a Github issue if you find any incompatibility.
+
 ## Switching from `databrickslabs` to `databricks` namespace
 
 To make Databricks Terraform Provider generally available, we've moved it from [https://github.com/databrickslabs](https://github.com/databrickslabs) to [https://github.com/databricks](https://github.com/databricks). We've worked closely with the Terraform Registry team at Hashicorp to ensure a smooth migration. Existing terraform deployments continue to work as expected without any action from your side. We ask you to replace `databrickslabs/databricks` with `databricks/databricks` in all your `.tf` files.

access/resource_ip_access_list_test.go

@@ -182,7 +182,7 @@ func TestIPACLUpdate_Error(t *testing.T) {
 			{
 				Method:   http.MethodPatch,
 				Resource: "/api/2.0/ip-access-lists/" + TestingId,
-				ExpectedRequest: ipAccessListUpdateRequest{
+				ExpectedRequest: settings.UpdateIpAccessList{
 					Enabled: TestingEnabled,
 				},
 				Response: apierr.APIErrorBody{

access/resource_permission_assignment.go

@@ -81,8 +81,7 @@ func ResourcePermissionAssignment() *schema.Resource {
 		PrincipalId int64    `json:"principal_id"`
 		Permissions []string `json:"permissions" tf:"slice_as_set"`
 	}
-	s := common.StructToSchema(entity{},
-		common.NoCustomize)
+	s := common.StructToSchema(entity{}, common.NoCustomize)
 	return common.Resource{
 		Schema: s,
 		Create: func(ctx context.Context, d *schema.ResourceData, c *common.DatabricksClient) error {
@@ -101,11 +100,15 @@ func ResourcePermissionAssignment() *schema.Resource {
 			if err != nil {
 				return err
 			}
-			permissions, err := list.ForPrincipal(mustInt64(d.Id()))
+			data := entity{
+				PrincipalId: mustInt64(d.Id()),
+			}
+			permissions, err := list.ForPrincipal(data.PrincipalId)
 			if err != nil {
 				return err
 			}
-			return common.StructToData(permissions, s, d)
+			data.Permissions = permissions.Permissions
+			return common.StructToData(data, s, d)
 		},
 		Delete: func(ctx context.Context, d *schema.ResourceData, c *common.DatabricksClient) error {
 			return NewPermissionAssignmentAPI(ctx, c).Remove(d.Id())

access/resource_permission_assignment_test.go

@@ -31,16 +31,43 @@ func TestPermissionAssignmentCreate(t *testing.T) {
 				},
 			},
 		},
-		Resource:  ResourcePermissionAssignment(),
-		Create:    true,
-		AccountID: "abc",
+		Resource: ResourcePermissionAssignment(),
+		Create:   true,
 		HCL: `
 		principal_id = 345
 		permissions  = ["USER"]
 		`,
 	}.ApplyNoError(t)
 }
 
+func TestPermissionAssignmentRead(t *testing.T) {
+	qa.ResourceFixture{
+		Fixtures: []qa.HTTPFixture{
+			{
+				Method:   "GET",
+				Resource: "/api/2.0/preview/permissionassignments",
+				Response: PermissionAssignmentList{
+					PermissionAssignments: []PermissionAssignment{
+						{
+							Permissions: []string{"USER"},
+							Principal: Principal{
+								PrincipalID: 345,
+							},
+						},
+					},
+				},
+			},
+		},
+		Resource: ResourcePermissionAssignment(),
+		Read:     true,
+		New:      true,
+		ID:       "345",
+	}.ApplyAndExpectData(t, map[string]any{
+		"principal_id": 345,
+		"permissions":  []any{"USER"},
+	})
+}
+
 func TestPermissionAssignmentReadNotFound(t *testing.T) {
 	qa.ResourceFixture{
 		Fixtures: []qa.HTTPFixture{