From fae977cfe5ee52607e1d40ad1747a8140f44182f Mon Sep 17 00:00:00 2001 From: Flavio Malavazi Date: Tue, 14 Feb 2023 16:52:00 -0300 Subject: [PATCH 1/5] Adding Unity Catalog + Data Exfil Example Changes: - Creating a `test_azure_uc_data_exfiltration_protection` example - Bumped versions for providers to the latest ones - Removed the providers file from the modules so that I could add depends_on blocks on the main file (I was having errors with the UC trying to read from an inexistent resource group, then the same issue with the firewall and the workspace, etc etc) - Replaced the `enforce_private_link_endpoint_network_policies` argument in the `terraform-databricks-lakehouse-blueprints/modules/azure_spoke_vnet/endpoints.tf` file with `private_endpoint_network_policies_enabled` (the enforce argument will be deprecated) - Added the `databricks_workspace_id` output to the `terraform-databricks-lakehouse-blueprints/modules/azure_vnet_injected_databricks_workspace/outputs.tf` file so that I could use our numerical workspace ID with an UC attachment. The output `workspace_id`, which is taken from `azurerm_databricks_workspace.this.id` is the Azure ID (alphanumerical), which does not work with the UC module. Also added a `tonumber` on that output because it yields a string and the UC module needs an integer. - Added the Databricks provider on the example and initialized it with the `workspace_url` from the `module.spoke_databricks_workspace` --- .../README.md | 57 +++++++++++++ .../firewall.tf | 83 +++++++++++++++++++ .../main.tf | 63 ++++++++++++++ .../outputs.tf | 11 +++ .../providers.tf | 7 ++ .../variables.tf | 80 ++++++++++++++++++ .../versions.tf | 14 ++++ modules/azure_spoke_vnet/endpoints.tf | 2 +- modules/azure_spoke_vnet/providers.tf | 3 - modules/azure_spoke_vnet/versions.tf | 2 +- modules/azure_uc/versions.tf | 4 +- .../outputs.tf | 5 ++ .../providers.tf | 3 - .../versions.tf | 2 +- 14 files changed, 325 insertions(+), 11 deletions(-) create mode 100644 examples/test_azure_uc_data_exfiltration_protection/README.md create mode 100644 examples/test_azure_uc_data_exfiltration_protection/firewall.tf create mode 100644 examples/test_azure_uc_data_exfiltration_protection/main.tf create mode 100644 examples/test_azure_uc_data_exfiltration_protection/outputs.tf create mode 100644 examples/test_azure_uc_data_exfiltration_protection/providers.tf create mode 100644 examples/test_azure_uc_data_exfiltration_protection/variables.tf create mode 100644 examples/test_azure_uc_data_exfiltration_protection/versions.tf delete mode 100644 modules/azure_spoke_vnet/providers.tf delete mode 100644 modules/azure_vnet_injected_databricks_workspace/providers.tf diff --git a/examples/test_azure_uc_data_exfiltration_protection/README.md b/examples/test_azure_uc_data_exfiltration_protection/README.md new file mode 100644 index 0000000..74d52c9 --- /dev/null +++ b/examples/test_azure_uc_data_exfiltration_protection/README.md @@ -0,0 +1,57 @@ +--- +page_title: "Provisioning Azure Databricks Hub and Spoke Deployment as per Data Exfiltration Protection and Unity Catalog with Terraform" +--- + +# Provisioning Azure Databricks Hub and Spoke Deployment as per Data Exfiltration Protection with Terraform + +[Reference documentation and blog](https://databricks.com/blog/2020/03/27/data-exfiltration-protection-with-azure-databricks.html) +This Terraform configuration is an implementation of the above blog post. +Note: the firewall rules deviate slightly in that outbound traffic from the firewall is allowed to Databricks resources instead of specifying Databricks worker subnets. +This is to simplify outbound routing in the event that multiple `spoke`s are desired. + +This guide is provided as-is and you can use this guide as the basis for your custom Terraform module. + +It uses the following variables in configurations: + +## Required + +- `project_name`: (Required) The name of the project associated with the infrastructure to be managed by Terraform +- `location`: (Required) The location for the resources in this module +- `databricks_workspace_name`: (Required) The name of the Azure Databricks Workspace to deploy in the spoke vnet +- `privatelink_subnet_address_prefixes`: (Required) The address prefix(es) for the PrivateLink subnet +- `firewall_name`: (Required) The name of the Azure Firewall deployed in your hub Virtual Network +- `firewall_private_ip`: (Required) The hub firewall's private IP address + + +## Optional + +- `hub_resource_group_name`: (Optional) The name of the existing Resource Group containing the hub Virtual Network +- `hub_vnet_name`: (Optional) The name of the existing hub Virtual Network +- `hub_vnet_address_space`: (Optional) The address space for the hub Virtual Network +- `spoke_resource_group_name`: (Optional) The name of the Resource Group to create +- `spoke_vnet_address_space`: (Optional) The address space for the spoke Virtual Network +- `private_subnet_address_prefixes`: (Optional) The address prefix(es) for the Databricks private subnet +- `public_subnet_address_prefixes`: (Optional) The address prefix(es) for the Databricks public subnet +- `firewall_subnet_address_prefixes`: (Optional) The address prefixes for the Azure firewall subnet +- `public_repos`: (Optional) List of public repository IP addresses to allow access to. +- `tags`: (Optional) Map of tags to attach to resources + +## Provider initialization + +```hcl +terraform { + required_providers { + azurerm = { + source = "hashicorp/azurerm" + version = "~>3.43.0" + } + + databricks = { + source = "databricks/databricks" + version = ">=1.9.2" + } + + } +} + +``` diff --git a/examples/test_azure_uc_data_exfiltration_protection/firewall.tf b/examples/test_azure_uc_data_exfiltration_protection/firewall.tf new file mode 100644 index 0000000..44f5be4 --- /dev/null +++ b/examples/test_azure_uc_data_exfiltration_protection/firewall.tf @@ -0,0 +1,83 @@ +resource "azurerm_subnet" "firewall" { + name = "AzureFirewallSubnet" + resource_group_name = azurerm_resource_group.this.name + virtual_network_name = azurerm_virtual_network.this.name + + address_prefixes = [var.firewall_subnet_address_prefixes] + service_endpoints = [ + "Microsoft.Storage", + "Microsoft.AzureActiveDirectory" + ] +} + +resource "azurerm_public_ip" "this" { + name = "firewall-public-ip" + location = azurerm_resource_group.this.location + resource_group_name = azurerm_resource_group.this.name + allocation_method = "Static" + sku = "Standard" +} + +resource "azurerm_firewall_policy" "this" { + name = "databricks-fwpolicy" + resource_group_name = var.hub_resource_group_name + location = azurerm_resource_group.this.location +} + +resource "azurerm_firewall_policy_rule_collection_group" "this" { + name = "databricks-fwpolicy-rcg" + firewall_policy_id = azurerm_firewall_policy.this.id + priority = 200 + application_rule_collection { + name = "databricks-app-rc" + priority = 200 + action = "Allow" + + rule { + name = "public-repos" + source_addresses = ["*"] + destination_fqdns = var.public_repos + protocols { + port = "443" + type = "Https" + } + protocols { + port = "80" + type = "Http" + } + } + + rule { + name = "IPinfo" + source_addresses = ["*"] + destination_fqdns = ["*.ipinfo.io"] + protocols { + port = "443" + type = "Https" + } + protocols { + port = "8080" + type = "Http" + } + protocols { + port = "80" + type = "Http" + } + } + } +} + +resource "azurerm_firewall" "this" { + name = "${azurerm_virtual_network.this.name}-firewall" + location = azurerm_resource_group.this.location + resource_group_name = azurerm_resource_group.this.name + sku_name = "AZFW_VNet" + sku_tier = "Standard" + firewall_policy_id = azurerm_firewall_policy.this.id + + ip_configuration { + name = "firewall-public-ip-config" + subnet_id = azurerm_subnet.firewall.id + public_ip_address_id = azurerm_public_ip.this.id + } +} diff --git a/examples/test_azure_uc_data_exfiltration_protection/main.tf b/examples/test_azure_uc_data_exfiltration_protection/main.tf new file mode 100644 index 0000000..be740e9 --- /dev/null +++ b/examples/test_azure_uc_data_exfiltration_protection/main.tf @@ -0,0 +1,63 @@ +resource "azurerm_resource_group" "this" { + name = var.hub_resource_group_name + location = var.location +} + +resource "azurerm_virtual_network" "this" { + name = var.hub_vnet_name + location = azurerm_resource_group.this.location + resource_group_name = azurerm_resource_group.this.name + address_space = [var.hub_vnet_address_space] +} + +module "spoke_vnet" { + # TODO: Get rid of redundant variables - source them from `id`s or something + # TODO: Add Routes for service tags to the route table + source = "../../modules/azure_spoke_vnet" + project_name = var.project_name + location = azurerm_virtual_network.this.location + hub_vnet_name = azurerm_virtual_network.this.name + hub_resource_group_name = azurerm_resource_group.this.name + firewall_private_ip = azurerm_firewall.this.ip_configuration[0].private_ip_address + spoke_vnet_address_space = var.spoke_vnet_address_space + spoke_resource_group_name = var.spoke_resource_group_name + privatelink_subnet_address_prefixes = var.privatelink_subnet_address_prefixes + tags = var.tags + depends_on = [ + resource.azurerm_resource_group.this, + resource.azurerm_virtual_network.this + ] + +} + +module "spoke_databricks_workspace" { + source = "../../modules/azure_vnet_injected_databricks_workspace" + workspace_name = var.databricks_workspace_name + databricks_resource_group_name = module.spoke_vnet.rg_name + location = azurerm_virtual_network.this.location + vnet_id = module.spoke_vnet.vnet_id + vnet_name = module.spoke_vnet.vnet_name + nsg_id = module.spoke_vnet.nsg_id + route_table_id = module.spoke_vnet.route_table_id + private_subnet_address_prefixes = var.private_subnet_address_prefixes + public_subnet_address_prefixes = var.public_subnet_address_prefixes + tags = var.tags + + depends_on = [ + module.spoke_vnet + ] + +} + + +module "unity_catalog" { + source = "../../modules/azure_uc" + + resource_group_id = azurerm_resource_group.this.id + workspaces_to_associate = [module.spoke_databricks_workspace.databricks_workspace_id] + + depends_on = [ + module.spoke_databricks_workspace + ] + +} diff --git a/examples/test_azure_uc_data_exfiltration_protection/outputs.tf b/examples/test_azure_uc_data_exfiltration_protection/outputs.tf new file mode 100644 index 0000000..fd33a30 --- /dev/null +++ b/examples/test_azure_uc_data_exfiltration_protection/outputs.tf @@ -0,0 +1,11 @@ +output "resource_group_name" { + value = azurerm_resource_group.this.name +} + +output "virtual_network_name" { + value = azurerm_virtual_network.this.name +} + +output "firewall_name" { + value = azurerm_firewall.this.name +} diff --git a/examples/test_azure_uc_data_exfiltration_protection/providers.tf b/examples/test_azure_uc_data_exfiltration_protection/providers.tf new file mode 100644 index 0000000..d28613c --- /dev/null +++ b/examples/test_azure_uc_data_exfiltration_protection/providers.tf @@ -0,0 +1,7 @@ +provider "azurerm" { + features {} +} + +provider "databricks" { + host = module.spoke_databricks_workspace.workspace_url +} diff --git a/examples/test_azure_uc_data_exfiltration_protection/variables.tf b/examples/test_azure_uc_data_exfiltration_protection/variables.tf new file mode 100644 index 0000000..1fb54bb --- /dev/null +++ b/examples/test_azure_uc_data_exfiltration_protection/variables.tf @@ -0,0 +1,80 @@ +variable "project_name" { + type = string + description = "(Required) The name of the project associated with the infrastructure to be managed by Terraform" +} + +variable "location" { + type = string + description = "(Required) The location for the resources in this module" +} + +variable "hub_resource_group_name" { + type = string + description = "(Optional) The name for the hub Resource Group" + default = "hub-rg" +} + +variable "hub_vnet_name" { + type = string + description = "(Optional) The name for the hub Virtual Network" + default = "hub-vnet" +} + +variable "hub_vnet_address_space" { + type = string + description = "(Optional) The address space for the hub Virtual Network" + default = "10.3.1.0/24" +} + +variable "spoke_resource_group_name" { + type = string + description = "(Optional) The name of the Resource Group to create" + default = "spoke-rg" +} + +variable "spoke_vnet_address_space" { + type = string + description = "(Optional) The address space for the spoke Virtual Network" + default = "10.2.1.0/24" +} + +variable "databricks_workspace_name" { + type = string + description = "(Required) The name of the Azure Databricks Workspace to deploy" +} + +variable "privatelink_subnet_address_prefixes" { + type = list(string) + description = "(Optional) The address prefix(es) for the PrivateLink subnet" + default = ["10.2.1.0/26"] +} + +variable "private_subnet_address_prefixes" { + type = list(string) + description = "(Optional) The address prefix(es) for the Databricks private subnet" + default = ["10.2.1.128/26"] +} + +variable "public_subnet_address_prefixes" { + type = list(string) + description = "(Optional) The address prefix(es) for the Databricks public subnet" + default = ["10.2.1.64/26"] +} + +variable "firewall_subnet_address_prefixes" { + type = string + description = "(Optional) The address prefixes for the Azure firewall subnet" + default = "10.3.1.0/26" +} + +variable "public_repos" { + type = list(string) + description = "(Optional) List of public repository IP addresses to allow access to." + default = ["*.pypi.org", "*pythonhosted.org", "cran.r-project.org"] +} + +variable "tags" { + type = map(string) + description = "(Optional) Map of tags to attach to resources" + default = {} +} diff --git a/examples/test_azure_uc_data_exfiltration_protection/versions.tf b/examples/test_azure_uc_data_exfiltration_protection/versions.tf new file mode 100644 index 0000000..d8f9cc4 --- /dev/null +++ b/examples/test_azure_uc_data_exfiltration_protection/versions.tf @@ -0,0 +1,14 @@ +terraform { + required_providers { + azurerm = { + source = "hashicorp/azurerm" + version = "~>3.43.0" + } + + databricks = { + source = "databricks/databricks" + version = ">=1.9.2" + } + + } +} diff --git a/modules/azure_spoke_vnet/endpoints.tf b/modules/azure_spoke_vnet/endpoints.tf index ca5e37d..02380e3 100644 --- a/modules/azure_spoke_vnet/endpoints.tf +++ b/modules/azure_spoke_vnet/endpoints.tf @@ -4,7 +4,7 @@ resource "azurerm_subnet" "privatelink" { virtual_network_name = azurerm_virtual_network.this.name address_prefixes = var.privatelink_subnet_address_prefixes - enforce_private_link_endpoint_network_policies = true + private_endpoint_network_policies_enabled = true } resource "random_string" "suffix" { diff --git a/modules/azure_spoke_vnet/providers.tf b/modules/azure_spoke_vnet/providers.tf deleted file mode 100644 index ab91b24..0000000 --- a/modules/azure_spoke_vnet/providers.tf +++ /dev/null @@ -1,3 +0,0 @@ -provider "azurerm" { - features {} -} diff --git a/modules/azure_spoke_vnet/versions.tf b/modules/azure_spoke_vnet/versions.tf index c37aea3..d7cb699 100644 --- a/modules/azure_spoke_vnet/versions.tf +++ b/modules/azure_spoke_vnet/versions.tf @@ -2,7 +2,7 @@ terraform { required_providers { azurerm = { source = "hashicorp/azurerm" - version = "~>3.13.0" + version = "~>3.43.0" } } } diff --git a/modules/azure_uc/versions.tf b/modules/azure_uc/versions.tf index 2170e70..302663a 100644 --- a/modules/azure_uc/versions.tf +++ b/modules/azure_uc/versions.tf @@ -2,11 +2,11 @@ terraform { required_providers { azurerm = { source = "hashicorp/azurerm" - version = ">=3.30.0" + version = ">=3.43.0" } databricks = { source = "databricks/databricks" - version = ">=1.6.4" + version = ">=1.9.2" } } } diff --git a/modules/azure_vnet_injected_databricks_workspace/outputs.tf b/modules/azure_vnet_injected_databricks_workspace/outputs.tf index 4993b82..6931d1a 100755 --- a/modules/azure_vnet_injected_databricks_workspace/outputs.tf +++ b/modules/azure_vnet_injected_databricks_workspace/outputs.tf @@ -8,6 +8,11 @@ output "workspace_id" { description = "ID of the Databricks workspace" } +output "databricks_workspace_id" { + value = tonumber(azurerm_databricks_workspace.this.workspace_id) + description = "ID of the Databricks workspace in the Databricks Control Plane" +} + output "workspace_url" { value = azurerm_databricks_workspace.this.workspace_url } diff --git a/modules/azure_vnet_injected_databricks_workspace/providers.tf b/modules/azure_vnet_injected_databricks_workspace/providers.tf deleted file mode 100644 index ab91b24..0000000 --- a/modules/azure_vnet_injected_databricks_workspace/providers.tf +++ /dev/null @@ -1,3 +0,0 @@ -provider "azurerm" { - features {} -} diff --git a/modules/azure_vnet_injected_databricks_workspace/versions.tf b/modules/azure_vnet_injected_databricks_workspace/versions.tf index ebec7af..8e7620c 100755 --- a/modules/azure_vnet_injected_databricks_workspace/versions.tf +++ b/modules/azure_vnet_injected_databricks_workspace/versions.tf @@ -2,7 +2,7 @@ terraform { required_providers { azurerm = { source = "hashicorp/azurerm" - version = ">= 3.10.0" + version = ">= 3.43.0" } } } From 1805645f4016a9b1571de128928e3a1c58910dd1 Mon Sep 17 00:00:00 2001 From: Flavio Malavazi Date: Tue, 14 Feb 2023 21:56:17 -0300 Subject: [PATCH 2/5] Adjusting dependencies and firewall Changes: - Adjusted dependencies for network resources in order to avoid failed states - Removed dependencies between modules as it was causing unintended consequences - Added adjusted python hosts to install pypi libraries through the firewall - Removed empty default list from the azure_uc module in order to avoid failed state where unity tries to be associated with no workspaces --- .../firewall.tf | 12 +++++++++++- .../main.tf | 13 +------------ .../variables.tf | 2 +- modules/azure_uc/variables.tf | 1 - 4 files changed, 13 insertions(+), 15 deletions(-) diff --git a/examples/test_azure_uc_data_exfiltration_protection/firewall.tf b/examples/test_azure_uc_data_exfiltration_protection/firewall.tf index 44f5be4..318079d 100644 --- a/examples/test_azure_uc_data_exfiltration_protection/firewall.tf +++ b/examples/test_azure_uc_data_exfiltration_protection/firewall.tf @@ -50,7 +50,7 @@ resource "azurerm_firewall_policy_rule_collection_group" "this" { rule { name = "IPinfo" source_addresses = ["*"] - destination_fqdns = ["*.ipinfo.io"] + destination_fqdns = ["*.ipinfo.io", "ipinfo.io"] protocols { port = "443" type = "Https" @@ -65,6 +65,11 @@ resource "azurerm_firewall_policy_rule_collection_group" "this" { } } } + + depends_on = [ + resource.azurerm_firewall_policy.this + ] + } resource "azurerm_firewall" "this" { @@ -80,4 +85,9 @@ resource "azurerm_firewall" "this" { subnet_id = azurerm_subnet.firewall.id public_ip_address_id = azurerm_public_ip.this.id } + + depends_on = [ + resource.azurerm_firewall_policy_rule_collection_group.this + ] + } diff --git a/examples/test_azure_uc_data_exfiltration_protection/main.tf b/examples/test_azure_uc_data_exfiltration_protection/main.tf index be740e9..130f8b6 100644 --- a/examples/test_azure_uc_data_exfiltration_protection/main.tf +++ b/examples/test_azure_uc_data_exfiltration_protection/main.tf @@ -24,10 +24,8 @@ module "spoke_vnet" { privatelink_subnet_address_prefixes = var.privatelink_subnet_address_prefixes tags = var.tags depends_on = [ - resource.azurerm_resource_group.this, - resource.azurerm_virtual_network.this + resource.azurerm_resource_group.this ] - } module "spoke_databricks_workspace" { @@ -42,11 +40,6 @@ module "spoke_databricks_workspace" { private_subnet_address_prefixes = var.private_subnet_address_prefixes public_subnet_address_prefixes = var.public_subnet_address_prefixes tags = var.tags - - depends_on = [ - module.spoke_vnet - ] - } @@ -56,8 +49,4 @@ module "unity_catalog" { resource_group_id = azurerm_resource_group.this.id workspaces_to_associate = [module.spoke_databricks_workspace.databricks_workspace_id] - depends_on = [ - module.spoke_databricks_workspace - ] - } diff --git a/examples/test_azure_uc_data_exfiltration_protection/variables.tf b/examples/test_azure_uc_data_exfiltration_protection/variables.tf index 1fb54bb..b415370 100644 --- a/examples/test_azure_uc_data_exfiltration_protection/variables.tf +++ b/examples/test_azure_uc_data_exfiltration_protection/variables.tf @@ -70,7 +70,7 @@ variable "firewall_subnet_address_prefixes" { variable "public_repos" { type = list(string) description = "(Optional) List of public repository IP addresses to allow access to." - default = ["*.pypi.org", "*pythonhosted.org", "cran.r-project.org"] + default = ["python.org", "*.python.org", "pypi.org", "*.pypi.org", "pythonhosted.org", "*.pythonhosted.org", "cran.r-project.org", "*.cran.r-project.org", "r-project.org"] } variable "tags" { diff --git a/modules/azure_uc/variables.tf b/modules/azure_uc/variables.tf index 778a3be..3e1eeba 100644 --- a/modules/azure_uc/variables.tf +++ b/modules/azure_uc/variables.tf @@ -6,5 +6,4 @@ variable "resource_group_id" { variable "workspaces_to_associate" { type = list(string) description = "(Optional) List of Databricks Workspace IDs to associate with Unity Catalog" - default = [] } From a74312512ed420e3a33ea72805a83db527861e92 Mon Sep 17 00:00:00 2001 From: Flavio Malavazi Date: Tue, 14 Feb 2023 22:34:11 -0300 Subject: [PATCH 3/5] Adding terraform lock to example --- .../.terraform.lock.hcl | 59 +++++++++++++++++++ 1 file changed, 59 insertions(+) create mode 100644 examples/test_azure_uc_data_exfiltration_protection/.terraform.lock.hcl diff --git a/examples/test_azure_uc_data_exfiltration_protection/.terraform.lock.hcl b/examples/test_azure_uc_data_exfiltration_protection/.terraform.lock.hcl new file mode 100644 index 0000000..18635c7 --- /dev/null +++ b/examples/test_azure_uc_data_exfiltration_protection/.terraform.lock.hcl @@ -0,0 +1,59 @@ +# This file is maintained automatically by "terraform init". +# Manual edits may be lost in future updates. + +provider "registry.terraform.io/databricks/databricks" { + version = "1.9.2" + constraints = ">= 1.9.2" + hashes = [ + "h1:nFvHH92ID3dgaUyTgbbl1hFWuQJ/fEK1Q8UhF+zZmaY=", + "zh:28d4ae6d2175aeaa03544433fe96649a5f4ae060b72a2275bb2cd9df7ae415b2", + "zh:717f2906f943f6f64b5ce4450fc004d99da5743c1c215cc215a67d1e532b224e", + "zh:872b873537f3e12b225fc8c557e8ed6d31cb083623994a804e5df5429cbfd62e", + "zh:8a113fd0882bb21295b4c9ccdd9898fe2d4a30e931dd4b37a213f9eb0bd04ca5", + "zh:8c00a9d2f7be0af1c7854f1fe9a924dc4311a941c89fdc531e8160a59d9dae72", + "zh:a94cd0b3ba25ee73d8a92f8dc9d74bea2aae9c7e2a9b2ebfd20dc859a9c3e5a2", + "zh:b00b06600680cf7052189b78bcda548e68bca944f1f901e59f8c9afd08b27267", + "zh:b06988861eb0d82dfe0a7bbda20de82a6e424b4b6eab36cddca6c1d5c8cd2d1d", + "zh:d647bac7e19c1c9bd80241da2968c8f823adc2c20a27bfa286e8225b05b46430", + "zh:f2f42e9bb47c11af9f4b004f2d630d4a87f14d3446f2db529b6b2cb05a43b0ee", + ] +} + +provider "registry.terraform.io/hashicorp/azurerm" { + version = "3.43.0" + constraints = ">= 3.43.0, ~> 3.43.0" + hashes = [ + "h1:Oc/do6RovvNcKddQglsUeX1gWDeA1F7Fet+ajqWYb8M=", + "zh:1a6d3553a8b9c85193d8334e8678aae305d14ec1d69b0d45799c322145d41475", + "zh:1cb9ecd6531060c8f52d4f70863754ef18d3c297dee2aa173ce6dbd6f3c62621", + "zh:21effe14cf1f5bace7aa172198ee2aa6ffc78324e4648af9b8df8b29995fa711", + "zh:29e53d13567d1497388c4264fea7548a45a3d1065129a475f0c8708eb0b9fa4d", + "zh:6c9036ed1371220709fab11ecd790953bb066bc8113707d3f4b9334d07fddf11", + "zh:7f26877a5216fb92e2a1594da7eb61058a984e7f8b305c45745ad181c0357b71", + "zh:a080ea3a591b353dd3432d5f1a7fe717dc733a02429b50ff38ef0fba92bd93e2", + "zh:b880602640876fbccf7d2d6dbbf6a076bf42126e07990975a29995c1a899563b", + "zh:c46125c6fcf67f69b8d33f29e5362ae78fc305787be77076c10f36561c2076d2", + "zh:f08642f55085ac03bfad32917abd42b75f2dffb7b9d8e7c310cc230f9a15e756", + "zh:f569b65999264a9416862bca5cd2a6177d94ccb0424f3a4ef424428912b9cb3c", + "zh:f85af4d3af54ebad40ea6cf4dfd5cb1f7b0666cf4793711a1bff719f059177f7", + ] +} + +provider "registry.terraform.io/hashicorp/random" { + version = "3.4.3" + hashes = [ + "h1:saZR+mhthL0OZl4SyHXZraxyaBNVMxiZzks78nWcZ2o=", + "zh:41c53ba47085d8261590990f8633c8906696fa0a3c4b384ff6a7ecbf84339752", + "zh:59d98081c4475f2ad77d881c4412c5129c56214892f490adf11c7e7a5a47de9b", + "zh:686ad1ee40b812b9e016317e7f34c0d63ef837e084dea4a1f578f64a6314ad53", + "zh:78d5eefdd9e494defcb3c68d282b8f96630502cac21d1ea161f53cfe9bb483b3", + "zh:84103eae7251384c0d995f5a257c72b0096605048f757b749b7b62107a5dccb3", + "zh:8ee974b110adb78c7cd18aae82b2729e5124d8f115d484215fd5199451053de5", + "zh:9dd4561e3c847e45de603f17fa0c01ae14cae8c4b7b4e6423c9ef3904b308dda", + "zh:bb07bb3c2c0296beba0beec629ebc6474c70732387477a65966483b5efabdbc6", + "zh:e891339e96c9e5a888727b45b2e1bb3fcbdfe0fd7c5b4396e4695459b38c8cb1", + "zh:ea4739860c24dfeaac6c100b2a2e357106a89d18751f7693f3c31ecf6a996f8d", + "zh:f0c76ac303fd0ab59146c39bc121c5d7d86f878e9a69294e29444d4c653786f8", + "zh:f143a9a5af42b38fed328a161279906759ff39ac428ebcfe55606e05e1518b93", + ] +} From 387e8d3072170e58ef607170bdbfb56b5d59a632 Mon Sep 17 00:00:00 2001 From: Flavio Malavazi Date: Wed, 15 Feb 2023 16:08:30 -0300 Subject: [PATCH 4/5] Adding dependencies on network resources to avoid failed states Changes: - Added output workspace URL to make it easier to find the workspace later on - Adding dependencies between network resources (private endpoints, subnets etc) to avoid failed states on terraform apply - --- .../test_azure_uc_data_exfiltration_protection/outputs.tf | 4 ++++ modules/azure_spoke_vnet/endpoints.tf | 5 +++++ 2 files changed, 9 insertions(+) diff --git a/examples/test_azure_uc_data_exfiltration_protection/outputs.tf b/examples/test_azure_uc_data_exfiltration_protection/outputs.tf index fd33a30..37637ca 100644 --- a/examples/test_azure_uc_data_exfiltration_protection/outputs.tf +++ b/examples/test_azure_uc_data_exfiltration_protection/outputs.tf @@ -9,3 +9,7 @@ output "virtual_network_name" { output "firewall_name" { value = azurerm_firewall.this.name } + +output "workspace_url" { + value = module.spoke_databricks_workspace.workspace_url +} diff --git a/modules/azure_spoke_vnet/endpoints.tf b/modules/azure_spoke_vnet/endpoints.tf index 02380e3..c85d109 100644 --- a/modules/azure_spoke_vnet/endpoints.tf +++ b/modules/azure_spoke_vnet/endpoints.tf @@ -50,6 +50,11 @@ resource "azurerm_private_endpoint" "storage" { name = "privatelink.dfs.core.windows.net" private_dns_zone_ids = [azurerm_private_dns_zone.storage.id] } + + depends_on = [ + resource.azurerm_subnet.privatelink, + resource.azurerm_virtual_network.this + ] } resource "azurerm_private_dns_a_record" "this" { From db883f4f1060926d7c04ca13ea13061c9ff36cee Mon Sep 17 00:00:00 2001 From: Flavio Malavazi Date: Thu, 16 Feb 2023 17:11:44 -0300 Subject: [PATCH 5/5] Adjusting data type for workspaces to associate UC Changes: - list should be of numbers, not strings! --- modules/azure_uc/variables.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/azure_uc/variables.tf b/modules/azure_uc/variables.tf index 3e1eeba..8f4b0f1 100644 --- a/modules/azure_uc/variables.tf +++ b/modules/azure_uc/variables.tf @@ -4,6 +4,6 @@ variable "resource_group_id" { } variable "workspaces_to_associate" { - type = list(string) + type = list(number) description = "(Optional) List of Databricks Workspace IDs to associate with Unity Catalog" }