diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml
index 2e32634..7724649 100644
--- a/.github/workflows/e2e.yml
+++ b/.github/workflows/e2e.yml
@@ -1,99 +1,29 @@
---
-name: e2e test
-
-on:
- pull_request:
- types: ['opened', 'reopened', 'synchronize']
- merge_group:
- workflow_dispatch:
-
-permissions:
- contents: read
- id-token: write
-
-jobs:
- getexamples:
- if: github.event.repository.name != 'terraform-azurerm-avm-template'
- runs-on: ubuntu-latest
- outputs:
- examples: ${{ steps.getexamples.outputs.examples }}
- steps:
- - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 #v4.1.7
- - name: get examples
- id: getexamples
- uses: Azure/terraform-azurerm-avm-template/.github/actions/e2e-getexamples@main
- with:
- github-token: ${{ secrets.GITHUB_TOKEN }}
-
- testexamples:
- if: github.event.repository.name != 'terraform-azurerm-avm-template'
- runs-on: [ self-hosted, 1ES.Pool=terraform-azurerm-avm-ptn-aks-production ]
- needs: getexamples
- environment: test
- env:
- TF_IN_AUTOMATION: 1
- TF_VAR_enable_telemetry: false
- strategy:
- matrix:
- example: ${{ fromJson(needs.getexamples.outputs.examples) }}
- fail-fast: false
- steps:
- - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 #v4.1.7
-
- - name: Test example
- shell: bash
- env:
- SECRETS_CONTEXT: ${{ toJson(secrets) }}
- VARS_CONTEXT: ${{ toJson(vars) }}
- run: |
- set -e
- MAX_RETRIES=10
- RETRY_COUNT=0
- until [ $RETRY_COUNT -ge $MAX_RETRIES ]
- do
- az login --identity --username $MSI_ID > /dev/null && break
- RETRY_COUNT=$[$RETRY_COUNT+1]
- sleep 10
- done
- if [ $RETRY_COUNT -eq $MAX_RETRIES ]; then
- echo "Failed to login after $MAX_RETRIES attempts."
- exit 1
- fi
-
- declare -A secrets
- eval "$(echo $SECRETS_CONTEXT | jq -r 'to_entries[] | @sh "secrets[\(.key|tostring)]=\(.value|tostring)"')"
-
- declare -A variables
- eval "$(echo $VARS_CONTEXT | jq -r 'to_entries[] | @sh "variables[\(.key|tostring)]=\(.value|tostring)"')"
-
- for key in "${!secrets[@]}"; do
- if [[ $key = \TF_VAR_* ]]; then
- lowerKey=$(echo "$key" | tr '[:upper:]' '[:lower:]')
- finalKey=${lowerKey/tf_var_/TF_VAR_}
- export "$finalKey"="${secrets[$key]}"
- fi
- done
-
- for key in "${!variables[@]}"; do
- if [[ $key = \TF_VAR_* ]]; then
- lowerKey=$(echo "$key" | tr '[:upper:]' '[:lower:]')
- finalKey=${lowerKey/tf_var_/TF_VAR_}
- export "$finalKey"="${variables[$key]}"
- fi
- done
-
- echo -e "Custom environment variables:\n$(env | grep TF_VAR_ | grep -v ' "TF_VAR_')"
-
- export ARM_SUBSCRIPTION_ID=$(az login --identity --username $MSI_ID | jq -r '.[0] | .id')
- export ARM_TENANT_ID=$(az login --identity --username $MSI_ID | jq -r '.[0] | .tenantId')
- export ARM_CLIENT_ID=$(az identity list | jq -r --arg MSI_ID "$MSI_ID" '.[] | select(.principalId == $MSI_ID) | .clientId')
- docker run --rm -v /var/run/docker.sock:/var/run/docker.sock -v $(pwd):/src -w /src --network=host -e TF_IN_AUTOMATION -e TF_VAR_enable_telemetry -e AVM_MOD_PATH=/src -e AVM_EXAMPLE=${{ matrix.example }} -e MSI_ID -e ARM_SUBSCRIPTION_ID -e ARM_TENANT_ID -e ARM_CLIENT_ID -e ARM_USE_MSI=true --env-file <(env | grep TF_VAR_ | grep -v ' "TF_VAR_') mcr.microsoft.com/azterraform:latest make test-example
-
- # This job is only run when all the previous jobs are successful.
- # We can use it for PR validation to ensure all examples have completed.
- testexamplescomplete:
- if: github.event.repository.name != 'terraform-azurerm-avm-template'
- runs-on: ubuntu-latest
- needs: testexamples
- steps:
- - run: echo "All tests passed"
+ name: test examples
+ on:
+ pull_request:
+ types: ['opened', 'reopened', 'synchronize']
+ merge_group:
+ workflow_dispatch:
+
+ jobs:
+ check:
+ runs-on: ubuntu-latest
+ steps:
+ - name: Checking for Fork
+ shell: pwsh
+ run: |
+ $isFork = "${{ github.event.pull_request.head.repo.fork }}"
+ if($isFork -eq "true") {
+ echo "### WARNING: This workflow is disabled for forked repositories. Please follow the [release branch process](https://azure.github.io/Azure-Verified-Modules/contributing/terraform/terraform-contribution-flow/#5-create-a-pull-request-to-the-upstream-repository) if end to end tests are required." >> $env:GITHUB_STEP_SUMMARY
+ }
+
+ run-e2e-tests:
+ if: github.event.repository.name != 'terraform-azurerm-avm-template' && github.event.pull_request.head.repo.fork == false
+ uses: Azure/terraform-azurerm-avm-template/.github/workflows/test-examples-template.yml@main
+ name: end to end
+ secrets: inherit
+ permissions:
+ id-token: write
+ contents: read
+
\ No newline at end of file
diff --git a/README.md b/README.md
index 96ec940..ec9a476 100644
--- a/README.md
+++ b/README.md
@@ -54,12 +54,14 @@ The following resources are used by this module:
- [azurerm_management_lock.this](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/resources/management_lock) (resource)
- [azurerm_monitor_diagnostic_setting.aks](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/resources/monitor_diagnostic_setting) (resource)
- [azurerm_role_assignment.acr](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/resources/role_assignment) (resource)
+- [azurerm_role_assignment.network_contributor_on_subnet](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/resources/role_assignment) (resource)
- [azurerm_user_assigned_identity.aks](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/resources/user_assigned_identity) (resource)
- [modtm_telemetry.telemetry](https://registry.terraform.io/providers/Azure/modtm/latest/docs/resources/telemetry) (resource)
- [null_resource.kubernetes_version_keeper](https://registry.terraform.io/providers/hashicorp/null/latest/docs/resources/resource) (resource)
- [random_string.acr_suffix](https://registry.terraform.io/providers/hashicorp/random/latest/docs/resources/string) (resource)
- [random_uuid.telemetry](https://registry.terraform.io/providers/hashicorp/random/latest/docs/resources/uuid) (resource)
- [azurerm_client_config.telemetry](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/data-sources/client_config) (data source)
+- [azurerm_user_assigned_identity.cluster_user_defined_identity](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/data-sources/user_assigned_identity) (data source)
- [local_file.compute_provider](https://registry.terraform.io/providers/hashicorp/local/latest/docs/data-sources/file) (data source)
- [local_file.locations](https://registry.terraform.io/providers/hashicorp/local/latest/docs/data-sources/file) (data source)
- [modtm_module_source.telemetry](https://registry.terraform.io/providers/Azure/modtm/latest/docs/data-sources/module_source) (data source)
@@ -180,6 +182,14 @@ Type: `string`
Default: `null`
+### [node\_labels](#input\_node\_labels)
+
+Description: (Optional) A map of Kubernetes labels which should be applied to nodes in this Node Pool.
+
+Type: `map(string)`
+
+Default: `{}`
+
### [node\_pools](#input\_node\_pools)
Description: A map of node pools that need to be created and attached on the Kubernetes cluster. The key of the map can be the name of the node pool, and the key must be static string. The value of the map is a `node_pool` block as defined below:
@@ -193,6 +203,7 @@ map(object({
mode = (Optional) Should this Node Pool be used for System or User resources? Possible values are `System` and `User`. Defaults to `User`.
os\_disk\_size\_gb = (Optional) The Agent Operating System disk size in GB. Changing this forces a new resource to be created.
tags = (Optional) A mapping of tags to assign to the resource. At this time there's a bug in the AKS API where Tags for a Node Pool are not stored in the correct case - you [may wish to use Terraform's `ignore_changes` functionality to ignore changes to the casing](https://www.terraform.io/language/meta-arguments/lifecycle#ignore_changess) until this is fixed in the AKS API.
+ labels = (Optional) A map of Kubernetes labels which should be applied to nodes in this Node Pool.
zones = (Optional) Specifies a list of Availability Zones in which this Kubernetes Cluster Node Pool should be located. Changing this forces a new Kubernetes Cluster Node Pool to be created.
}))
@@ -234,6 +245,7 @@ map(object({
mode = optional(string)
os_disk_size_gb = optional(number, null)
tags = optional(map(string), {})
+ labels = optional(map(string), {})
zones = optional(set(string))
}))
```
diff --git a/avm b/avm
index 1bbbe36..9716dfb 100755
--- a/avm
+++ b/avm
@@ -27,7 +27,7 @@ fi
# Check if we are running in a container
# If we are then just run make directly
if [ -z "$AVM_IN_CONTAINER" ]; then
- $CONTAINER_RUNTIME run --pull always --user "$(id -u):$(id -g)" --rm $AZURE_VOLUME -v /etc/passwd:/etc/passwd -v /etc/group:/etc/group -v "$(pwd)":/src -w /src -e GITHUB_REPOSITORY -e GITHUB_REPOSITORY_OWNER mcr.microsoft.com/azterraform make "$1"
+ $CONTAINER_RUNTIME run --pull always --user "$(id -u):$(id -g)" --rm $AZURE_VOLUME -v /etc/passwd:/etc/passwd -v /etc/group:/etc/group -v "$(pwd)":/src -w /src -e GITHUB_REPOSITORY -e ARM_SUBSCRIPTION_ID -e GITHUB_REPOSITORY_OWNER mcr.microsoft.com/azterraform make "$1"
else
make "$1"
fi
diff --git a/avm.bat b/avm.bat
index 9138191..6b177be 100644
--- a/avm.bat
+++ b/avm.bat
@@ -18,6 +18,6 @@ IF "%~1"=="" (
)
REM Run the make target with CONTAINER_RUNTIME
-%CONTAINER_RUNTIME% run --pull always --rm -v "%cd%":/src -w /src --user "1000:1000" -e GITHUB_REPOSITORY -e GITHUB_REPOSITORY_OWNER mcr.microsoft.com/azterraform make %1
+%CONTAINER_RUNTIME% run --pull always --rm -v "%cd%":/src -w /src --user "1000:1000" -e ARM_SUBSCRIPTION_ID -e GITHUB_REPOSITORY -e GITHUB_REPOSITORY_OWNER mcr.microsoft.com/azterraform make %1
ENDLOCAL
diff --git a/locals.tf b/locals.tf
index 99a0b57..9845fbd 100644
--- a/locals.tf
+++ b/locals.tf
@@ -38,6 +38,7 @@ locals {
orchestrator_version = pool.orchestrator_version
max_count = pool.max_count
min_count = pool.min_count
+ labels = pool.labels
os_sku = pool.os_sku
mode = pool.mode
os_disk_size_gb = pool.os_disk_size_gb
@@ -64,7 +65,6 @@ locals {
this = {
type = "UserAssigned"
user_assigned_resource_ids = azurerm_user_assigned_identity.aks[*].id
-
}
}
}
diff --git a/main.tf b/main.tf
index 551c994..88dad2b 100644
--- a/main.tf
+++ b/main.tf
@@ -29,6 +29,23 @@ resource "azurerm_user_assigned_identity" "aks" {
tags = var.tags
}
+data "azurerm_user_assigned_identity" "cluster_user_defined_identity" {
+ count = length(var.managed_identities.user_assigned_resource_ids) > 0 ? length(var.managed_identities.user_assigned_resource_ids) : 0
+
+ # /subscriptions/00000000-0000-0000-0000-000000000000/resourcegroups/acceptanceTestResourceGroup1/providers/Microsoft.ManagedIdentity/userAssignedIdentities/testIdentity
+ # name is obtained from the above string which is the user assigned resource id - reference https://github.com/Azure/terraform-azurerm-aks/blob/decb533e2785f965673698b0ac9949faca963f68/role_assignments.tf#L11
+ name = split("/", tolist(var.managed_identities.user_assigned_resource_ids)[count.index])[8]
+ resource_group_name = split("/", tolist(var.managed_identities.user_assigned_resource_ids)[count.index])[4]
+}
+
+resource "azurerm_role_assignment" "network_contributor_on_subnet" {
+ # Use the principal_id from the user assigned identity if it exists, otherwise use the principal_id from the AKS cluster
+ # reference https://github.com/Azure/terraform-azurerm-aks/blob/decb533e2785f965673698b0ac9949faca963f68/role_assignments.tf#L27
+ principal_id = azurerm_kubernetes_cluster.this.identity.principal_id
+ scope = module.avm_res_network_virtualnetwork.subnets["subnet"].resource_id
+ role_definition_name = "Network Contributor"
+}
+
resource "azurerm_kubernetes_cluster" "this" {
location = var.location
name = "aks-${var.name}"
@@ -54,6 +71,7 @@ resource "azurerm_kubernetes_cluster" "this" {
max_count = 9
max_pods = 110
min_count = 3
+ node_labels = var.node_labels
orchestrator_version = var.orchestrator_version
os_sku = "Ubuntu"
tags = merge(var.tags, var.agents_tags)
@@ -237,7 +255,6 @@ resource "azurerm_kubernetes_cluster_node_pool" "this" {
enable_auto_scaling = true
max_count = each.value.max_count
min_count = each.value.min_count
- mode = each.value.mode
orchestrator_version = each.value.orchestrator_version
os_disk_size_gb = each.value.os_disk_size_gb
os_sku = each.value.os_sku
@@ -284,4 +301,4 @@ module "avm_res_network_virtualnetwork" {
address_prefixes = var.node_cidr != null ? [var.node_cidr] : ["10.31.0.0/16"]
}
}
-}
+}
\ No newline at end of file
diff --git a/variables.tf b/variables.tf
index a048089..7c04631 100644
--- a/variables.tf
+++ b/variables.tf
@@ -99,6 +99,12 @@ variable "node_cidr" {
description = "(Optional) The CIDR to use for node IPs in the Kubernetes cluster. Changing this forces a new resource to be created."
}
+variable "node_labels" {
+ type = map(string)
+ default = {}
+ description = "(Optional) A map of Kubernetes labels which should be applied to nodes in this Node Pool."
+}
+
variable "node_pools" {
type = map(object({
name = string
@@ -111,6 +117,7 @@ variable "node_pools" {
mode = optional(string)
os_disk_size_gb = optional(number, null)
tags = optional(map(string), {})
+ labels = optional(map(string), {})
zones = optional(set(string))
}))
default = {}
@@ -126,6 +133,7 @@ map(object({
mode = (Optional) Should this Node Pool be used for System or User resources? Possible values are `System` and `User`. Defaults to `User`.
os_disk_size_gb = (Optional) The Agent Operating System disk size in GB. Changing this forces a new resource to be created.
tags = (Optional) A mapping of tags to assign to the resource. At this time there's a bug in the AKS API where Tags for a Node Pool are not stored in the correct case - you [may wish to use Terraform's `ignore_changes` functionality to ignore changes to the casing](https://www.terraform.io/language/meta-arguments/lifecycle#ignore_changess) until this is fixed in the AKS API.
+ labels = (Optional) A map of Kubernetes labels which should be applied to nodes in this Node Pool.
zones = (Optional) Specifies a list of Availability Zones in which this Kubernetes Cluster Node Pool should be located. Changing this forces a new Kubernetes Cluster Node Pool to be created.
}))