diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml index 2e32634..7724649 100644 --- a/.github/workflows/e2e.yml +++ b/.github/workflows/e2e.yml @@ -1,99 +1,29 @@ --- -name: e2e test - -on: - pull_request: - types: ['opened', 'reopened', 'synchronize'] - merge_group: - workflow_dispatch: - -permissions: - contents: read - id-token: write - -jobs: - getexamples: - if: github.event.repository.name != 'terraform-azurerm-avm-template' - runs-on: ubuntu-latest - outputs: - examples: ${{ steps.getexamples.outputs.examples }} - steps: - - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 #v4.1.7 - - name: get examples - id: getexamples - uses: Azure/terraform-azurerm-avm-template/.github/actions/e2e-getexamples@main - with: - github-token: ${{ secrets.GITHUB_TOKEN }} - - testexamples: - if: github.event.repository.name != 'terraform-azurerm-avm-template' - runs-on: [ self-hosted, 1ES.Pool=terraform-azurerm-avm-ptn-aks-production ] - needs: getexamples - environment: test - env: - TF_IN_AUTOMATION: 1 - TF_VAR_enable_telemetry: false - strategy: - matrix: - example: ${{ fromJson(needs.getexamples.outputs.examples) }} - fail-fast: false - steps: - - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 #v4.1.7 - - - name: Test example - shell: bash - env: - SECRETS_CONTEXT: ${{ toJson(secrets) }} - VARS_CONTEXT: ${{ toJson(vars) }} - run: | - set -e - MAX_RETRIES=10 - RETRY_COUNT=0 - until [ $RETRY_COUNT -ge $MAX_RETRIES ] - do - az login --identity --username $MSI_ID > /dev/null && break - RETRY_COUNT=$[$RETRY_COUNT+1] - sleep 10 - done - if [ $RETRY_COUNT -eq $MAX_RETRIES ]; then - echo "Failed to login after $MAX_RETRIES attempts." - exit 1 - fi - - declare -A secrets - eval "$(echo $SECRETS_CONTEXT | jq -r 'to_entries[] | @sh "secrets[\(.key|tostring)]=\(.value|tostring)"')" - - declare -A variables - eval "$(echo $VARS_CONTEXT | jq -r 'to_entries[] | @sh "variables[\(.key|tostring)]=\(.value|tostring)"')" - - for key in "${!secrets[@]}"; do - if [[ $key = \TF_VAR_* ]]; then - lowerKey=$(echo "$key" | tr '[:upper:]' '[:lower:]') - finalKey=${lowerKey/tf_var_/TF_VAR_} - export "$finalKey"="${secrets[$key]}" - fi - done - - for key in "${!variables[@]}"; do - if [[ $key = \TF_VAR_* ]]; then - lowerKey=$(echo "$key" | tr '[:upper:]' '[:lower:]') - finalKey=${lowerKey/tf_var_/TF_VAR_} - export "$finalKey"="${variables[$key]}" - fi - done - - echo -e "Custom environment variables:\n$(env | grep TF_VAR_ | grep -v ' "TF_VAR_')" - - export ARM_SUBSCRIPTION_ID=$(az login --identity --username $MSI_ID | jq -r '.[0] | .id') - export ARM_TENANT_ID=$(az login --identity --username $MSI_ID | jq -r '.[0] | .tenantId') - export ARM_CLIENT_ID=$(az identity list | jq -r --arg MSI_ID "$MSI_ID" '.[] | select(.principalId == $MSI_ID) | .clientId') - docker run --rm -v /var/run/docker.sock:/var/run/docker.sock -v $(pwd):/src -w /src --network=host -e TF_IN_AUTOMATION -e TF_VAR_enable_telemetry -e AVM_MOD_PATH=/src -e AVM_EXAMPLE=${{ matrix.example }} -e MSI_ID -e ARM_SUBSCRIPTION_ID -e ARM_TENANT_ID -e ARM_CLIENT_ID -e ARM_USE_MSI=true --env-file <(env | grep TF_VAR_ | grep -v ' "TF_VAR_') mcr.microsoft.com/azterraform:latest make test-example - - # This job is only run when all the previous jobs are successful. - # We can use it for PR validation to ensure all examples have completed. - testexamplescomplete: - if: github.event.repository.name != 'terraform-azurerm-avm-template' - runs-on: ubuntu-latest - needs: testexamples - steps: - - run: echo "All tests passed" + name: test examples + on: + pull_request: + types: ['opened', 'reopened', 'synchronize'] + merge_group: + workflow_dispatch: + + jobs: + check: + runs-on: ubuntu-latest + steps: + - name: Checking for Fork + shell: pwsh + run: | + $isFork = "${{ github.event.pull_request.head.repo.fork }}" + if($isFork -eq "true") { + echo "### WARNING: This workflow is disabled for forked repositories. Please follow the [release branch process](https://azure.github.io/Azure-Verified-Modules/contributing/terraform/terraform-contribution-flow/#5-create-a-pull-request-to-the-upstream-repository) if end to end tests are required." >> $env:GITHUB_STEP_SUMMARY + } + + run-e2e-tests: + if: github.event.repository.name != 'terraform-azurerm-avm-template' && github.event.pull_request.head.repo.fork == false + uses: Azure/terraform-azurerm-avm-template/.github/workflows/test-examples-template.yml@main + name: end to end + secrets: inherit + permissions: + id-token: write + contents: read + \ No newline at end of file diff --git a/README.md b/README.md index 96ec940..ec9a476 100644 --- a/README.md +++ b/README.md @@ -54,12 +54,14 @@ The following resources are used by this module: - [azurerm_management_lock.this](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/resources/management_lock) (resource) - [azurerm_monitor_diagnostic_setting.aks](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/resources/monitor_diagnostic_setting) (resource) - [azurerm_role_assignment.acr](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/resources/role_assignment) (resource) +- [azurerm_role_assignment.network_contributor_on_subnet](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/resources/role_assignment) (resource) - [azurerm_user_assigned_identity.aks](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/resources/user_assigned_identity) (resource) - [modtm_telemetry.telemetry](https://registry.terraform.io/providers/Azure/modtm/latest/docs/resources/telemetry) (resource) - [null_resource.kubernetes_version_keeper](https://registry.terraform.io/providers/hashicorp/null/latest/docs/resources/resource) (resource) - [random_string.acr_suffix](https://registry.terraform.io/providers/hashicorp/random/latest/docs/resources/string) (resource) - [random_uuid.telemetry](https://registry.terraform.io/providers/hashicorp/random/latest/docs/resources/uuid) (resource) - [azurerm_client_config.telemetry](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/data-sources/client_config) (data source) +- [azurerm_user_assigned_identity.cluster_user_defined_identity](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/data-sources/user_assigned_identity) (data source) - [local_file.compute_provider](https://registry.terraform.io/providers/hashicorp/local/latest/docs/data-sources/file) (data source) - [local_file.locations](https://registry.terraform.io/providers/hashicorp/local/latest/docs/data-sources/file) (data source) - [modtm_module_source.telemetry](https://registry.terraform.io/providers/Azure/modtm/latest/docs/data-sources/module_source) (data source) @@ -180,6 +182,14 @@ Type: `string` Default: `null` +### [node\_labels](#input\_node\_labels) + +Description: (Optional) A map of Kubernetes labels which should be applied to nodes in this Node Pool. + +Type: `map(string)` + +Default: `{}` + ### [node\_pools](#input\_node\_pools) Description: A map of node pools that need to be created and attached on the Kubernetes cluster. The key of the map can be the name of the node pool, and the key must be static string. The value of the map is a `node_pool` block as defined below: @@ -193,6 +203,7 @@ map(object({ mode = (Optional) Should this Node Pool be used for System or User resources? Possible values are `System` and `User`. Defaults to `User`. os\_disk\_size\_gb = (Optional) The Agent Operating System disk size in GB. Changing this forces a new resource to be created. tags = (Optional) A mapping of tags to assign to the resource. At this time there's a bug in the AKS API where Tags for a Node Pool are not stored in the correct case - you [may wish to use Terraform's `ignore_changes` functionality to ignore changes to the casing](https://www.terraform.io/language/meta-arguments/lifecycle#ignore_changess) until this is fixed in the AKS API. + labels = (Optional) A map of Kubernetes labels which should be applied to nodes in this Node Pool. zones = (Optional) Specifies a list of Availability Zones in which this Kubernetes Cluster Node Pool should be located. Changing this forces a new Kubernetes Cluster Node Pool to be created. })) @@ -234,6 +245,7 @@ map(object({ mode = optional(string) os_disk_size_gb = optional(number, null) tags = optional(map(string), {}) + labels = optional(map(string), {}) zones = optional(set(string)) })) ``` diff --git a/avm b/avm index 1bbbe36..9716dfb 100755 --- a/avm +++ b/avm @@ -27,7 +27,7 @@ fi # Check if we are running in a container # If we are then just run make directly if [ -z "$AVM_IN_CONTAINER" ]; then - $CONTAINER_RUNTIME run --pull always --user "$(id -u):$(id -g)" --rm $AZURE_VOLUME -v /etc/passwd:/etc/passwd -v /etc/group:/etc/group -v "$(pwd)":/src -w /src -e GITHUB_REPOSITORY -e GITHUB_REPOSITORY_OWNER mcr.microsoft.com/azterraform make "$1" + $CONTAINER_RUNTIME run --pull always --user "$(id -u):$(id -g)" --rm $AZURE_VOLUME -v /etc/passwd:/etc/passwd -v /etc/group:/etc/group -v "$(pwd)":/src -w /src -e GITHUB_REPOSITORY -e ARM_SUBSCRIPTION_ID -e GITHUB_REPOSITORY_OWNER mcr.microsoft.com/azterraform make "$1" else make "$1" fi diff --git a/avm.bat b/avm.bat index 9138191..6b177be 100644 --- a/avm.bat +++ b/avm.bat @@ -18,6 +18,6 @@ IF "%~1"=="" ( ) REM Run the make target with CONTAINER_RUNTIME -%CONTAINER_RUNTIME% run --pull always --rm -v "%cd%":/src -w /src --user "1000:1000" -e GITHUB_REPOSITORY -e GITHUB_REPOSITORY_OWNER mcr.microsoft.com/azterraform make %1 +%CONTAINER_RUNTIME% run --pull always --rm -v "%cd%":/src -w /src --user "1000:1000" -e ARM_SUBSCRIPTION_ID -e GITHUB_REPOSITORY -e GITHUB_REPOSITORY_OWNER mcr.microsoft.com/azterraform make %1 ENDLOCAL diff --git a/locals.tf b/locals.tf index 99a0b57..9845fbd 100644 --- a/locals.tf +++ b/locals.tf @@ -38,6 +38,7 @@ locals { orchestrator_version = pool.orchestrator_version max_count = pool.max_count min_count = pool.min_count + labels = pool.labels os_sku = pool.os_sku mode = pool.mode os_disk_size_gb = pool.os_disk_size_gb @@ -64,7 +65,6 @@ locals { this = { type = "UserAssigned" user_assigned_resource_ids = azurerm_user_assigned_identity.aks[*].id - } } } diff --git a/main.tf b/main.tf index 551c994..88dad2b 100644 --- a/main.tf +++ b/main.tf @@ -29,6 +29,23 @@ resource "azurerm_user_assigned_identity" "aks" { tags = var.tags } +data "azurerm_user_assigned_identity" "cluster_user_defined_identity" { + count = length(var.managed_identities.user_assigned_resource_ids) > 0 ? length(var.managed_identities.user_assigned_resource_ids) : 0 + + # /subscriptions/00000000-0000-0000-0000-000000000000/resourcegroups/acceptanceTestResourceGroup1/providers/Microsoft.ManagedIdentity/userAssignedIdentities/testIdentity + # name is obtained from the above string which is the user assigned resource id - reference https://github.com/Azure/terraform-azurerm-aks/blob/decb533e2785f965673698b0ac9949faca963f68/role_assignments.tf#L11 + name = split("/", tolist(var.managed_identities.user_assigned_resource_ids)[count.index])[8] + resource_group_name = split("/", tolist(var.managed_identities.user_assigned_resource_ids)[count.index])[4] +} + +resource "azurerm_role_assignment" "network_contributor_on_subnet" { + # Use the principal_id from the user assigned identity if it exists, otherwise use the principal_id from the AKS cluster + # reference https://github.com/Azure/terraform-azurerm-aks/blob/decb533e2785f965673698b0ac9949faca963f68/role_assignments.tf#L27 + principal_id = azurerm_kubernetes_cluster.this.identity.principal_id + scope = module.avm_res_network_virtualnetwork.subnets["subnet"].resource_id + role_definition_name = "Network Contributor" +} + resource "azurerm_kubernetes_cluster" "this" { location = var.location name = "aks-${var.name}" @@ -54,6 +71,7 @@ resource "azurerm_kubernetes_cluster" "this" { max_count = 9 max_pods = 110 min_count = 3 + node_labels = var.node_labels orchestrator_version = var.orchestrator_version os_sku = "Ubuntu" tags = merge(var.tags, var.agents_tags) @@ -237,7 +255,6 @@ resource "azurerm_kubernetes_cluster_node_pool" "this" { enable_auto_scaling = true max_count = each.value.max_count min_count = each.value.min_count - mode = each.value.mode orchestrator_version = each.value.orchestrator_version os_disk_size_gb = each.value.os_disk_size_gb os_sku = each.value.os_sku @@ -284,4 +301,4 @@ module "avm_res_network_virtualnetwork" { address_prefixes = var.node_cidr != null ? [var.node_cidr] : ["10.31.0.0/16"] } } -} +} \ No newline at end of file diff --git a/variables.tf b/variables.tf index a048089..7c04631 100644 --- a/variables.tf +++ b/variables.tf @@ -99,6 +99,12 @@ variable "node_cidr" { description = "(Optional) The CIDR to use for node IPs in the Kubernetes cluster. Changing this forces a new resource to be created." } +variable "node_labels" { + type = map(string) + default = {} + description = "(Optional) A map of Kubernetes labels which should be applied to nodes in this Node Pool." +} + variable "node_pools" { type = map(object({ name = string @@ -111,6 +117,7 @@ variable "node_pools" { mode = optional(string) os_disk_size_gb = optional(number, null) tags = optional(map(string), {}) + labels = optional(map(string), {}) zones = optional(set(string)) })) default = {} @@ -126,6 +133,7 @@ map(object({ mode = (Optional) Should this Node Pool be used for System or User resources? Possible values are `System` and `User`. Defaults to `User`. os_disk_size_gb = (Optional) The Agent Operating System disk size in GB. Changing this forces a new resource to be created. tags = (Optional) A mapping of tags to assign to the resource. At this time there's a bug in the AKS API where Tags for a Node Pool are not stored in the correct case - you [may wish to use Terraform's `ignore_changes` functionality to ignore changes to the casing](https://www.terraform.io/language/meta-arguments/lifecycle#ignore_changess) until this is fixed in the AKS API. + labels = (Optional) A map of Kubernetes labels which should be applied to nodes in this Node Pool. zones = (Optional) Specifies a list of Availability Zones in which this Kubernetes Cluster Node Pool should be located. Changing this forces a new Kubernetes Cluster Node Pool to be created. }))