Skip to content

Commit

Permalink
align node pool variables
Browse files Browse the repository at this point in the history
  • Loading branch information
nellyk committed Oct 1, 2024
2 parents b4768d5 + 3c521fd commit c754fed
Show file tree
Hide file tree
Showing 7 changed files with 70 additions and 103 deletions.
126 changes: 28 additions & 98 deletions .github/workflows/e2e.yml
Original file line number Diff line number Diff line change
@@ -1,99 +1,29 @@
---
name: e2e test

on:
pull_request:
types: ['opened', 'reopened', 'synchronize']
merge_group:
workflow_dispatch:

permissions:
contents: read
id-token: write

jobs:
getexamples:
if: github.event.repository.name != 'terraform-azurerm-avm-template'
runs-on: ubuntu-latest
outputs:
examples: ${{ steps.getexamples.outputs.examples }}
steps:
- uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 #v4.1.7
- name: get examples
id: getexamples
uses: Azure/terraform-azurerm-avm-template/.github/actions/e2e-getexamples@main
with:
github-token: ${{ secrets.GITHUB_TOKEN }}

testexamples:
if: github.event.repository.name != 'terraform-azurerm-avm-template'
runs-on: [ self-hosted, 1ES.Pool=terraform-azurerm-avm-ptn-aks-production ]
needs: getexamples
environment: test
env:
TF_IN_AUTOMATION: 1
TF_VAR_enable_telemetry: false
strategy:
matrix:
example: ${{ fromJson(needs.getexamples.outputs.examples) }}
fail-fast: false
steps:
- uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 #v4.1.7

- name: Test example
shell: bash
env:
SECRETS_CONTEXT: ${{ toJson(secrets) }}
VARS_CONTEXT: ${{ toJson(vars) }}
run: |
set -e
MAX_RETRIES=10
RETRY_COUNT=0
until [ $RETRY_COUNT -ge $MAX_RETRIES ]
do
az login --identity --username $MSI_ID > /dev/null && break
RETRY_COUNT=$[$RETRY_COUNT+1]
sleep 10
done
if [ $RETRY_COUNT -eq $MAX_RETRIES ]; then
echo "Failed to login after $MAX_RETRIES attempts."
exit 1
fi
declare -A secrets
eval "$(echo $SECRETS_CONTEXT | jq -r 'to_entries[] | @sh "secrets[\(.key|tostring)]=\(.value|tostring)"')"
declare -A variables
eval "$(echo $VARS_CONTEXT | jq -r 'to_entries[] | @sh "variables[\(.key|tostring)]=\(.value|tostring)"')"
for key in "${!secrets[@]}"; do
if [[ $key = \TF_VAR_* ]]; then
lowerKey=$(echo "$key" | tr '[:upper:]' '[:lower:]')
finalKey=${lowerKey/tf_var_/TF_VAR_}
export "$finalKey"="${secrets[$key]}"
fi
done
for key in "${!variables[@]}"; do
if [[ $key = \TF_VAR_* ]]; then
lowerKey=$(echo "$key" | tr '[:upper:]' '[:lower:]')
finalKey=${lowerKey/tf_var_/TF_VAR_}
export "$finalKey"="${variables[$key]}"
fi
done
echo -e "Custom environment variables:\n$(env | grep TF_VAR_ | grep -v ' "TF_VAR_')"
export ARM_SUBSCRIPTION_ID=$(az login --identity --username $MSI_ID | jq -r '.[0] | .id')
export ARM_TENANT_ID=$(az login --identity --username $MSI_ID | jq -r '.[0] | .tenantId')
export ARM_CLIENT_ID=$(az identity list | jq -r --arg MSI_ID "$MSI_ID" '.[] | select(.principalId == $MSI_ID) | .clientId')
docker run --rm -v /var/run/docker.sock:/var/run/docker.sock -v $(pwd):/src -w /src --network=host -e TF_IN_AUTOMATION -e TF_VAR_enable_telemetry -e AVM_MOD_PATH=/src -e AVM_EXAMPLE=${{ matrix.example }} -e MSI_ID -e ARM_SUBSCRIPTION_ID -e ARM_TENANT_ID -e ARM_CLIENT_ID -e ARM_USE_MSI=true --env-file <(env | grep TF_VAR_ | grep -v ' "TF_VAR_') mcr.microsoft.com/azterraform:latest make test-example
# This job is only run when all the previous jobs are successful.
# We can use it for PR validation to ensure all examples have completed.
testexamplescomplete:
if: github.event.repository.name != 'terraform-azurerm-avm-template'
runs-on: ubuntu-latest
needs: testexamples
steps:
- run: echo "All tests passed"
name: test examples
on:
pull_request:
types: ['opened', 'reopened', 'synchronize']
merge_group:
workflow_dispatch:

jobs:
check:
runs-on: ubuntu-latest
steps:
- name: Checking for Fork
shell: pwsh
run: |
$isFork = "${{ github.event.pull_request.head.repo.fork }}"
if($isFork -eq "true") {
echo "### WARNING: This workflow is disabled for forked repositories. Please follow the [release branch process](https://azure.github.io/Azure-Verified-Modules/contributing/terraform/terraform-contribution-flow/#5-create-a-pull-request-to-the-upstream-repository) if end to end tests are required." >> $env:GITHUB_STEP_SUMMARY
}
run-e2e-tests:
if: github.event.repository.name != 'terraform-azurerm-avm-template' && github.event.pull_request.head.repo.fork == false
uses: Azure/terraform-azurerm-avm-template/.github/workflows/test-examples-template.yml@main
name: end to end
secrets: inherit
permissions:
id-token: write
contents: read

12 changes: 12 additions & 0 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -54,12 +54,14 @@ The following resources are used by this module:
- [azurerm_management_lock.this](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/resources/management_lock) (resource)
- [azurerm_monitor_diagnostic_setting.aks](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/resources/monitor_diagnostic_setting) (resource)
- [azurerm_role_assignment.acr](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/resources/role_assignment) (resource)
- [azurerm_role_assignment.network_contributor_on_subnet](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/resources/role_assignment) (resource)
- [azurerm_user_assigned_identity.aks](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/resources/user_assigned_identity) (resource)
- [modtm_telemetry.telemetry](https://registry.terraform.io/providers/Azure/modtm/latest/docs/resources/telemetry) (resource)
- [null_resource.kubernetes_version_keeper](https://registry.terraform.io/providers/hashicorp/null/latest/docs/resources/resource) (resource)
- [random_string.acr_suffix](https://registry.terraform.io/providers/hashicorp/random/latest/docs/resources/string) (resource)
- [random_uuid.telemetry](https://registry.terraform.io/providers/hashicorp/random/latest/docs/resources/uuid) (resource)
- [azurerm_client_config.telemetry](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/data-sources/client_config) (data source)
- [azurerm_user_assigned_identity.cluster_user_defined_identity](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/data-sources/user_assigned_identity) (data source)
- [local_file.compute_provider](https://registry.terraform.io/providers/hashicorp/local/latest/docs/data-sources/file) (data source)
- [local_file.locations](https://registry.terraform.io/providers/hashicorp/local/latest/docs/data-sources/file) (data source)
- [modtm_module_source.telemetry](https://registry.terraform.io/providers/Azure/modtm/latest/docs/data-sources/module_source) (data source)
Expand Down Expand Up @@ -180,6 +182,14 @@ Type: `string`

Default: `null`

### <a name="input_node_labels"></a> [node\_labels](#input\_node\_labels)

Description: (Optional) A map of Kubernetes labels which should be applied to nodes in this Node Pool.

Type: `map(string)`

Default: `{}`

### <a name="input_node_pools"></a> [node\_pools](#input\_node\_pools)

Description: A map of node pools that need to be created and attached on the Kubernetes cluster. The key of the map can be the name of the node pool, and the key must be static string. The value of the map is a `node_pool` block as defined below:
Expand All @@ -193,6 +203,7 @@ map(object({
mode = (Optional) Should this Node Pool be used for System or User resources? Possible values are `System` and `User`. Defaults to `User`.
os\_disk\_size\_gb = (Optional) The Agent Operating System disk size in GB. Changing this forces a new resource to be created.
tags = (Optional) A mapping of tags to assign to the resource. At this time there's a bug in the AKS API where Tags for a Node Pool are not stored in the correct case - you [may wish to use Terraform's `ignore_changes` functionality to ignore changes to the casing](https://www.terraform.io/language/meta-arguments/lifecycle#ignore_changess) until this is fixed in the AKS API.
labels = (Optional) A map of Kubernetes labels which should be applied to nodes in this Node Pool.
zones = (Optional) Specifies a list of Availability Zones in which this Kubernetes Cluster Node Pool should be located. Changing this forces a new Kubernetes Cluster Node Pool to be created.
}))

Expand Down Expand Up @@ -234,6 +245,7 @@ map(object({
mode = optional(string)
os_disk_size_gb = optional(number, null)
tags = optional(map(string), {})
labels = optional(map(string), {})
zones = optional(set(string))
}))
```
Expand Down
2 changes: 1 addition & 1 deletion avm
Original file line number Diff line number Diff line change
Expand Up @@ -27,7 +27,7 @@ fi
# Check if we are running in a container
# If we are then just run make directly
if [ -z "$AVM_IN_CONTAINER" ]; then
$CONTAINER_RUNTIME run --pull always --user "$(id -u):$(id -g)" --rm $AZURE_VOLUME -v /etc/passwd:/etc/passwd -v /etc/group:/etc/group -v "$(pwd)":/src -w /src -e GITHUB_REPOSITORY -e GITHUB_REPOSITORY_OWNER mcr.microsoft.com/azterraform make "$1"
$CONTAINER_RUNTIME run --pull always --user "$(id -u):$(id -g)" --rm $AZURE_VOLUME -v /etc/passwd:/etc/passwd -v /etc/group:/etc/group -v "$(pwd)":/src -w /src -e GITHUB_REPOSITORY -e ARM_SUBSCRIPTION_ID -e GITHUB_REPOSITORY_OWNER mcr.microsoft.com/azterraform make "$1"
else
make "$1"
fi
2 changes: 1 addition & 1 deletion avm.bat
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,6 @@ IF "%~1"=="" (
)

REM Run the make target with CONTAINER_RUNTIME
%CONTAINER_RUNTIME% run --pull always --rm -v "%cd%":/src -w /src --user "1000:1000" -e GITHUB_REPOSITORY -e GITHUB_REPOSITORY_OWNER mcr.microsoft.com/azterraform make %1
%CONTAINER_RUNTIME% run --pull always --rm -v "%cd%":/src -w /src --user "1000:1000" -e ARM_SUBSCRIPTION_ID -e GITHUB_REPOSITORY -e GITHUB_REPOSITORY_OWNER mcr.microsoft.com/azterraform make %1

ENDLOCAL
2 changes: 1 addition & 1 deletion locals.tf
Original file line number Diff line number Diff line change
Expand Up @@ -38,6 +38,7 @@ locals {
orchestrator_version = pool.orchestrator_version
max_count = pool.max_count
min_count = pool.min_count
labels = pool.labels
os_sku = pool.os_sku
mode = pool.mode
os_disk_size_gb = pool.os_disk_size_gb
Expand All @@ -64,7 +65,6 @@ locals {
this = {
type = "UserAssigned"
user_assigned_resource_ids = azurerm_user_assigned_identity.aks[*].id

}
}
}
Expand Down
21 changes: 19 additions & 2 deletions main.tf
Original file line number Diff line number Diff line change
Expand Up @@ -29,6 +29,23 @@ resource "azurerm_user_assigned_identity" "aks" {
tags = var.tags
}

data "azurerm_user_assigned_identity" "cluster_user_defined_identity" {
count = length(var.managed_identities.user_assigned_resource_ids) > 0 ? length(var.managed_identities.user_assigned_resource_ids) : 0

# /subscriptions/00000000-0000-0000-0000-000000000000/resourcegroups/acceptanceTestResourceGroup1/providers/Microsoft.ManagedIdentity/userAssignedIdentities/testIdentity
# name is obtained from the above string which is the user assigned resource id - reference https://github.com/Azure/terraform-azurerm-aks/blob/decb533e2785f965673698b0ac9949faca963f68/role_assignments.tf#L11
name = split("/", tolist(var.managed_identities.user_assigned_resource_ids)[count.index])[8]
resource_group_name = split("/", tolist(var.managed_identities.user_assigned_resource_ids)[count.index])[4]
}

resource "azurerm_role_assignment" "network_contributor_on_subnet" {
# Use the principal_id from the user assigned identity if it exists, otherwise use the principal_id from the AKS cluster
# reference https://github.com/Azure/terraform-azurerm-aks/blob/decb533e2785f965673698b0ac9949faca963f68/role_assignments.tf#L27
principal_id = azurerm_kubernetes_cluster.this.identity.principal_id
scope = module.avm_res_network_virtualnetwork.subnets["subnet"].resource_id
role_definition_name = "Network Contributor"
}

resource "azurerm_kubernetes_cluster" "this" {
location = var.location
name = "aks-${var.name}"
Expand All @@ -54,6 +71,7 @@ resource "azurerm_kubernetes_cluster" "this" {
max_count = 9
max_pods = 110
min_count = 3
node_labels = var.node_labels
orchestrator_version = var.orchestrator_version
os_sku = "Ubuntu"
tags = merge(var.tags, var.agents_tags)
Expand Down Expand Up @@ -237,7 +255,6 @@ resource "azurerm_kubernetes_cluster_node_pool" "this" {
enable_auto_scaling = true
max_count = each.value.max_count
min_count = each.value.min_count
mode = each.value.mode
orchestrator_version = each.value.orchestrator_version
os_disk_size_gb = each.value.os_disk_size_gb
os_sku = each.value.os_sku
Expand Down Expand Up @@ -284,4 +301,4 @@ module "avm_res_network_virtualnetwork" {
address_prefixes = var.node_cidr != null ? [var.node_cidr] : ["10.31.0.0/16"]
}
}
}
}
8 changes: 8 additions & 0 deletions variables.tf
Original file line number Diff line number Diff line change
Expand Up @@ -99,6 +99,12 @@ variable "node_cidr" {
description = "(Optional) The CIDR to use for node IPs in the Kubernetes cluster. Changing this forces a new resource to be created."
}

variable "node_labels" {
type = map(string)
default = {}
description = "(Optional) A map of Kubernetes labels which should be applied to nodes in this Node Pool."
}

variable "node_pools" {
type = map(object({
name = string
Expand All @@ -111,6 +117,7 @@ variable "node_pools" {
mode = optional(string)
os_disk_size_gb = optional(number, null)
tags = optional(map(string), {})
labels = optional(map(string), {})
zones = optional(set(string))
}))
default = {}
Expand All @@ -126,6 +133,7 @@ map(object({
mode = (Optional) Should this Node Pool be used for System or User resources? Possible values are `System` and `User`. Defaults to `User`.
os_disk_size_gb = (Optional) The Agent Operating System disk size in GB. Changing this forces a new resource to be created.
tags = (Optional) A mapping of tags to assign to the resource. At this time there's a bug in the AKS API where Tags for a Node Pool are not stored in the correct case - you [may wish to use Terraform's `ignore_changes` functionality to ignore changes to the casing](https://www.terraform.io/language/meta-arguments/lifecycle#ignore_changess) until this is fixed in the AKS API.
labels = (Optional) A map of Kubernetes labels which should be applied to nodes in this Node Pool.
zones = (Optional) Specifies a list of Availability Zones in which this Kubernetes Cluster Node Pool should be located. Changing this forces a new Kubernetes Cluster Node Pool to be created.
}))
Expand Down

0 comments on commit c754fed

Please sign in to comment.