diff --git a/azure_devops_agent_custom_image/packer/script-config.sh b/azure_devops_agent_custom_image/packer/script-config.sh index 85aceec1..f90f81c4 100644 --- a/azure_devops_agent_custom_image/packer/script-config.sh +++ b/azure_devops_agent_custom_image/packer/script-config.sh @@ -14,18 +14,19 @@ function check_command(){ # install zip unzip ca-certificates curl wget apt-transport-https lsb-release gnupg jq apt-get -y update apt-get -y --allow-unauthenticated install zip unzip ca-certificates curl wget apt-transport-https lsb-release gnupg jq - check_command "zip" check_command "unzip" check_command "jq" # install az cli curl -sL https://aka.ms/InstallAzureCLIDeb | bash +check_command "az" az acr helm install-cli -y --client-version 3.12.0 +check_command "helm" az aks install-cli --client-version 1.25.10 --kubelogin-version 0.0.29 - +check_command "kubectl" # setup Docker installation from https://docs.docker.com/engine/install/ubuntu/ curl -fsSL https://download.docker.com/linux/ubuntu/gpg | @@ -38,22 +39,17 @@ echo \ apt-get -y update apt-get -y install python3-pip -apt-get -y --allow-unauthenticated install docker-ce docker-ce-cli containerd.io docker-compose-plugin - +check_command "python3" -check_command "az" -check_command "kubectl" +apt-get -y --allow-unauthenticated install docker-ce docker-ce-cli containerd.io docker-compose-plugin check_command "docker" -check_command "helm" -check_command "python3" + # install yq from https://github.com/mikefarah/yq#install YQ_VERSION="v4.33.3" YQ_BINARY="yq_linux_amd64" wget https://github.com/mikefarah/yq/releases/download/${YQ_VERSION}/${YQ_BINARY}.tar.gz -O - |\ tar xz && mv ${YQ_BINARY} /usr/bin/yq - - check_command "yq" # install SOPS from https://github.com/mozilla/sops @@ -61,9 +57,14 @@ SOPS_VERSION="v3.7.3" SOPS_BINARY="3.7.3_amd64.deb" wget https://github.com/mozilla/sops/releases/download/v3.7.3/sops_3.7.3_amd64.deb apt install -y $PWD/sops_3.7.3_amd64.deb - check_command "sops" +VELERO_VERSION=v1.11.1 +wget https://github.com/vmware-tanzu/velero/releases/download/${VELERO_VERSION}/velero-${VELERO_VERSION}-linux-amd64.tar.gz && \ +tar -zxvf velero-${VELERO_VERSION}-linux-amd64.tar.gz && \ +sudo mv velero-${VELERO_VERSION}-linux-amd64/velero /usr/bin/velero +check_command "velero" + # prepare machine for k6 large load test sysctl -w net.ipv4.ip_local_port_range="1024 65535" diff --git a/kubernetes_cluster/99_variables.tf b/kubernetes_cluster/99_variables.tf index adaefa24..1880bf1e 100644 --- a/kubernetes_cluster/99_variables.tf +++ b/kubernetes_cluster/99_variables.tf @@ -340,3 +340,4 @@ variable "sec_storage_id" { variable "tags" { type = map(any) } + diff --git a/kubernetes_cluster/README.md b/kubernetes_cluster/README.md index 297a1807..bcaae3fc 100644 --- a/kubernetes_cluster/README.md +++ b/kubernetes_cluster/README.md @@ -13,11 +13,13 @@ By default the modules have a default set of metric alerts. * If you want is possible to add new **custom metrics alerts** using the varible: `custom_metric_alerts` * Or override the **default metrics alert** using the variable: `default_metric_alerts`. (is prefered to add new metrics) + + ## How to use it ### Variable definition example -```ts +```hcl # # ⛴ AKS PROD # @@ -437,7 +439,7 @@ variable "keda_helm_version" { ### Variables values -```ts +```hcl rg_vnet_aks_name = "dvopla-d-neu-dev01-aks-vnet-rg" vnet_aks_name = "dvopla-d-neu-dev01-aks-vnet" public_ip_aksoutbound_name = "dvopla-d-dev01-aksoutbound-pip-1" @@ -484,7 +486,7 @@ keda_helm_version = "2.6.2" ### AKS Cluster -```ts +```hcl resource "azurerm_resource_group" "rg_aks" { name = local.aks_rg_name location = var.location @@ -552,7 +554,7 @@ keda_helm_version = "2.6.2" service_cidr = "10.250.0.0/16" } # end network - + rbac_enabled = true aad_admin_group_ids = var.env_short == "d" ? [data.azuread_group.adgroup_admin.object_id, data.azuread_group.adgroup_developers.object_id, data.azuread_group.adgroup_externals.object_id] : [data.azuread_group.adgroup_admin.object_id] diff --git a/kubernetes_cluster_velero/01_main.tf b/kubernetes_cluster_velero/01_main.tf new file mode 100644 index 00000000..797a2a08 --- /dev/null +++ b/kubernetes_cluster_velero/01_main.tf @@ -0,0 +1,160 @@ +locals { + sa_prefix = replace(replace(var.prefix, "-", ""), "_", "") +} + +data "azurerm_kubernetes_cluster" "aks_cluster" { + name = var.aks_cluster_name + resource_group_name = var.aks_cluster_rg +} + +module "velero_storage_account" { + source = "git::https://github.com/pagopa/terraform-azurerm-v3.git//storage_account?ref=v7.2.0" + + name = "${local.sa_prefix}velerosa" + account_kind = "BlobStorage" + account_tier = var.storage_account_tier + account_replication_type = var.storage_account_replication_type + blob_versioning_enabled = true + resource_group_name = var.resource_group_name + location = var.location + allow_nested_items_to_be_public = false + advanced_threat_protection = true + enable_low_availability_alert = false + public_network_access_enabled = var.use_storage_private_endpoint ? false : true + tags = var.tags +} + +resource "azurerm_private_endpoint" "velero_storage_private_endpoint" { + count = var.use_storage_private_endpoint ? 1 : 0 + + name = "${var.prefix}-velerosa-private-endpoint" + location = var.location + resource_group_name = var.resource_group_name + subnet_id = var.private_endpoint_subnet_id + + private_dns_zone_group { + name = "${var.prefix}-velerosa-private-dns-zone-group" + private_dns_zone_ids = [var.storage_account_private_dns_zone_id] + } + + private_service_connection { + name = "${var.prefix}-velerosa-private-service-connection" + private_connection_resource_id = module.velero_storage_account.id + is_manual_connection = false + subresource_names = ["blob"] + } + + tags = var.tags +} + +resource "azurerm_storage_container" "velero_backup_container" { + name = "${var.prefix}-velero-backup" + storage_account_name = module.velero_storage_account.name + container_access_type = "private" + + depends_on = [azurerm_private_endpoint.velero_storage_private_endpoint] +} + +data "azuread_client_config" "current" {} + + +resource "azuread_application" "velero_application" { + display_name = "${var.prefix}-velero-application" + owners = [data.azuread_client_config.current.object_id] +} + +resource "azuread_application_password" "velero_application_password" { + application_object_id = azuread_application.velero_application.object_id +} + +resource "azuread_service_principal" "velero_sp" { + application_id = azuread_application.velero_application.application_id + owners = [data.azuread_client_config.current.object_id] +} + +resource "azuread_service_principal_password" "velero_principal_password" { + service_principal_id = azuread_service_principal.velero_sp.object_id +} + +resource "azurerm_role_assignment" "velero_sp_aks_role" { + scope = data.azurerm_kubernetes_cluster.aks_cluster.id #var.aks_cluster_id + role_definition_name = "Azure Kubernetes Service Cluster Admin Role" + principal_id = azuread_service_principal.velero_sp.object_id +} + +resource "azurerm_role_assignment" "velero_sp_storage_role" { + scope = module.velero_storage_account.id + role_definition_name = "Storage Account Contributor" + principal_id = azuread_service_principal.velero_sp.object_id +} + +resource "local_file" "credentials" { + + content = templatefile("${path.module}/velero-credentials.tpl", { + subscription_id = var.subscription_id + tenant_id = var.tenant_id + client_id = azuread_application.velero_application.application_id + client_secret = azuread_application_password.velero_application_password.value + backup_rg = var.resource_group_name + }) + filename = "${path.module}/credentials-velero.txt" + + lifecycle { + replace_triggered_by = [ + azurerm_storage_container.velero_backup_container, + azuread_application.velero_application, + azuread_service_principal.velero_sp, + azuread_application_password.velero_application_password + ] + } +} + + +resource "null_resource" "install_velero" { + depends_on = [local_file.credentials] + + triggers = { + bucket = azurerm_storage_container.velero_backup_container.name + storage_account = module.velero_storage_account.name + subscription_id = var.subscription_id + tenant_id = var.tenant_id + client_id = azuread_application.velero_application.application_id + client_secret = azuread_application_password.velero_application_password.value + resource_group = var.resource_group_name + plugin_version = var.plugin_version + cluster_name = var.aks_cluster_name + credentials_file_name = local_file.credentials.filename + } + + provisioner "local-exec" { + when = destroy + command = <"`, so you should have that context available in your `.kube` folder. +This is achievable using the utility script `k8setup.sh` included in the aks-setup folder of your IaC project + + +## How to use it + +```hcl + resource "azurerm_resource_group" "rg_velero_backup" { + name = local.aks_rg_name + location = var.location + tags = var.tags + } + + module "velero" { + source = "git::https://github.com/pagopa/terraform-azurerm-v3.git//kubernetes_cluster_velero?ref=" + + # required + backup_storage_account_name = module.velero_storage_account.name + backup_storage_container_name = "velero-backup" + resource_group_name = azurerm_resource_group.rg_velero_backup.name + subscription_id = data.azurerm_subscription.current.subscription_id + tenant_id = data.azurerm_subscription.current.tenant_id + prefix = "my-app" + location = var.location + aks_cluster_name = module.aks.name + aks_cluster_rg = azurerm_resource_group.rg_aks.name + + # optional + plugin_version = "v1.7.1" + use_storage_private_endpoint = true + + # required if use_storage_private_endpoint = true + storage_account_private_dns_zone_id = azurerm_private_dns_zone.storage_account.id + private_endpoint_subnet_id = module.private_endpoint_snet.id + + tags = var.tags + } + + +``` + + + + +## Requirements + +| Name | Version | +|------|---------| +| [terraform](#requirement\_terraform) | >= 1.3.0 | +| [azurerm](#requirement\_azurerm) | >= 3.30.0, <= 3.64.0 | +| [null](#requirement\_null) | <= 3.2.1 | + +## Modules + +| Name | Source | Version | +|------|--------|---------| +| [velero\_storage\_account](#module\_velero\_storage\_account) | git::https://github.com/pagopa/terraform-azurerm-v3.git//storage_account | v7.2.0 | + +## Resources + +| Name | Type | +|------|------| +| [azuread_application.velero_application](https://registry.terraform.io/providers/hashicorp/azuread/latest/docs/resources/application) | resource | +| [azuread_application_password.velero_application_password](https://registry.terraform.io/providers/hashicorp/azuread/latest/docs/resources/application_password) | resource | +| [azuread_service_principal.velero_sp](https://registry.terraform.io/providers/hashicorp/azuread/latest/docs/resources/service_principal) | resource | +| [azuread_service_principal_password.velero_principal_password](https://registry.terraform.io/providers/hashicorp/azuread/latest/docs/resources/service_principal_password) | resource | +| [azurerm_private_endpoint.velero_storage_private_endpoint](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/resources/private_endpoint) | resource | +| [azurerm_role_assignment.velero_sp_aks_role](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/resources/role_assignment) | resource | +| [azurerm_role_assignment.velero_sp_storage_role](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/resources/role_assignment) | resource | +| [azurerm_storage_container.velero_backup_container](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/resources/storage_container) | resource | +| [local_file.credentials](https://registry.terraform.io/providers/hashicorp/local/latest/docs/resources/file) | resource | +| [null_resource.install_velero](https://registry.terraform.io/providers/hashicorp/null/latest/docs/resources/resource) | resource | +| [azuread_client_config.current](https://registry.terraform.io/providers/hashicorp/azuread/latest/docs/data-sources/client_config) | data source | +| [azurerm_kubernetes_cluster.aks_cluster](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/data-sources/kubernetes_cluster) | data source | + +## Inputs + +| Name | Description | Type | Default | Required | +|------|-------------|------|---------|:--------:| +| [aks\_cluster\_name](#input\_aks\_cluster\_name) | (Required) Name of the aks cluster on which Velero will be installed | `string` | n/a | yes | +| [aks\_cluster\_rg](#input\_aks\_cluster\_rg) | (Required) AKS cluster resource group name | `string` | n/a | yes | +| [backup\_storage\_container\_name](#input\_backup\_storage\_container\_name) | (Required) Name of the storage container where Velero keeps the backups | `string` | n/a | yes | +| [location](#input\_location) | (Required) Resource location | `string` | n/a | yes | +| [plugin\_version](#input\_plugin\_version) | (Optional) Version for the velero plugin | `string` | `"v1.7.1"` | no | +| [prefix](#input\_prefix) | (Required) Prefix used in the Velero dedicated resource names | `string` | n/a | yes | +| [private\_endpoint\_subnet\_id](#input\_private\_endpoint\_subnet\_id) | (Optional) Subnet id where to create the private endpoint for backups storage account | `string` | `null` | no | +| [resource\_group\_name](#input\_resource\_group\_name) | (Required) Name of the resource group in which the backup storage account is located | `string` | n/a | yes | +| [storage\_account\_private\_dns\_zone\_id](#input\_storage\_account\_private\_dns\_zone\_id) | (Optional) Storage account private dns zone id, used in the private endpoint creation | `string` | `null` | no | +| [storage\_account\_replication\_type](#input\_storage\_account\_replication\_type) | (Optional) Replication type used for the backup storage account | `string` | `"ZRS"` | no | +| [storage\_account\_tier](#input\_storage\_account\_tier) | (Optional) Tier used for the backup storage account | `string` | `"Standard"` | no | +| [subscription\_id](#input\_subscription\_id) | (Required) ID of the subscription | `string` | n/a | yes | +| [tags](#input\_tags) | n/a | `map(any)` | n/a | yes | +| [tenant\_id](#input\_tenant\_id) | (Required) ID of the tenant | `string` | n/a | yes | +| [use\_storage\_private\_endpoint](#input\_use\_storage\_private\_endpoint) | (Optional) Whether to make the storage account private and use a private endpoint to connect | `bool` | `true` | no | + +## Outputs + +No outputs. + diff --git a/kubernetes_cluster_velero/velero-credentials.tpl b/kubernetes_cluster_velero/velero-credentials.tpl new file mode 100644 index 00000000..97bfb749 --- /dev/null +++ b/kubernetes_cluster_velero/velero-credentials.tpl @@ -0,0 +1,6 @@ +AZURE_SUBSCRIPTION_ID=${subscription_id} +AZURE_TENANT_ID=${tenant_id} +AZURE_CLIENT_ID=${client_id} +AZURE_CLIENT_SECRET=${client_secret} +AZURE_RESOURCE_GROUP=${backup_rg} +AZURE_CLOUD_NAME=AzurePublicCloud diff --git a/kubernetes_velero_backup/01_main.tf b/kubernetes_velero_backup/01_main.tf new file mode 100644 index 00000000..62408a56 --- /dev/null +++ b/kubernetes_velero_backup/01_main.tf @@ -0,0 +1,27 @@ +resource "null_resource" "schedule_backup" { + for_each = toset(var.namespaces) + + triggers = { + backup_name = var.backup_name + schedule = var.schedule + volume_snapshot = var.volume_snapshot + ttl = var.ttl + namespace = each.value + cluster_name = var.aks_cluster_name + } + + provisioner "local-exec" { + when = destroy + command = <"`, so you should have that context available in your `.kube` folder. +This is achievable using the utility script `k8setup.sh` included in the aks-setup folder of your IaC project + + +## How to use it + +This module requires Velero to be installed; check module `kubernetes_cluster_velero` for details on the installation + +### Variable definition example + +```hcl +module "aks_namespace_backup" { + source = "git::https://github.com/pagopa/terraform-azurerm-v3.git//kubernetes_velero_backup?ref=" + + # required + aks_cluster_name = module.aks.name + backup_name = "daily-backup" + namespaces = ["my-namespace-name"] + + # optional + ttl = "100h0m0s" + schedule = "0 3 * * *" #refers to UTC timezone + volume_snapshot = false +} +``` + +You can declare which namespace to backup, and you can also use the keyword `ALL` to trigger an all-namespaces backup. It can be used alongside the other namespace names + +The final backup name will be: `backup_name`-`namespace_name`-`datetime` + + + +## Requirements + +| Name | Version | +|------|---------| +| [terraform](#requirement\_terraform) | >= 1.3.0 | +| [null](#requirement\_null) | <= 3.2.1 | + +## Modules + +No modules. + +## Resources + +| Name | Type | +|------|------| +| [null_resource.schedule_backup](https://registry.terraform.io/providers/hashicorp/null/latest/docs/resources/resource) | resource | + +## Inputs + +| Name | Description | Type | Default | Required | +|------|-------------|------|---------|:--------:| +| [aks\_cluster\_name](#input\_aks\_cluster\_name) | (Required) Name of the aks cluster on which Velero will be installed | `string` | n/a | yes | +| [backup\_name](#input\_backup\_name) | (Required) Name assigned to the backup, used as prefix for the namespace name | `string` | n/a | yes | +| [namespaces](#input\_namespaces) | (Required) List of namespace names to backup. Use 'ALL' for an all-namespaces backup | `list(string)` | n/a | yes | +| [schedule](#input\_schedule) | (Optional) Cron expression for the scheduled velero backup, in UTC timezone. ref: https://velero.io/docs/v1.9/backup-reference/ | `string` | `"0 3 * * *"` | no | +| [ttl](#input\_ttl) | (Optional) TTL for velero backup, expressed using 'hms' format | `string` | `"360h0m0s"` | no | +| [volume\_snapshot](#input\_volume\_snapshot) | (Optional) Whether or not to execute the persistence volume snapshot. Disabled by default | `bool` | `false` | no | + +## Outputs + +No outputs. +