From 723f1dd761f5affe99a971ee581fbeae45e2ef67 Mon Sep 17 00:00:00 2001 From: Andrea Grillo Date: Wed, 18 Oct 2023 12:13:51 +0200 Subject: [PATCH] feat: Module for GitHub self hosted runner on Azure Container Apps Jobs (#171) --- container_app_job_gh_runner/README.md | 76 +++++++++++++++ container_app_job_gh_runner/locals.tf | 49 ++++++++++ container_app_job_gh_runner/main.tf | 97 +++++++++++++++++++ container_app_job_gh_runner/outputs.tf | 24 +++++ container_app_job_gh_runner/tests/backend.ini | 1 + container_app_job_gh_runner/tests/main.tf | 41 ++++++++ container_app_job_gh_runner/tests/output.tf | 28 ++++++ .../tests/resources.tf | 61 ++++++++++++ .../tests/terraform.sh | 34 +++++++ .../tests/variables.tf | 69 +++++++++++++ container_app_job_gh_runner/variables.tf | 82 ++++++++++++++++ container_app_job_gh_runner/versions.tf | 17 ++++ 12 files changed, 579 insertions(+) create mode 100644 container_app_job_gh_runner/README.md create mode 100644 container_app_job_gh_runner/locals.tf create mode 100644 container_app_job_gh_runner/main.tf create mode 100644 container_app_job_gh_runner/outputs.tf create mode 100644 container_app_job_gh_runner/tests/backend.ini create mode 100644 container_app_job_gh_runner/tests/main.tf create mode 100644 container_app_job_gh_runner/tests/output.tf create mode 100644 container_app_job_gh_runner/tests/resources.tf create mode 100644 container_app_job_gh_runner/tests/terraform.sh create mode 100644 container_app_job_gh_runner/tests/variables.tf create mode 100644 container_app_job_gh_runner/variables.tf create mode 100644 container_app_job_gh_runner/versions.tf diff --git a/container_app_job_gh_runner/README.md b/container_app_job_gh_runner/README.md new file mode 100644 index 00000000..e6da8f51 --- /dev/null +++ b/container_app_job_gh_runner/README.md @@ -0,0 +1,76 @@ +# Azure Container App Job as GitHub Runners + +This module creates the infrastructure to host GitHub self hosted runners using Azure Container Apps jobs. + +## Included resources + +The following resources are created and managed by this module: + +- resource group +- subnet +- container app environment +- container app job + +## How to use it + +Give a try to the example saved in `terraform-azurerm-v3/container_app_job_gh_runner/tests` to see a working demo of this module + +### Prerequisites + +Before running the demo, you need to manually create: + +- vnet +- keyvault +- a valid GitHub PAT stored as secret + +Names of these resources are required as module input + + + +## Requirements + +| Name | Version | +|------|---------| +| [terraform](#requirement\_terraform) | >= 1.3.0 | +| [azapi](#requirement\_azapi) | <= 1.9.0 | +| [azurerm](#requirement\_azurerm) | >= 3.44.0, <= 3.76.0 | + +## Modules + +No modules. + +## Resources + +| Name | Type | +|------|------| +| [azapi_resource.runner_environment](https://registry.terraform.io/providers/azure/azapi/latest/docs/resources/resource) | resource | +| [azapi_resource.runner_job](https://registry.terraform.io/providers/azure/azapi/latest/docs/resources/resource) | resource | +| [azurerm_key_vault_access_policy.keyvault_containerapp](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/resources/key_vault_access_policy) | resource | +| [azurerm_resource_group.runner_rg](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/resources/resource_group) | resource | +| [azurerm_subnet.runner_subnet](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/resources/subnet) | resource | +| [azurerm_key_vault.key_vault](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/data-sources/key_vault) | data source | +| [azurerm_subscription.current](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/data-sources/subscription) | data source | + +## Inputs + +| Name | Description | Type | Default | Required | +|------|-------------|------|---------|:--------:| +| [app](#input\_app) | n/a | 
object({
    repo_owner = optional(string, "pagopa")
    repos      = set(string)
    image      = optional(string, "ghcr.io/pagopa/github-self-hosted-runner-azure:beta-dockerfile-v2@sha256:ed51ac419d78b6410be96ecaa8aa8dbe645aa0309374132886412178e2739a47")
  })
| n/a | yes | +| [env\_short](#input\_env\_short) | Short environment prefix | `string` | n/a | yes | +| [environment](#input\_environment) | n/a | 
object({
    workspace_id = string
    customerId   = string
    sharedKey    = string
  })
| n/a | yes | +| [key\_vault](#input\_key\_vault) | n/a | 
object({
    resource_group_name = string
    name                = string
    secret_name         = string
  })
| n/a | yes | +| [location](#input\_location) | Resource group and resources location | `string` | n/a | yes | +| [network](#input\_network) | n/a | 
object({
    rg_vnet      = string
    vnet         = string
    cidr_subnets = list(string)
  })
| n/a | yes | +| [prefix](#input\_prefix) | Project prefix | `string` | n/a | yes | +| [tags](#input\_tags) | Tags for new resources | `map(any)` | 
{
  "CreatedBy": "Terraform"
}
| no | + +## Outputs + +| Name | Description | +|------|-------------| +| [ca\_name](#output\_ca\_name) | Container App job name | +| [cae\_name](#output\_cae\_name) | Container App Environment name | +| [resource\_group](#output\_resource\_group) | Resource group name | +| [subnet\_cidr](#output\_subnet\_cidr) | Subnet CIDR blocks | +| [subnet\_name](#output\_subnet\_name) | Subnet name | + diff --git a/container_app_job_gh_runner/locals.tf b/container_app_job_gh_runner/locals.tf new file mode 100644 index 00000000..3547a6e1 --- /dev/null +++ b/container_app_job_gh_runner/locals.tf @@ -0,0 +1,49 @@ +locals { + name = "${var.prefix}-${var.env_short}" + resource_group_name = "${local.name}-github-runner-rg" + + rules = [for repo in var.app.repos : + { + name = "github-runner-${repo}" + type = "github-runner" + metadata = { + owner = var.app.repo_owner + runnerScope = "repo" + repos = "${repo}" + targetWorkflowQueueLength = "1" + labels = "github-runner-${repo}" + } + auth = [ + { + secretRef = "personal-access-token" + triggerParameter = "personalAccessToken" + } + ] + } + ] + + containers = [for repo in var.app.repos : + { + env = [ + { + name = "GITHUB_PAT" + secretRef = "personal-access-token" + }, + { + name = "REPO_URL" + value = "https://github.com/${var.app.repo_owner}/${repo}" + }, + { + name = "REGISTRATION_TOKEN_API_URL" + value = "https://api.github.com/repos/${var.app.repo_owner}/${repo}/actions/runners/registration-token" + } + ] + image = var.app.image + name = "github-runner-${repo}" + resources = { + cpu = 1.0 + memory = "2Gi" + } + } + ] +} diff --git a/container_app_job_gh_runner/main.tf b/container_app_job_gh_runner/main.tf new file mode 100644 index 00000000..82195825 --- /dev/null +++ b/container_app_job_gh_runner/main.tf @@ -0,0 +1,97 @@ +data "azurerm_key_vault" "key_vault" { + resource_group_name = var.key_vault.resource_group_name + name = var.key_vault.name +} + +resource "azurerm_resource_group" "runner_rg" { + name = local.resource_group_name + location = var.location + + tags = var.tags +} + +resource "azurerm_subnet" "runner_subnet" { + name = "${local.name}-github-runner-snet" + resource_group_name = var.network.rg_vnet + virtual_network_name = var.network.vnet + address_prefixes = var.network.cidr_subnets + service_endpoints = [] +} + +resource "azapi_resource" "runner_environment" { + type = "Microsoft.App/managedEnvironments@2023-05-01" + name = "${local.name}-github-runner-cae" + location = azurerm_resource_group.runner_rg.location + parent_id = azurerm_resource_group.runner_rg.id + + tags = var.tags + + body = jsonencode({ + properties = { + appLogsConfiguration = { + destination = "log-analytics" + logAnalyticsConfiguration = { + customerId = var.environment.customerId + sharedKey = var.environment.sharedKey + } + } + zoneRedundant = true + vnetConfiguration = { + infrastructureSubnetId = azurerm_subnet.runner_subnet.id + internal = true + } + } + }) +} + +resource "azapi_resource" "runner_job" { + type = "Microsoft.App/jobs@2023-05-01" + name = "${local.name}-github-runner-job" + location = azurerm_resource_group.runner_rg.location + parent_id = azurerm_resource_group.runner_rg.id + + tags = var.tags + + identity { + type = "SystemAssigned" + } + + body = jsonencode({ + properties = { + environmentId = azapi_resource.runner_environment.id + configuration = { + replicaRetryLimit = 1 + replicaTimeout = 1800 + eventTriggerConfig = { + parallelism = 1 + replicaCompletionCount = 1 + scale = { + maxExecutions = 10 + minExecutions = 0 + pollingInterval = 20 + rules = local.rules + } + } + secrets = [{ + keyVaultUrl = "${data.azurerm_key_vault.key_vault.vault_uri}secrets/${var.key_vault.secret_name}" # no versioning + identity = "system" + name = "personal-access-token" + }] + triggerType = "Event" + } + template = { + containers = local.containers + } + } + }) +} + +resource "azurerm_key_vault_access_policy" "keyvault_containerapp" { + key_vault_id = data.azurerm_key_vault.key_vault.id + tenant_id = azapi_resource.runner_job.identity[0].tenant_id + object_id = azapi_resource.runner_job.identity[0].principal_id + + secret_permissions = [ + "Get", + ] +} diff --git a/container_app_job_gh_runner/outputs.tf b/container_app_job_gh_runner/outputs.tf new file mode 100644 index 00000000..3ffd6dbc --- /dev/null +++ b/container_app_job_gh_runner/outputs.tf @@ -0,0 +1,24 @@ +output "resource_group" { + value = azurerm_resource_group.runner_rg.name + description = "Resource group name" +} + +output "subnet_name" { + value = azurerm_subnet.runner_subnet.name + description = "Subnet name" +} + +output "subnet_cidr" { + value = azurerm_subnet.runner_subnet.address_prefixes + description = "Subnet CIDR blocks" +} + +output "cae_name" { + value = azapi_resource.runner_environment.name + description = "Container App Environment name" +} + +output "ca_name" { + value = azapi_resource.runner_job.name + description = "Container App job name" +} diff --git a/container_app_job_gh_runner/tests/backend.ini b/container_app_job_gh_runner/tests/backend.ini new file mode 100644 index 00000000..a7cc599b --- /dev/null +++ b/container_app_job_gh_runner/tests/backend.ini @@ -0,0 +1 @@ +subscription=DevOpsLab diff --git a/container_app_job_gh_runner/tests/main.tf b/container_app_job_gh_runner/tests/main.tf new file mode 100644 index 00000000..1773bbb2 --- /dev/null +++ b/container_app_job_gh_runner/tests/main.tf @@ -0,0 +1,41 @@ +terraform { + required_version = ">= 1.3.0" + + required_providers { + azurerm = { + source = "hashicorp/azurerm" + version = "<= 3.76.0" + } + + azapi = { + source = "azure/azapi" + version = "<= 1.9.0" + } + } +} + +provider "azurerm" { + features { + key_vault { + purge_soft_delete_on_destroy = false + } + resource_group { + prevent_deletion_if_contains_resources = false + } + } +} + +data "azurerm_client_config" "current" { +} + +resource "random_id" "unique" { + byte_length = 3 +} + +locals { + project = "${var.prefix}${random_id.unique.hex}" + env_short = substr(random_id.unique.hex, 0, 1) + rg_name = "${local.project}-${local.env_short}-github-runner-rg" + key_vault_name = "${local.project}-${local.env_short}-kv" + vnet_name = "${local.project}-${local.env_short}-vnet" +} diff --git a/container_app_job_gh_runner/tests/output.tf b/container_app_job_gh_runner/tests/output.tf new file mode 100644 index 00000000..e9c9a3dc --- /dev/null +++ b/container_app_job_gh_runner/tests/output.tf @@ -0,0 +1,28 @@ +output "random_id" { + value = random_id.unique.hex +} + +output "resource_group" { + value = module.runner.resource_group + description = "Resource group name" +} + +output "subnet_name" { + value = module.runner.subnet_name + description = "Subnet name" +} + +output "subnet_cidr" { + value = module.runner.subnet_cidr + description = "Subnet CIDR blocks" +} + +output "cae_name" { + value = module.runner.cae_name + description = "Container App Environment name" +} + +output "ca_name" { + value = module.runner.ca_name + description = "Container App job name" +} diff --git a/container_app_job_gh_runner/tests/resources.tf b/container_app_job_gh_runner/tests/resources.tf new file mode 100644 index 00000000..b1a0eaa5 --- /dev/null +++ b/container_app_job_gh_runner/tests/resources.tf @@ -0,0 +1,61 @@ +resource "azurerm_resource_group" "rg" { + name = local.rg_name + location = var.location +} + +#tfsec:ignore:azure-keyvault-specify-network-acl +#tfsec:ignore:azure-keyvault-no-purge +resource "azurerm_key_vault" "key_vault" { + resource_group_name = azurerm_resource_group.rg.name + name = local.key_vault_name + sku_name = "standard" + location = azurerm_resource_group.rg.location + tenant_id = data.azurerm_client_config.current.tenant_id + public_network_access_enabled = true + soft_delete_retention_days = 7 +} + +resource "azurerm_virtual_network" "vnet" { + resource_group_name = azurerm_resource_group.rg.name + name = local.vnet_name + location = azurerm_resource_group.rg.location + address_space = ["10.0.0.0/16"] +} + +# module to use +module "runner" { + source = "../" # change me with module URI + + location = var.location + prefix = var.prefix + env_short = local.env_short # change me with your env + + # set reference to the secret which holds the GitHub PAT with access to your repos + key_vault = { + resource_group_name = azurerm_key_vault.key_vault.resource_group_name + name = azurerm_key_vault.key_vault.name + secret_name = var.key_vault.secret_name + } + + # creates a subnet in the specified existing vnet. Use a /23 CIDR block + network = { + rg_vnet = azurerm_virtual_network.vnet.resource_group_name + vnet = azurerm_virtual_network.vnet.name + cidr_subnets = var.network.cidr_subnets + } + + # set reference to the log analytics workspace you want to use for logging + environment = { + workspace_id = var.environment.workspace_id + customerId = var.environment.customerId + sharedKey = var.environment.sharedKey + } + + # set app properties - especially the list of repos to support + app = { + repos = var.app.repos + repo_owner = var.app.repo_owner + } + + tags = var.tags +} diff --git a/container_app_job_gh_runner/tests/terraform.sh b/container_app_job_gh_runner/tests/terraform.sh new file mode 100644 index 00000000..b772d3a6 --- /dev/null +++ b/container_app_job_gh_runner/tests/terraform.sh @@ -0,0 +1,34 @@ +#!/bin/bash + +set -e + +action=$1 +shift 1 +other=$@ + +subscription="MOCK_VALUE" + +case $action in + "init" | "apply" | "plan" | "destroy" ) + # shellcheck source=/dev/null + if [ -e "./backend.ini" ]; then + source ./backend.ini + else + echo "Error: no backend.ini found!" + exit 1 + fi + + az account set -s "${subscription}" + + terraform init + terraform "$action" $other + ;; + "clean" ) + rm -rf .terraform* terraform.tfstate* + echo "cleaned..." + ;; + * ) + echo "Missed action: init, apply, plan, destroy clean" + exit 1 + ;; +esac diff --git a/container_app_job_gh_runner/tests/variables.tf b/container_app_job_gh_runner/tests/variables.tf new file mode 100644 index 00000000..63c4cab8 --- /dev/null +++ b/container_app_job_gh_runner/tests/variables.tf @@ -0,0 +1,69 @@ +variable "location" { + type = string + default = "westeurope" +} + +variable "prefix" { + description = "Resorce prefix" + type = string + default = "azrmtest" +} + +variable "tags" { + type = map(string) + description = "List of tags" + default = { + CreatedBy = "Terraform" + Source = "https://github.com/pagopa/terraform-azurerm-v3" + } +} + +variable "key_vault" { + type = object({ + resource_group_name = string + name = string + secret_name = string + }) + + default = { + resource_group_name = "azrmtest-keyvault-rg" + name = "azrmtest-keyvault" + secret_name = "gh-pat" + } +} + +variable "network" { + type = object({ + rg_vnet = string + vnet = string + cidr_subnets = list(string) + }) + + default = { + rg_vnet = "azrmtest-vnet-rg" + vnet = "azrmtest-vnet" + cidr_subnets = ["10.0.2.0/23"] + } +} + +variable "environment" { + type = object({ + workspace_id = string + customerId = string + sharedKey = string + }) +} + +variable "app" { + type = object({ + repos = optional(set(string)) + repo_owner = string + }) + + default = { + repo_owner = "pagopa" + repos = [ + "terraform-azurerm-v3" + ] + } +} diff --git a/container_app_job_gh_runner/variables.tf b/container_app_job_gh_runner/variables.tf new file mode 100644 index 00000000..a21ddf34 --- /dev/null +++ b/container_app_job_gh_runner/variables.tf @@ -0,0 +1,82 @@ +variable "tags" { + type = map(any) + description = "Tags for new resources" + default = { + CreatedBy = "Terraform" + } +} + +variable "location" { + type = string + description = "Resource group and resources location" +} + +variable "prefix" { + type = string + description = "Project prefix" + + validation { + condition = ( + length(var.prefix) < 6 + ) + error_message = "Max length is 6 chars." + } +} + +variable "env_short" { + type = string + description = "Short environment prefix" + + validation { + condition = ( + length(var.env_short) == 1 + ) + error_message = "Length is 1 chars." + } +} + +variable "network" { + type = object({ + rg_vnet = string + vnet = string + cidr_subnets = list(string) + }) + + validation { + condition = ( + length(var.network.cidr_subnets) >= 1 + ) + error_message = "CIDR block must be supplied" + } +} + +variable "environment" { + type = object({ + workspace_id = string + customerId = string + sharedKey = string + }) +} + +variable "app" { + type = object({ + repo_owner = optional(string, "pagopa") + repos = set(string) + image = optional(string, "ghcr.io/pagopa/github-self-hosted-runner-azure:beta-dockerfile-v2@sha256:ed51ac419d78b6410be96ecaa8aa8dbe645aa0309374132886412178e2739a47") + }) + + validation { + condition = ( + var.app.repos != null && length(var.app.repos) >= 1 + ) + error_message = "List of repos must supplied" + } +} + +variable "key_vault" { + type = object({ + resource_group_name = string + name = string + secret_name = string + }) +} diff --git a/container_app_job_gh_runner/versions.tf b/container_app_job_gh_runner/versions.tf new file mode 100644 index 00000000..8e275c9c --- /dev/null +++ b/container_app_job_gh_runner/versions.tf @@ -0,0 +1,17 @@ +terraform { + required_version = ">= 1.3.0" + + required_providers { + azurerm = { + source = "hashicorp/azurerm" + version = ">= 3.44.0, <= 3.76.0" + } + + azapi = { + source = "azure/azapi" + version = "<= 1.9.0" + } + } +} + +data "azurerm_subscription" "current" {}