From 648b8a2d310c01b2226aa3f88ef1db9a977d3233 Mon Sep 17 00:00:00 2001 From: Arnaud Tincelin <25585350+arnaud-tincelin@users.noreply.github.com> Date: Tue, 28 Jun 2022 19:26:19 +0200 Subject: [PATCH] Add ARM implementation --- .gitignore | 2 +- README.md | 38 ++- arm/aca.json | 224 ++++++++++++++++++ arm/acr.json | 84 +++++++ arm/deploy.sh | 39 +++ .../container_apps.json | 4 +- main.tf => terraform/main.tf | 6 +- providers.tf => terraform/providers.tf | 0 variables.tf => terraform/variables.tf | 0 versions.tf => terraform/versions.tf | 2 +- 10 files changed, 381 insertions(+), 18 deletions(-) create mode 100644 arm/aca.json create mode 100644 arm/acr.json create mode 100755 arm/deploy.sh rename container_apps.json => terraform/container_apps.json (98%) rename main.tf => terraform/main.tf (91%) rename providers.tf => terraform/providers.tf (100%) rename variables.tf => terraform/variables.tf (100%) rename versions.tf => terraform/versions.tf (88%) diff --git a/.gitignore b/.gitignore index cdafab4..78c2efb 100644 --- a/.gitignore +++ b/.gitignore @@ -1,4 +1,4 @@ *.tfvars .terraform .terraform* -*.tfstate +*.tfstate* diff --git a/README.md b/README.md index c9166d7..b026c93 100644 --- a/README.md +++ b/README.md @@ -4,14 +4,19 @@ This example deploys a full environment to support azure devops agents. For each Note: If there is no agent in the pool, the pipeline will fail. To workaround this limitation, a "placeholder" agent must be configured in order to always have 1 agent in the pool. This agent will be a "fake" agent as it will always be marked as offline. -## PAT required permissions +## Pre-requisites -- `Agent Pools`: `Read & manage` to be able to register / unregister agents -- `Build`: `Read` for Keda to be able to get the queue of pending builds +1. Create an agent pool and save its **name** and **id**. + Note: the pool ID corresponds to the `queueId` from the url when a pool is selected in the browser. + Example: `https://dev.azure.com/myOrg/myProject/_settings/agentqueues?queueId=11&view=jobs`, the pool ID is 11 +1. Create a **PAT** (Personal Access Token) with the following permissions: + - `Agent Pools`: `Read & manage` to be able to register / unregister agents + - `Build`: `Read` for Keda to be able to get the queue of pending builds +1. Configure the **max number of parallel jobs** in Azure Devops so that your ACA does scale accordingly ## Deploy the solution -Using Terraform, this solution deploys: +This repository deploys: - a resource group - a VNET with 2 subnets to host the container apps @@ -19,13 +24,23 @@ Using Terraform, this solution deploys: - a log analytics workspace to store logs from the container apps - a container apps environment + a container apps -To deploy, run the following commands: +There are 2 implementations of the same infrastructure in this repository: -```bash -az login -terraform init -terraform apply -auto-approve -``` +- using Terraform in the `terraform` folder: + To deploy, run the following commands: + + ```bash + az login + terraform init + terraform apply -auto-approve + ``` + +- using ARM template in the `arm` folder: + To deploy, run the following commands: + + ```bash + ./deploy.sh myRG myLocation myACR myWorkspace https://dev.azure.com/myOrg myPoolName myPoolID myToken MaxRunnerCount + ``` ## Create a "placeholder" agent @@ -43,10 +58,11 @@ There are 2 possibilities to deploy a placeholder agent: cd - ``` -- From an azure devops agent pool, click on "New Agent" and follow the instructions. You might need a temporary VM to install the agent. The VM might be deleted once the agent has been registered in the pool. +- From an azure devops agent pool, click on "New Agent" and follow the instructions. You will need a temporary VM to install the agent. The VM might be deleted once the agent has been registered in the pool. ## References +- [Container Apps ARM Template](https://docs.microsoft.com/en-us/azure/templates/microsoft.app/containerapps?tabs=json) - [Docker Agents for Azure Devops](https://docs.microsoft.com/en-us/azure/devops/pipelines/agents/docker?view=azure-devops#linux) - [Keda for Azure Pipelines](https://keda.sh/docs/2.5/scalers/azure-pipelines/) - [Keda](https://keda.sh/docs/2.6/deploy/#yaml) diff --git a/arm/aca.json b/arm/aca.json new file mode 100644 index 0000000..d7f444b --- /dev/null +++ b/arm/aca.json @@ -0,0 +1,224 @@ +{ + "$schema": "https://schema.management.azure.com/schemas/2019-04-01/deploymentTemplate.json#", + "contentVersion": "1.0.0.0", + "parameters": { + "location": { + "type": "String" + }, + "workspaceName": { + "type": "string", + "metadata": { + "description": "Name of the log analytics workspace to create." + } + }, + "azp_url": { + "type": "String" + }, + "azp_token": { + "type": "securestring" + }, + "azp_pool": { + "type": "String" + }, + "azp_poolID": { + "type": "String" + }, + "registry_server": { + "type": "String" + }, + "registry_username": { + "type": "String" + }, + "registry_password": { + "type": "securestring" + }, + "image": { + "type": "String" + }, + "min_replicas": { + "type": "Int" + }, + "max_replicas": { + "type": "Int" + } + }, + "functions": [], + "variables": {}, + "resources": [ + { + "type": "Microsoft.Network/virtualNetworks", + "apiVersion": "2020-11-01", + "name": "devops-agents", + "location": "[parameters('location')]", + "properties": { + "addressSpace": { + "addressPrefixes": [ + "10.0.0.0/16" + ] + }, + "dhcpOptions": { + "dnsServers": [] + }, + "subnets": [ + { + "name": "infrastructure", + "properties": { + "addressPrefix": "10.0.0.0/21", + "serviceEndpoints": [], + "delegations": [], + "privateEndpointNetworkPolicies": "Enabled", + "privateLinkServiceNetworkPolicies": "Enabled" + } + }, + { + "name": "runtime", + "properties": { + "addressPrefix": "10.0.8.0/21", + "serviceEndpoints": [], + "delegations": [], + "privateEndpointNetworkPolicies": "Enabled", + "privateLinkServiceNetworkPolicies": "Enabled" + } + } + ], + "virtualNetworkPeerings": [], + "enableDdosProtection": false + } + }, + { + "type": "microsoft.operationalinsights/workspaces", + "apiVersion": "2021-12-01-preview", + "name": "[parameters('workspaceName')]", + "location": "westeurope", + "properties": { + "sku": { + "name": "pergb2018" + }, + "retentionInDays": 30, + "features": { + "enableLogAccessUsingOnlyResourcePermissions": true + }, + "workspaceCapping": { + "dailyQuotaGb": -1 + }, + "publicNetworkAccessForIngestion": "Enabled", + "publicNetworkAccessForQuery": "Enabled" + } + }, + { + "type": "Microsoft.App/managedEnvironments", + "apiVersion": "2022-03-01", + "name": "devops-agent", + "location": "[parameters('location')]", + "dependsOn": [ + "[resourceId('microsoft.operationalinsights/workspaces', parameters('workspaceName'))]", + "[resourceId('Microsoft.Network/virtualNetworks', 'devops-agents')]" + ], + "properties": { + "appLogsConfiguration": { + "destination": "log-analytics", + "logAnalyticsConfiguration": { + "customerId": "[reference(resourceId('microsoft.operationalinsights/workspaces', parameters('workspaceName')), '2021-12-01-preview').customerId]", + "sharedKey": "[listKeys(resourceId('microsoft.operationalinsights/workspaces', parameters('workspaceName')), '2021-12-01-preview').primarySharedKey]" + } + }, + "vnetConfiguration": { + "infrastructureSubnetId": "[resourceId('Microsoft.Network/virtualNetworks/subnets', 'devops-agents', 'infrastructure')]", + "internal": true, + "runtimeSubnetId": "[resourceId('Microsoft.Network/virtualNetworks/subnets', 'devops-agents', 'runtime')]" + } + } + }, + { + "type": "Microsoft.App/containerApps", + "apiVersion": "2022-03-01", + "name": "devops-agent", + "location": "[parameters('location')]", + "dependsOn": [ + "[resourceId('Microsoft.App/managedEnvironments', 'devops-agent')]" + ], + "identity": { + "type": "SystemAssigned" + }, + "properties": { + "managedEnvironmentId": "[resourceId('Microsoft.App/managedEnvironments', 'devops-agent')]", + "configuration": { + "secrets": [ + { + "name": "registry-secret", + "value": "[parameters('registry_password')]" + }, + { + "name": "azp-token", + "value": "[parameters('azp_token')]" + }, + { + "name": "azp-url", + "value": "[parameters('azp_url')]" + } + ], + "registries": [ + { + "server": "[parameters('registry_server')]", + "username": "[parameters('registry_username')]", + "passwordSecretRef": "registry-secret" + } + ] + }, + "template": { + "containers": [ + { + "image": "[parameters('image')]", + "name": "devops-agent", + "resources": { + "cpu": 1.75, + "memory": "3.5Gi" + }, + "env": [ + { + "name": "AZP_TOKEN", + "secretRef": "azp-token" + }, + { + "name": "AZP_URL", + "secretRef": "azp-url" + }, + { + "name": "AZP_POOL", + "value": "[parameters('azp_pool')]" + } + ] + } + ], + "scale": { + "minReplicas": "[parameters('min_replicas')]", + "maxReplicas": "[parameters('max_replicas')]", + "rules": [ + { + "name": "azure-pipelines", + "custom": { + "type": "azure-pipelines", + "metadata": { + "poolID": "[parameters('azp_poolID')]", + "targetPipelinesQueueLength": "1" + }, + "auth": [ + { + "secretRef": "azp-token", + "triggerParameter": "personalAccessToken" + }, + { + "secretRef": "azp-url", + "triggerParameter": "organizationURL" + } + ] + } + } + ] + } + } + } + } + ], + "outputs": {} +} diff --git a/arm/acr.json b/arm/acr.json new file mode 100644 index 0000000..ca19dcb --- /dev/null +++ b/arm/acr.json @@ -0,0 +1,84 @@ +{ + "$schema": "https://schema.management.azure.com/schemas/2019-04-01/deploymentTemplate.json#", + "contentVersion": "1.0.0.0", + "parameters": { + "name": { + "type": "string", + "metadata": { + "description": "Name of the container registry to create." + } + }, + "location": { + "type": "string", + "metadata": { + "description": "The location of the resource." + } + } + }, + "functions": [], + "variables": {}, + "resources": [ + { + "type": "Microsoft.ContainerRegistry/registries", + "apiVersion": "2022-02-01-preview", + "name": "[parameters('name')]", + "location": "[parameters('location')]", + "sku": { + "name": "Standard", + "tier": "Standard" + }, + "properties": { + "adminUserEnabled": true, + "policies": { + "quarantinePolicy": { + "status": "disabled" + }, + "trustPolicy": { + "type": "Notary", + "status": "disabled" + }, + "retentionPolicy": { + "days": 7, + "status": "disabled" + }, + "exportPolicy": { + "status": "enabled" + }, + "azureADAuthenticationAsArmPolicy": { + "status": "enabled" + }, + "softDeletePolicy": { + "retentionDays": 7, + "status": "disabled" + } + }, + "encryption": { + "status": "disabled" + }, + "dataEndpointEnabled": false, + "publicNetworkAccess": "Enabled", + "networkRuleBypassOptions": "AzureServices", + "zoneRedundancy": "Disabled", + "anonymousPullEnabled": false + } + } + ], + "outputs": { + "name": { + "type": "string", + "value": "[parameters('name')]" + }, + "loginServer": { + "type": "string", + "value": "[reference(resourceId('Microsoft.ContainerRegistry/registries', parameters('name'))).loginServer]" + }, + "username": { + "type": "string", + "value": "[listCredentials(resourceId('Microsoft.ContainerRegistry/registries', parameters('name')), '2022-02-01-preview').username]" + }, + "password": { + "type": "string", + "value": "[listCredentials(resourceId('Microsoft.ContainerRegistry/registries', parameters('name')), '2022-02-01-preview').passwords[0].value]" + } + } +} diff --git a/arm/deploy.sh b/arm/deploy.sh new file mode 100755 index 0000000..c7f375e --- /dev/null +++ b/arm/deploy.sh @@ -0,0 +1,39 @@ +#!/bin/bash + +set -e + +RG_NAME=$1 +LOCATION=$2 +ACR_NAME=$3 +WORKSPACE_NAME=$4 +AZP_URL=$5 +AZP_POOL=$6 +AZP_POOLID=$7 +AZP_TOKEN=$8 +MAX_REPLICAS=$9 + +az login -o none + +image="devops/agent:1.0" + +acr=`az deployment group create --resource-group ${RG_NAME} --template-file ./acr.json --parameters name=${ACR_NAME} location=${LOCATION}` +loginServer=`jq -r '.properties.outputs.loginServer.value' <<<${acr}` +username=`jq -r '.properties.outputs.username.value' <<<${acr}` +password=`jq -r '.properties.outputs.password.value' <<<${acr}` + +az acr login --name ${loginServer} -u ${username} -p ${password} +az acr build --registry ${loginServer} -t ${image} ../docker + +az deployment group create --resource-group ${RG_NAME} --template-file ./aca.json \ + --parameters location=${LOCATION} \ + workspaceName=${WORKSPACE_NAME} \ + azp_url=${AZP_URL} \ + azp_token=${AZP_TOKEN} \ + azp_pool=${AZP_POOL} \ + azp_poolID=${AZP_POOLID} \ + registry_server=${loginServer} \ + registry_username=${username} \ + registry_password=${password} \ + image=${loginServer}/${image} \ + min_replicas=0 \ + max_replicas=${MAX_REPLICAS} diff --git a/container_apps.json b/terraform/container_apps.json similarity index 98% rename from container_apps.json rename to terraform/container_apps.json index 77814cc..37e83fd 100644 --- a/container_apps.json +++ b/terraform/container_apps.json @@ -64,7 +64,7 @@ "resources": [ { "type": "Microsoft.App/managedEnvironments", - "apiVersion": "2022-01-01-preview", + "apiVersion": "2022-03-01", "name": "devops-agent", "location": "[parameters('location')]", "properties": { @@ -84,7 +84,7 @@ }, { "type": "Microsoft.App/containerApps", - "apiVersion": "2022-01-01-preview", + "apiVersion": "2022-03-01", "name": "devops-agent", "location": "[parameters('location')]", "dependsOn": [ diff --git a/main.tf b/terraform/main.tf similarity index 91% rename from main.tf rename to terraform/main.tf index 4be0eae..8e0df41 100644 --- a/main.tf +++ b/terraform/main.tf @@ -46,8 +46,8 @@ locals { resource "null_resource" "build_image" { triggers = { - dockerfile = filemd5("${path.module}/docker/Dockerfile") # this variable aims to trigger image rebuild on file changes - startfile = filemd5("${path.module}/docker/start.sh") # this variable aims to trigger image rebuild on file changes + dockerfile = filemd5("${path.module}/../docker/Dockerfile") # this variable aims to trigger image rebuild on file changes + startfile = filemd5("${path.module}/../docker/start.sh") # this variable aims to trigger image rebuild on file changes acr_name = azurerm_container_registry.this.name acr_rg = azurerm_container_registry.this.resource_group_name image_name = local.image_name @@ -56,7 +56,7 @@ resource "null_resource" "build_image" { provisioner "local-exec" { command = <