From c59422654769f75d039fd97bbfaf5b8d01d3633e Mon Sep 17 00:00:00 2001 From: Henri Rosten Date: Wed, 20 Mar 2024 13:07:55 +0200 Subject: [PATCH] arm-builder: Always deploy arm-builders to eun Hard-code the arm builder location to 'northeurope' due to limited support of arm-based VMs in many other Azure regions. Specifically, ghaf-infra deployment on UAE would fail without this change, since Azure UAE locations do not support arm-based VMs. The workaround done in this change is to deploy the arm-builder on 'northeurope', even if other ghaf-infra resources are deployed on locations other than eun. Signed-off-by: Henri Rosten --- terraform/arm-builder.tf | 48 +++++++++++++++++-- terraform/jenkins-controller.tf | 13 ++--- terraform/main.tf | 2 +- terraform/modules/arm-builder-vm/variables.tf | 5 ++ .../modules/arm-builder-vm/virtual_machine.tf | 16 +++++-- .../modules/azurerm-linux-vm/variables.tf | 5 ++ .../azurerm-linux-vm/virtual_machine.tf | 16 +++++-- 7 files changed, 83 insertions(+), 22 deletions(-) diff --git a/terraform/arm-builder.tf b/terraform/arm-builder.tf index af06bd96..3515fa13 100644 --- a/terraform/arm-builder.tf +++ b/terraform/arm-builder.tf @@ -3,6 +3,37 @@ locals { arm_num_builders = local.opts[local.conf].num_builders_aarch64 + # Hard-code the arm builder location to 'northeurope' due to limited support + # of arm-based VMs in (many) Azure regions. For reference, see: + # https://github.com/tiiuae/ghaf-infra/pull/81#pullrequestreview-1927417660 + arm_vm_location = "northeurope" + # Is the rest of the infra also being deployed to 'northeurope'? + infra_in_eun = azurerm_resource_group.infra.location == "northeurope" + # If all resources are deployed to 'northeurope', jenkins-controller will + # access the arm builder over the private network. If the rest of the infra + # is not deployed in 'northeurope', jenkins-controller will access the arm + # builder VM over the public network. + allow_ssh_from = local.infra_in_eun ? azurerm_subnet.jenkins.address_prefixes[0] : module.jenkins_controller_vm.virtual_machine_ip_address + # If all resources are deployed to 'northeurope', deploy all VMs on the + # same subnet. If the rest of the infra is not deployed in 'northeurope' + # deploy the arm builder in its own subnet. + subnet_id = local.infra_in_eun ? azurerm_subnet.builders.id : azurerm_subnet.builders_arm[0].id +} + +resource "azurerm_virtual_network" "vnet_arm" { + count = local.infra_in_eun ? 0 : 1 + name = "ghaf-infra-vnet-arm" + address_space = ["10.0.0.0/16"] + location = local.arm_vm_location + resource_group_name = azurerm_resource_group.infra.name +} + +resource "azurerm_subnet" "builders_arm" { + count = local.infra_in_eun ? 0 : 1 + name = "ghaf-infra-builders-arm" + resource_group_name = azurerm_resource_group.infra.name + virtual_network_name = azurerm_virtual_network.vnet_arm[0].name + address_prefixes = ["10.0.4.0/28"] } module "arm_builder_vm" { @@ -11,7 +42,7 @@ module "arm_builder_vm" { count = local.arm_num_builders resource_group_name = azurerm_resource_group.infra.name - location = azurerm_resource_group.infra.location + location = local.arm_vm_location virtual_machine_name = "ghaf-builder-aarch64-${count.index}-${local.ws}" virtual_machine_size = local.opts[local.conf].vm_size_builder_aarch64 virtual_machine_osdisk_size = local.opts[local.conf].osdisk_size_builder @@ -31,7 +62,14 @@ module "arm_builder_vm" { ], })]) - subnet_id = azurerm_subnet.builders.id + # Currently, we always deploy arm builder VMs to location 'northeurope'. + # In case other resources in this infra (namely, the jenkins-controller VM) + # are deployed to location other than 'northeurope', jenkins-controller + # cannot access the arm builder using its private IP. Therefore, when the arm + # builder and jenkins-controller are deployed on different locations, builder + # needs to be accessed over its public IP. + access_over_public_ip = !(local.infra_in_eun) + subnet_id = local.subnet_id } # Allow inbound SSH from the jenkins subnet (only) @@ -45,9 +83,9 @@ resource "azurerm_network_interface_security_group_association" "arm_builder_vm" resource "azurerm_network_security_group" "arm_builder_vm" { count = local.arm_num_builders - name = "arm-builder-vm-${count.index}" + name = "arm-builder-vm-${local.shortloc}-${count.index}" resource_group_name = azurerm_resource_group.infra.name - location = azurerm_resource_group.infra.location + location = local.arm_vm_location security_rule { name = "AllowSSHFromJenkins" @@ -57,7 +95,7 @@ resource "azurerm_network_security_group" "arm_builder_vm" { protocol = "Tcp" source_port_range = "*" destination_port_ranges = [22] - source_address_prefix = azurerm_subnet.jenkins.address_prefixes[0] + source_address_prefix = local.allow_ssh_from destination_address_prefix = "*" } } diff --git a/terraform/jenkins-controller.tf b/terraform/jenkins-controller.tf index 694c8758..e9cf21bb 100644 --- a/terraform/jenkins-controller.tf +++ b/terraform/jenkins-controller.tf @@ -54,16 +54,16 @@ module "jenkins_controller_vm" { # changed. { content = join("\n", concat( - [for ip in toset(module.builder_vm[*].virtual_machine_private_ip_address) : "ssh://remote-build@${ip} x86_64-linux /etc/secrets/remote-build-ssh-key 10 1 kvm,nixos-test,benchmark,big-parallel - -"], - [for ip in toset(module.arm_builder_vm[*].virtual_machine_private_ip_address) : "ssh://remote-build@${ip} aarch64-linux /etc/secrets/remote-build-ssh-key 8 1 kvm,nixos-test,benchmark,big-parallel - -"] + [for ip in toset(module.builder_vm[*].virtual_machine_ip_address) : "ssh://remote-build@${ip} x86_64-linux /etc/secrets/remote-build-ssh-key 10 1 kvm,nixos-test,benchmark,big-parallel - -"], + [for ip in toset(module.arm_builder_vm[*].virtual_machine_ip_address) : "ssh://remote-build@${ip} aarch64-linux /etc/secrets/remote-build-ssh-key 8 1 kvm,nixos-test,benchmark,big-parallel - -"] )), "path" = "/etc/nix/machines" }, # Render /var/lib/builder-keyscan/scanlist, so known_hosts can be populated. { content = join("\n", toset(concat( - module.builder_vm[*].virtual_machine_private_ip_address, - module.arm_builder_vm[*].virtual_machine_private_ip_address + module.builder_vm[*].virtual_machine_ip_address, + module.arm_builder_vm[*].virtual_machine_ip_address ))), "path" = "/var/lib/builder-keyscan/scanlist" }, @@ -74,8 +74,9 @@ module "jenkins_controller_vm" { ] })]) - allocate_public_ip = true - subnet_id = azurerm_subnet.jenkins.id + allocate_public_ip = true + access_over_public_ip = true + subnet_id = azurerm_subnet.jenkins.id # Attach disk to the VM data_disks = [ diff --git a/terraform/main.tf b/terraform/main.tf index dc50b037..43daa688 100644 --- a/terraform/main.tf +++ b/terraform/main.tf @@ -154,7 +154,7 @@ resource "azurerm_subnet" "jenkins" { address_prefixes = ["10.0.2.0/24"] } -# Slice out a subnet for the buidlers +# Slice out a subnet for the builders resource "azurerm_subnet" "builders" { name = "ghaf-infra-builders" resource_group_name = azurerm_resource_group.infra.name diff --git a/terraform/modules/arm-builder-vm/variables.tf b/terraform/modules/arm-builder-vm/variables.tf index c45de13f..c2fb5f26 100644 --- a/terraform/modules/arm-builder-vm/variables.tf +++ b/terraform/modules/arm-builder-vm/variables.tf @@ -31,6 +31,11 @@ variable "allocate_public_ip" { default = false } +variable "access_over_public_ip" { + type = bool + default = false +} + variable "subnet_id" { type = string description = "The subnet ID to attach to the VM and allocate an IP from" diff --git a/terraform/modules/arm-builder-vm/virtual_machine.tf b/terraform/modules/arm-builder-vm/virtual_machine.tf index 856a9cfc..91de6487 100644 --- a/terraform/modules/arm-builder-vm/virtual_machine.tf +++ b/terraform/modules/arm-builder-vm/virtual_machine.tf @@ -1,6 +1,12 @@ # SPDX-FileCopyrightText: 2022-2024 TII (SSRC) and the Ghaf contributors # SPDX-License-Identifier: Apache-2.0 +locals { + # Both var.allocate_public_ip and var.access_over_public_ip + # will enable assigning public IP to the VM + set_public_ip = var.allocate_public_ip || var.access_over_public_ip +} + resource "azurerm_virtual_machine" "main" { name = var.virtual_machine_name resource_group_name = var.resource_group_name @@ -88,12 +94,12 @@ resource "azurerm_network_interface" "default" { name = "internal" subnet_id = var.subnet_id private_ip_address_allocation = "Dynamic" - public_ip_address_id = (var.allocate_public_ip) ? azurerm_public_ip.default[0].id : null + public_ip_address_id = (local.set_public_ip) ? azurerm_public_ip.default[0].id : null } } resource "azurerm_public_ip" "default" { - count = (var.allocate_public_ip) ? 1 : 0 + count = (local.set_public_ip) ? 1 : 0 name = "${var.virtual_machine_name}-pub-ip" domain_name_label = var.virtual_machine_name @@ -142,7 +148,7 @@ output "virtual_machine_network_interface_id" { value = azurerm_network_interface.default.id } -output "virtual_machine_private_ip_address" { - description = "The first private IP address of the network interface." - value = azurerm_network_interface.default.private_ip_address +output "virtual_machine_ip_address" { + description = "IP address other VMs in the infra will use to access the VM." + value = var.access_over_public_ip ? azurerm_public_ip.default[0].ip_address : azurerm_network_interface.default.private_ip_address } diff --git a/terraform/modules/azurerm-linux-vm/variables.tf b/terraform/modules/azurerm-linux-vm/variables.tf index 36464c9e..2dddb5a5 100644 --- a/terraform/modules/azurerm-linux-vm/variables.tf +++ b/terraform/modules/azurerm-linux-vm/variables.tf @@ -35,6 +35,11 @@ variable "allocate_public_ip" { default = false } +variable "access_over_public_ip" { + type = bool + default = false +} + variable "subnet_id" { type = string description = "The subnet ID to attach to the VM and allocate an IP from" diff --git a/terraform/modules/azurerm-linux-vm/virtual_machine.tf b/terraform/modules/azurerm-linux-vm/virtual_machine.tf index 18d61344..476db9cc 100644 --- a/terraform/modules/azurerm-linux-vm/virtual_machine.tf +++ b/terraform/modules/azurerm-linux-vm/virtual_machine.tf @@ -1,6 +1,12 @@ # SPDX-FileCopyrightText: 2022-2024 TII (SSRC) and the Ghaf contributors # SPDX-License-Identifier: Apache-2.0 +locals { + # Both var.allocate_public_ip and var.access_over_public_ip + # will enable assigning public IP to the VM + set_public_ip = var.allocate_public_ip || var.access_over_public_ip +} + resource "azurerm_virtual_machine" "main" { name = var.virtual_machine_name resource_group_name = var.resource_group_name @@ -85,12 +91,12 @@ resource "azurerm_network_interface" "default" { name = "internal" subnet_id = var.subnet_id private_ip_address_allocation = "Dynamic" - public_ip_address_id = (var.allocate_public_ip) ? azurerm_public_ip.default[0].id : null + public_ip_address_id = (local.set_public_ip) ? azurerm_public_ip.default[0].id : null } } resource "azurerm_public_ip" "default" { - count = (var.allocate_public_ip) ? 1 : 0 + count = (local.set_public_ip) ? 1 : 0 name = "${var.virtual_machine_name}-pub-ip" domain_name_label = var.virtual_machine_name @@ -127,7 +133,7 @@ output "virtual_machine_network_interface_id" { value = azurerm_network_interface.default.id } -output "virtual_machine_private_ip_address" { - description = "The first private IP address of the network interface." - value = azurerm_network_interface.default.private_ip_address +output "virtual_machine_ip_address" { + description = "IP address other VMs in the infra will use to access the VM." + value = var.access_over_public_ip ? azurerm_public_ip.default[0].ip_address : azurerm_network_interface.default.private_ip_address }