arm-builder: Always deploy arm-builders to eun

Hard-code the arm builder location to 'northeurope' due to limited support of arm-based VMs in many other Azure regions. Specifically, ghaf-infra deployment on UAE would fail without this change, since Azure UAE locations do not support arm-based VMs. The workaround done in this change is to deploy the arm-builder on 'northeurope', even if other ghaf-infra resources are deployed on locations other than eun. Signed-off-by: Henri Rosten <[email protected]>
tiiuae · Mar 21, 2024 · c594226 · c594226
1 parent 3c45ed6
commit c594226
Show file tree

Hide file tree

Showing 7 changed files with 83 additions and 22 deletions.
diff --git a/terraform/arm-builder.tf b/terraform/arm-builder.tf
@@ -3,6 +3,37 @@
 
 locals {
   arm_num_builders = local.opts[local.conf].num_builders_aarch64
+  # Hard-code the arm builder location to 'northeurope' due to limited support
+  # of arm-based VMs in (many) Azure regions. For reference, see:
+  # https://github.com/tiiuae/ghaf-infra/pull/81#pullrequestreview-1927417660
+  arm_vm_location = "northeurope"
+  # Is the rest of the infra also being deployed to 'northeurope'?
+  infra_in_eun = azurerm_resource_group.infra.location == "northeurope"
+  # If all resources are deployed to 'northeurope', jenkins-controller will
+  # access the arm builder over the private network. If the rest of the infra
+  # is not deployed in 'northeurope', jenkins-controller will access the arm
+  # builder VM over the public network.
+  allow_ssh_from = local.infra_in_eun ? azurerm_subnet.jenkins.address_prefixes[0] : module.jenkins_controller_vm.virtual_machine_ip_address
+  # If all resources are deployed to 'northeurope', deploy all VMs on the
+  # same subnet. If the rest of the infra is not deployed in 'northeurope'
+  # deploy the arm builder in its own subnet.
+  subnet_id = local.infra_in_eun ? azurerm_subnet.builders.id : azurerm_subnet.builders_arm[0].id
+}
+
+resource "azurerm_virtual_network" "vnet_arm" {
+  count               = local.infra_in_eun ? 0 : 1
+  name                = "ghaf-infra-vnet-arm"
+  address_space       = ["10.0.0.0/16"]
+  location            = local.arm_vm_location
+  resource_group_name = azurerm_resource_group.infra.name
+}
+
+resource "azurerm_subnet" "builders_arm" {
+  count                = local.infra_in_eun ? 0 : 1
+  name                 = "ghaf-infra-builders-arm"
+  resource_group_name  = azurerm_resource_group.infra.name
+  virtual_network_name = azurerm_virtual_network.vnet_arm[0].name
+  address_prefixes     = ["10.0.4.0/28"]
 }
 
 module "arm_builder_vm" {
@@ -11,7 +42,7 @@ module "arm_builder_vm" {
   count = local.arm_num_builders
 
   resource_group_name         = azurerm_resource_group.infra.name
-  location                    = azurerm_resource_group.infra.location
+  location                    = local.arm_vm_location
   virtual_machine_name        = "ghaf-builder-aarch64-${count.index}-${local.ws}"
   virtual_machine_size        = local.opts[local.conf].vm_size_builder_aarch64
   virtual_machine_osdisk_size = local.opts[local.conf].osdisk_size_builder
@@ -31,7 +62,14 @@ module "arm_builder_vm" {
     ],
   })])
 
-  subnet_id = azurerm_subnet.builders.id
+  # Currently, we always deploy arm builder VMs to location 'northeurope'.
+  # In case other resources in this infra (namely, the jenkins-controller VM)
+  # are deployed to location other than 'northeurope', jenkins-controller
+  # cannot access the arm builder using its private IP. Therefore, when the arm
+  # builder and jenkins-controller are deployed on different locations, builder
+  # needs to be accessed over its public IP.
+  access_over_public_ip = !(local.infra_in_eun)
+  subnet_id             = local.subnet_id
 }
 
 # Allow inbound SSH from the jenkins subnet (only)
@@ -45,9 +83,9 @@ resource "azurerm_network_interface_security_group_association" "arm_builder_vm"
 resource "azurerm_network_security_group" "arm_builder_vm" {
   count = local.arm_num_builders
 
-  name                = "arm-builder-vm-${count.index}"
+  name                = "arm-builder-vm-${local.shortloc}-${count.index}"
   resource_group_name = azurerm_resource_group.infra.name
-  location            = azurerm_resource_group.infra.location
+  location            = local.arm_vm_location
 
   security_rule {
     name                       = "AllowSSHFromJenkins"
@@ -57,7 +95,7 @@ resource "azurerm_network_security_group" "arm_builder_vm" {
     protocol                   = "Tcp"
     source_port_range          = "*"
     destination_port_ranges    = [22]
-    source_address_prefix      = azurerm_subnet.jenkins.address_prefixes[0]
+    source_address_prefix      = local.allow_ssh_from
     destination_address_prefix = "*"
   }
 }

diff --git a/terraform/jenkins-controller.tf b/terraform/jenkins-controller.tf
@@ -54,16 +54,16 @@ module "jenkins_controller_vm" {
       # changed.
       {
         content = join("\n", concat(
-          [for ip in toset(module.builder_vm[*].virtual_machine_private_ip_address) : "ssh://remote-build@${ip} x86_64-linux /etc/secrets/remote-build-ssh-key 10 1 kvm,nixos-test,benchmark,big-parallel - -"],
-          [for ip in toset(module.arm_builder_vm[*].virtual_machine_private_ip_address) : "ssh://remote-build@${ip} aarch64-linux /etc/secrets/remote-build-ssh-key 8 1 kvm,nixos-test,benchmark,big-parallel - -"]
+          [for ip in toset(module.builder_vm[*].virtual_machine_ip_address) : "ssh://remote-build@${ip} x86_64-linux /etc/secrets/remote-build-ssh-key 10 1 kvm,nixos-test,benchmark,big-parallel - -"],
+          [for ip in toset(module.arm_builder_vm[*].virtual_machine_ip_address) : "ssh://remote-build@${ip} aarch64-linux /etc/secrets/remote-build-ssh-key 8 1 kvm,nixos-test,benchmark,big-parallel - -"]
         )),
         "path" = "/etc/nix/machines"
       },
       # Render /var/lib/builder-keyscan/scanlist, so known_hosts can be populated.
       {
         content = join("\n", toset(concat(
-          module.builder_vm[*].virtual_machine_private_ip_address,
-          module.arm_builder_vm[*].virtual_machine_private_ip_address
+          module.builder_vm[*].virtual_machine_ip_address,
+          module.arm_builder_vm[*].virtual_machine_ip_address
         ))),
         "path" = "/var/lib/builder-keyscan/scanlist"
       },
@@ -74,8 +74,9 @@ module "jenkins_controller_vm" {
     ]
   })])
 
-  allocate_public_ip = true
-  subnet_id          = azurerm_subnet.jenkins.id
+  allocate_public_ip    = true
+  access_over_public_ip = true
+  subnet_id             = azurerm_subnet.jenkins.id
 
   # Attach disk to the VM
   data_disks = [

diff --git a/terraform/main.tf b/terraform/main.tf
@@ -154,7 +154,7 @@ resource "azurerm_subnet" "jenkins" {
   address_prefixes     = ["10.0.2.0/24"]
 }
 
-# Slice out a subnet for the buidlers
+# Slice out a subnet for the builders
 resource "azurerm_subnet" "builders" {
   name                 = "ghaf-infra-builders"
   resource_group_name  = azurerm_resource_group.infra.name

diff --git a/terraform/modules/arm-builder-vm/variables.tf b/terraform/modules/arm-builder-vm/variables.tf
@@ -31,6 +31,11 @@ variable "allocate_public_ip" {
   default = false
 }
 
+variable "access_over_public_ip" {
+  type    = bool
+  default = false
+}
+
 variable "subnet_id" {
   type        = string
   description = "The subnet ID to attach to the VM and allocate an IP from"

diff --git a/terraform/modules/arm-builder-vm/virtual_machine.tf b/terraform/modules/arm-builder-vm/virtual_machine.tf
@@ -1,6 +1,12 @@
 # SPDX-FileCopyrightText: 2022-2024 TII (SSRC) and the Ghaf contributors
 # SPDX-License-Identifier: Apache-2.0
 
+locals {
+  # Both var.allocate_public_ip and var.access_over_public_ip
+  # will enable assigning public IP to the VM
+  set_public_ip = var.allocate_public_ip || var.access_over_public_ip
+}
+
 resource "azurerm_virtual_machine" "main" {
   name                = var.virtual_machine_name
   resource_group_name = var.resource_group_name
@@ -88,12 +94,12 @@ resource "azurerm_network_interface" "default" {
     name                          = "internal"
     subnet_id                     = var.subnet_id
     private_ip_address_allocation = "Dynamic"
-    public_ip_address_id          = (var.allocate_public_ip) ? azurerm_public_ip.default[0].id : null
+    public_ip_address_id          = (local.set_public_ip) ? azurerm_public_ip.default[0].id : null
   }
 }
 
 resource "azurerm_public_ip" "default" {
-  count = (var.allocate_public_ip) ? 1 : 0
+  count = (local.set_public_ip) ? 1 : 0
 
   name                = "${var.virtual_machine_name}-pub-ip"
   domain_name_label   = var.virtual_machine_name
@@ -142,7 +148,7 @@ output "virtual_machine_network_interface_id" {
   value = azurerm_network_interface.default.id
 }
 
-output "virtual_machine_private_ip_address" {
-  description = "The first private IP address of the network interface."
-  value       = azurerm_network_interface.default.private_ip_address
+output "virtual_machine_ip_address" {
+  description = "IP address other VMs in the infra will use to access the VM."
+  value       = var.access_over_public_ip ? azurerm_public_ip.default[0].ip_address : azurerm_network_interface.default.private_ip_address
 }
diff --git a/terraform/modules/azurerm-linux-vm/variables.tf b/terraform/modules/azurerm-linux-vm/variables.tf
@@ -35,6 +35,11 @@ variable "allocate_public_ip" {
   default = false
 }
 
+variable "access_over_public_ip" {
+  type    = bool
+  default = false
+}
+
 variable "subnet_id" {
   type        = string
   description = "The subnet ID to attach to the VM and allocate an IP from"

diff --git a/terraform/modules/azurerm-linux-vm/virtual_machine.tf b/terraform/modules/azurerm-linux-vm/virtual_machine.tf
@@ -1,6 +1,12 @@
 # SPDX-FileCopyrightText: 2022-2024 TII (SSRC) and the Ghaf contributors
 # SPDX-License-Identifier: Apache-2.0
 
+locals {
+  # Both var.allocate_public_ip and var.access_over_public_ip
+  # will enable assigning public IP to the VM
+  set_public_ip = var.allocate_public_ip || var.access_over_public_ip
+}
+
 resource "azurerm_virtual_machine" "main" {
   name                = var.virtual_machine_name
   resource_group_name = var.resource_group_name
@@ -85,12 +91,12 @@ resource "azurerm_network_interface" "default" {
     name                          = "internal"
     subnet_id                     = var.subnet_id
     private_ip_address_allocation = "Dynamic"
-    public_ip_address_id          = (var.allocate_public_ip) ? azurerm_public_ip.default[0].id : null
+    public_ip_address_id          = (local.set_public_ip) ? azurerm_public_ip.default[0].id : null
   }
 }
 
 resource "azurerm_public_ip" "default" {
-  count = (var.allocate_public_ip) ? 1 : 0
+  count = (local.set_public_ip) ? 1 : 0
 
   name                = "${var.virtual_machine_name}-pub-ip"
   domain_name_label   = var.virtual_machine_name
@@ -127,7 +133,7 @@ output "virtual_machine_network_interface_id" {
   value = azurerm_network_interface.default.id
 }
 
-output "virtual_machine_private_ip_address" {
-  description = "The first private IP address of the network interface."
-  value       = azurerm_network_interface.default.private_ip_address
+output "virtual_machine_ip_address" {
+  description = "IP address other VMs in the infra will use to access the VM."
+  value       = var.access_over_public_ip ? azurerm_public_ip.default[0].ip_address : azurerm_network_interface.default.private_ip_address
 }