From a5e66d920edf915b5e3d8d0034da34b6ce80ed7e Mon Sep 17 00:00:00 2001 From: Joonas Rautiola Date: Mon, 19 Feb 2024 19:35:08 +0200 Subject: [PATCH] Add arm remote builder with ubuntu image Signed-off-by: Joonas Rautiola --- ssh-keys.yaml | 3 + terraform/arm-builder.tf | 71 +++++++++ terraform/binary-cache.tf | 4 +- terraform/builder.tf | 8 +- terraform/jenkins-controller.tf | 11 +- terraform/main.tf | 30 ++-- terraform/modules/arm-builder-vm/README.md | 22 +++ .../modules/arm-builder-vm/ubuntu-builder.sh | 147 +++++++++++++++++ terraform/modules/arm-builder-vm/variables.tf | 39 +++++ .../modules/arm-builder-vm/virtual_machine.tf | 149 ++++++++++++++++++ 10 files changed, 463 insertions(+), 21 deletions(-) create mode 100644 terraform/arm-builder.tf create mode 100644 terraform/modules/arm-builder-vm/README.md create mode 100755 terraform/modules/arm-builder-vm/ubuntu-builder.sh create mode 100644 terraform/modules/arm-builder-vm/variables.tf create mode 100644 terraform/modules/arm-builder-vm/virtual_machine.tf diff --git a/ssh-keys.yaml b/ssh-keys.yaml index 2ec3ad95..8629bcf1 100644 --- a/ssh-keys.yaml +++ b/ssh-keys.yaml @@ -7,3 +7,6 @@ flokli: - ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIPTVTXOutUZZjXLB0lUSgeKcSY/8mxKkC0ingGK1whD2 hrosten: - ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIHFuB+uEjhoSdakwiKLD3TbNpbjnlXerEfZQbtRgvdSz +jrautiola: + - ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIGlFqSQFoSSuAS1IjmWBFXie329I5Aqf71QhVOnLTBG+ joonas@x1 + - ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIB3h/Aj66ndKFtqpQ8H53tE9KbbO0obThC0qbQQKFQRr joonas@zeus diff --git a/terraform/arm-builder.tf b/terraform/arm-builder.tf new file mode 100644 index 00000000..929f551d --- /dev/null +++ b/terraform/arm-builder.tf @@ -0,0 +1,71 @@ +# SPDX-FileCopyrightText: 2024 Technology Innovation Institute (TII) +# +# SPDX-License-Identifier: Apache-2.0 + +locals { + arm_num_builders = local.opts[local.conf].num_builders_aarch64 +} + +module "arm_builder_vm" { + source = "./modules/arm-builder-vm" + + count = local.arm_num_builders + + resource_group_name = azurerm_resource_group.infra.name + location = azurerm_resource_group.infra.location + virtual_machine_name = "ghaf-builder-aarch64-${count.index}-${local.env}" + virtual_machine_size = local.opts[local.conf].vm_size_builder_aarch64 + + virtual_machine_custom_data = join("\n", ["#cloud-config", yamlencode({ + users = [{ + name = "remote-build" + ssh_authorized_keys = [ + "${data.azurerm_key_vault_secret.ssh_remote_build_pub.value}" + ] + }] + write_files = [ + { + content = "AZURE_STORAGE_ACCOUNT_NAME=${data.azurerm_storage_account.binary_cache.name}", + "path" = "/var/lib/rclone-http/env" + } + ], + })]) + + subnet_id = azurerm_subnet.builders.id +} + +# Allow inbound SSH from the jenkins subnet (only) +resource "azurerm_network_interface_security_group_association" "arm_builder_vm" { + count = local.arm_num_builders + + network_interface_id = module.arm_builder_vm[count.index].virtual_machine_network_interface_id + network_security_group_id = azurerm_network_security_group.arm_builder_vm[count.index].id +} + +resource "azurerm_network_security_group" "arm_builder_vm" { + count = local.arm_num_builders + + name = "arm-builder-vm-${count.index}" + resource_group_name = azurerm_resource_group.infra.name + location = azurerm_resource_group.infra.location + + security_rule { + name = "AllowSSHFromJenkins" + priority = 400 + direction = "Inbound" + access = "Allow" + protocol = "Tcp" + source_port_range = "*" + destination_port_ranges = [22] + source_address_prefix = azurerm_subnet.jenkins.address_prefixes[0] + destination_address_prefix = "*" + } +} + +# Allow the VMs to read from the binary cache bucket +resource "azurerm_role_assignment" "arm_builder_access_binary_cache" { + count = local.arm_num_builders + scope = data.azurerm_storage_container.binary_cache_1.resource_manager_id + role_definition_name = "Storage Blob Data Reader" + principal_id = module.arm_builder_vm[count.index].virtual_machine_identity_principal_id +} diff --git a/terraform/binary-cache.tf b/terraform/binary-cache.tf index 485c443b..fb543141 100644 --- a/terraform/binary-cache.tf +++ b/terraform/binary-cache.tf @@ -1,4 +1,4 @@ -# SPDX-FileCopyrightText: 2023 Technology Innovation Institute (TII) +# SPDX-FileCopyrightText: 2023-2024 Technology Innovation Institute (TII) # # SPDX-License-Identifier: Apache-2.0 @@ -26,7 +26,7 @@ module "binary_cache_vm" { virtual_machine_custom_data = join("\n", ["#cloud-config", yamlencode({ users = [ - for user in toset(["bmg", "flokli", "hrosten"]) : { + for user in toset(["bmg", "flokli", "hrosten", "jrautiola"]) : { name = user sudo = "ALL=(ALL) NOPASSWD:ALL" ssh_authorized_keys = local.ssh_keys[user] diff --git a/terraform/builder.tf b/terraform/builder.tf index 99deddd0..aaf343e7 100644 --- a/terraform/builder.tf +++ b/terraform/builder.tf @@ -1,4 +1,4 @@ -# SPDX-FileCopyrightText: 2023 Technology Innovation Institute (TII) +# SPDX-FileCopyrightText: 2023-2024 Technology Innovation Institute (TII) # # SPDX-License-Identifier: Apache-2.0 @@ -16,7 +16,7 @@ module "builder_image" { } locals { - num_builders = local.opts[local.conf].num_builders + num_builders = local.opts[local.conf].num_builders_x86 } module "builder_vm" { @@ -26,8 +26,8 @@ module "builder_vm" { resource_group_name = azurerm_resource_group.infra.name location = azurerm_resource_group.infra.location - virtual_machine_name = "ghaf-builder-${count.index}-${local.env}" - virtual_machine_size = local.opts[local.conf].vm_size_builder + virtual_machine_name = "ghaf-builder-x86-${count.index}-${local.env}" + virtual_machine_size = local.opts[local.conf].vm_size_builder_x86 virtual_machine_source_image = module.builder_image.image_id virtual_machine_custom_data = join("\n", ["#cloud-config", yamlencode({ diff --git a/terraform/jenkins-controller.tf b/terraform/jenkins-controller.tf index 540fce27..921585c0 100644 --- a/terraform/jenkins-controller.tf +++ b/terraform/jenkins-controller.tf @@ -1,4 +1,4 @@ -# SPDX-FileCopyrightText: 2023 Technology Innovation Institute (TII) +# SPDX-FileCopyrightText: 2023-2024 Technology Innovation Institute (TII) # # SPDX-License-Identifier: Apache-2.0 @@ -28,7 +28,7 @@ module "jenkins_controller_vm" { virtual_machine_custom_data = join("\n", ["#cloud-config", yamlencode({ users = [ - for user in toset(["bmg", "flokli", "hrosten"]) : { + for user in toset(["bmg", "flokli", "hrosten", "jrautiola"]) : { name = user sudo = "ALL=(ALL) NOPASSWD:ALL" ssh_authorized_keys = local.ssh_keys[user] @@ -54,7 +54,12 @@ module "jenkins_controller_vm" { # changed. { content = join("\n", [ - for ip in toset(module.builder_vm[*].virtual_machine_private_ip_address) : "ssh://remote-build@${ip} x86_64-linux /etc/secrets/remote-build-ssh-key 10 10 kvm,big-parallel - -" + join("\n", [ + for ip in toset(module.builder_vm[*].virtual_machine_private_ip_address) : "ssh://remote-build@${ip} x86_64-linux /etc/secrets/remote-build-ssh-key 10 1 kvm,nixos-test,benchmark,big-parallel - -" + ]), + join("\n", [ + for ip in toset(module.arm_builder_vm[*].virtual_machine_private_ip_address) : "ssh://remote-build@${ip} aarch64-linux /etc/secrets/remote-build-ssh-key 8 1 kvm,nixos-test,benchmark,big-parallel - -" + ]) ]), "path" = "/etc/nix/machines" }, diff --git a/terraform/main.tf b/terraform/main.tf index 1a10b51f..c69bfa9b 100644 --- a/terraform/main.tf +++ b/terraform/main.tf @@ -68,22 +68,28 @@ locals { # E.g. 'Standard_D1_v2' means: 1 vCPU, 3.5 GiB RAM opts = { priv = { - vm_size_binarycache = "Standard_D1_v2" - vm_size_builder = "Standard_D2_v3" - vm_size_controller = "Standard_D2_v3" - num_builders = 1 + vm_size_binarycache = "Standard_D1_v2" + vm_size_builder_x86 = "Standard_D2_v3" + vm_size_builder_aarch64 = "Standard_D2ps_v5" + vm_size_controller = "Standard_D2_v3" + num_builders_x86 = 1 + num_builders_aarch64 = 1 } dev = { - vm_size_binarycache = "Standard_D1_v2" - vm_size_builder = "Standard_D4_v3" - vm_size_controller = "Standard_D4_v3" - num_builders = 1 + vm_size_binarycache = "Standard_D1_v2" + vm_size_builder_x86 = "Standard_D4_v3" + vm_size_builder_aarch64 = "Standard_D4ps_v5" + vm_size_controller = "Standard_D4_v3" + num_builders_x86 = 1 + num_builders_aarch64 = 1 } prod = { - vm_size_binarycache = "Standard_D2_v3" - vm_size_builder = "Standard_D8_v3" - vm_size_controller = "Standard_D8_v3" - num_builders = 2 + vm_size_binarycache = "Standard_D2_v3" + vm_size_builder_x86 = "Standard_D8_v3" + vm_size_builder_aarch64 = "Standard_D8ps_v5" + vm_size_controller = "Standard_D8_v3" + num_builders_x86 = 2 + num_builders_aarch64 = 2 } } diff --git a/terraform/modules/arm-builder-vm/README.md b/terraform/modules/arm-builder-vm/README.md new file mode 100644 index 00000000..5d2a0e0b --- /dev/null +++ b/terraform/modules/arm-builder-vm/README.md @@ -0,0 +1,22 @@ + + +# arm-builder-vm + +Terraform module spinning up a Azure aarch64 VM with ubuntu and nix. + +Modified from `azurerm-linux-vm` + +This uses the `azurerm_virtual_machine` resource to spin up the VM, as it allows +data disks to be attached on boot. + +This is due to + + +- with `azurerm_linux_virtual_machine` and +`azurerm_virtual_machine_data_disk_attachment` the disk only gets attached once +the VM is booted up, and the VM can't boot up if it waits for the data disk +to appear. diff --git a/terraform/modules/arm-builder-vm/ubuntu-builder.sh b/terraform/modules/arm-builder-vm/ubuntu-builder.sh new file mode 100755 index 00000000..1e1f57a0 --- /dev/null +++ b/terraform/modules/arm-builder-vm/ubuntu-builder.sh @@ -0,0 +1,147 @@ +#!/usr/bin/env bash + +# SPDX-FileCopyrightText: 2024 Technology Innovation Institute (TII) +# +# SPDX-License-Identifier: Apache-2.0 + +set -x # debug + +################################################################################ + +# Assume root if HOME and USER are unset +[ -z "${HOME}" ] && export HOME="/root" +[ -z "${USER}" ] && export USER="root" + +################################################################################ + +apt_update() { + sudo apt-get update -y + sudo apt-get upgrade -y + sudo apt-get install -y ca-certificates curl xz-utils +} + +install_nix() { + type="$1" + if [ "$type" = "single" ]; then + # Single-user + sh <(curl -L https://nixos.org/nix/install) --yes --no-daemon + elif [ "$type" = "multi" ]; then + # Multi-user + sh <(curl -L https://nixos.org/nix/install) --yes --daemon + else + echo "Error: unknown installation type: '$type'" + exit 1 + fi + # Fix https://github.com/nix-community/home-manager/issues/3734: + sudo mkdir -m 0755 -p /nix/var/nix/{profiles,gcroots}/per-user/"$USER" + sudo chown -R "$USER:nixbld" "/nix/var/nix/profiles/per-user/$USER" + # Enable flakes + extra_nix_conf="experimental-features = nix-command flakes" + sudo sh -c "printf '$extra_nix_conf\n'>>/etc/nix/nix.conf" + # https://github.com/NixOS/nix/issues/1078#issuecomment-1019327751 + for f in /nix/var/nix/profiles/default/bin/nix*; do + sudo ln -fs "$f" "/usr/bin/$(basename "$f")" + done +} + +configure_builder() { + # Add user: nix + # Extra nix config for the builder, + # for detailed description of each of the below options see: + # https://nixos.org/manual/nix/stable/command-ref/conf-file + extra_nix_conf=" +# 20 GB (20*1024*1024*1024) +min-free = 21474836480 +# 200 GB (200*1024*1024*1024) +max-free = 214748364800 +system-features = nixos-test benchmark big-parallel kvm +trusted-users = remote-build +substituters = https://ghaf-dev.cachix.org?priority=20 https://cache.vedenemo.dev https://cache.nixos.org +trusted-public-keys = ghaf-dev.cachix.org-1:S3M8x3no8LFQPBfHw1jl6nmP8A7cVWKntoMKN3IsEQY= cache.vedenemo.dev:8NhplARANhClUSWJyLVk4WMyy1Wb4rhmWW2u8AejH9E= cache.nixos.org-1:6NCHdD59X431o0gWypbMrAURkbJ16ZPMQFGspcDShjY=" + sudo sh -c "printf '$extra_nix_conf\n'>>/etc/nix/nix.conf" +} + +restart_nix_daemon() { + # Re-start nix-daemon + if systemctl list-units | grep -iq "nix-daemon"; then + sudo systemctl restart nix-daemon + if ! systemctl status nix-daemon; then + echo "Error: nix-daemon failed to start" + exit 1 + fi + fi +} + +uninstall_nix() { + # https://github.com/NixOS/nix/issues/1402 + if grep -q nixbld /etc/passwd; then + grep nixbld /etc/passwd | awk -F ":" '{print $1}' | xargs -t -n 1 sudo userdel -r + fi + if grep -q nixbld /etc/group; then + sudo groupdel nixbld + fi + rm -rf "$HOME/"{.nix-channels,.nix-defexpr,.nix-profile,.config/nixpkgs,.config/nix,.config/home-manager,.local/state/nix,.local/state/home-manager} + sudo rm -rf /etc/profile.d/nix.sh + if [ -d "/nix" ]; then + sudo rm -rf /nix + fi + if [ -d "/etc/nix" ]; then + sudo rm -fr /etc/nix + fi + sudo find /etc -iname "*backup-before-nix*" -delete + sudo find -L /usr/bin -iname "nix*" -delete + [ -f "$HOME/.profile" ] && sed -i "/\/nix/d" "$HOME/.profile" + [ -f "$HOME/.bash_profile" ] && sed -i "/\/nix/d" "$HOME/.bash_profile" + [ -f "$HOME/.bashrc" ] && sed -i "/\/nix/d" "$HOME/.bashrc" + if systemctl list-units | grep -iq "nix-daemon"; then + sudo systemctl stop nix-daemon nix-daemon.socket + sudo systemctl disable nix-daemon nix-daemon.socket + sudo find /etc/systemd -iname "*nix-daemon*" -delete + sudo find /usr/lib/systemd -iname "*nix-daemon*" -delete + sudo systemctl daemon-reload + sudo systemctl reset-failed + fi + unset NIX_PATH +} + +outro() { + set +x + echo "" + nixpkgs_ver=$(nix-instantiate --eval -E '(import {}).lib.version' 2>/dev/null) + if [ -n "$nixpkgs_ver" ]; then + echo "Installed nixpkgs version: $nixpkgs_ver" + else + echo "Failed reading installed nixpkgs version" + exit 1 + fi + echo "" + echo "Open a new terminal for the changes to take impact" + echo "" +} + +exit_unless_command_exists() { + if ! command -v "$1" 2>/dev/null; then + echo "Error: command '$1' is not installed" >&2 + exit 1 + fi +} + +################################################################################ + +main() { + exit_unless_command_exists "apt-get" + exit_unless_command_exists "systemctl" + apt_update + uninstall_nix + install_nix "multi" + configure_builder + restart_nix_daemon + exit_unless_command_exists "nix-shell" + outro +} + +################################################################################ + +main "$@" + +################################################################################ diff --git a/terraform/modules/arm-builder-vm/variables.tf b/terraform/modules/arm-builder-vm/variables.tf new file mode 100644 index 00000000..a633ec9a --- /dev/null +++ b/terraform/modules/arm-builder-vm/variables.tf @@ -0,0 +1,39 @@ +# SPDX-FileCopyrightText: 2024 Technology Innovation Institute (TII) +# +# SPDX-License-Identifier: Apache-2.0 + +variable "resource_group_name" { + type = string +} + +variable "location" { + type = string +} + +variable "virtual_machine_name" { + type = string +} + +variable "virtual_machine_size" { + type = string +} + +variable "virtual_machine_custom_data" { + type = string + default = "" +} + +variable "allocate_public_ip" { + type = bool + default = false +} + +variable "subnet_id" { + type = string + description = "The subnet ID to attach to the VM and allocate an IP from" +} + +variable "data_disks" { + description = "List of dict containing keys of the storage_data_disk block" + default = [] +} diff --git a/terraform/modules/arm-builder-vm/virtual_machine.tf b/terraform/modules/arm-builder-vm/virtual_machine.tf new file mode 100644 index 00000000..eca22e9c --- /dev/null +++ b/terraform/modules/arm-builder-vm/virtual_machine.tf @@ -0,0 +1,149 @@ +# SPDX-FileCopyrightText: 2024 Technology Innovation Institute (TII) +# +# SPDX-License-Identifier: Apache-2.0 + +resource "azurerm_virtual_machine" "main" { + name = var.virtual_machine_name + resource_group_name = var.resource_group_name + location = var.location + vm_size = var.virtual_machine_size + + delete_os_disk_on_termination = true + delete_data_disks_on_termination = false + + network_interface_ids = [azurerm_network_interface.default.id] + + storage_image_reference { + publisher = "Canonical" + offer = "0001-com-ubuntu-server-jammy" + sku = "22_04-lts-arm64" + version = "latest" + } + + identity { + type = "SystemAssigned" + } + + os_profile { + computer_name = var.virtual_machine_name + # Unused, but required by the API. May not be root either + admin_username = "foo" + admin_password = "S00persecret" + + # We only set custom_data here, not user_data. + # user_data is more recent, and allows updates without recreating the machine, + # but at least cloud-init 23.1.2 blocks boot if custom_data is not set. + # (It logs about not being able to mount /dev/sr0 to /metadata). + # This can be worked around by setting custom_data to a static placeholder, + # but user_data is still ignored. + # TODO: check this again with a more recent cloud-init version. + custom_data = (var.virtual_machine_custom_data == "") ? null : base64encode(var.virtual_machine_custom_data) + } + + os_profile_linux_config { + # We *don't* support password auth, and this doesn't change anything. + # However, if we don't set this to false we need to + # specify additional pubkeys. + disable_password_authentication = false + # We can't use admin_ssh_key, as it only works for the admin_username. + } + + boot_diagnostics { + enabled = true + # azurerm_virtual_machine doesn't support the managed storage account + storage_uri = azurerm_storage_account.boot_diag.primary_blob_endpoint + } + + storage_os_disk { + name = "${var.virtual_machine_name}-osdisk" # needs to be unique + caching = "ReadWrite" + create_option = "FromImage" + managed_disk_type = "Standard_LRS" + disk_size_gb = "100" + } + + dynamic "storage_data_disk" { + for_each = var.data_disks + + content { + # use lookup here, so keys can be set optionally + name = lookup(storage_data_disk.value, "name", null) + caching = lookup(storage_data_disk.value, "caching", null) + create_option = "Attach" + # This has to be passed, even for "Attach" + disk_size_gb = lookup(storage_data_disk.value, "disk_size_gb", null) + lun = lookup(storage_data_disk.value, "lun", null) + + managed_disk_type = lookup(storage_data_disk.value, "managed_disk_type", null) + managed_disk_id = lookup(storage_data_disk.value, "managed_disk_id", null) + } + } +} + +resource "azurerm_network_interface" "default" { + name = "${var.virtual_machine_name}-nic" + resource_group_name = var.resource_group_name + location = var.location + + ip_configuration { + name = "internal" + subnet_id = var.subnet_id + private_ip_address_allocation = "Dynamic" + public_ip_address_id = (var.allocate_public_ip) ? azurerm_public_ip.default[0].id : null + } +} + +resource "azurerm_public_ip" "default" { + count = (var.allocate_public_ip) ? 1 : 0 + + name = "${var.virtual_machine_name}-pub-ip" + domain_name_label = var.virtual_machine_name + resource_group_name = var.resource_group_name + location = var.location + allocation_method = "Static" +} + +# Create a random string, and a storage account using that random string. +resource "random_string" "boot_diag" { + length = "8" + special = "false" + upper = false +} + +resource "azurerm_storage_account" "boot_diag" { + name = "${random_string.boot_diag.result}bootdiag" + resource_group_name = var.resource_group_name + location = var.location + account_tier = "Standard" + account_replication_type = "GRS" +} + +resource "azurerm_virtual_machine_extension" "deploy_ubuntu_builder" { + name = "${var.virtual_machine_name}-vmext" + virtual_machine_id = azurerm_virtual_machine.main.id + publisher = "Microsoft.Azure.Extensions" + type = "CustomScript" + type_handler_version = "2.1" + settings = <