-
Notifications
You must be signed in to change notification settings - Fork 20
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Add arm remote builder with ubuntu image
Signed-off-by: Joonas Rautiola <[email protected]>
- Loading branch information
Showing
10 changed files
with
458 additions
and
16 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,71 @@ | ||
| # SPDX-FileCopyrightText: 2022-2024 TII (SSRC) and the Ghaf contributors | ||
| # SPDX-License-Identifier: Apache-2.0 | ||
|
|
||
| locals { | ||
| arm_num_builders = local.opts[local.conf].num_builders_aarch64 | ||
| } | ||
|
|
||
| module "arm_builder_vm" { | ||
| source = "./modules/arm-builder-vm" | ||
|
|
||
| count = local.arm_num_builders | ||
|
|
||
| resource_group_name = azurerm_resource_group.infra.name | ||
| location = azurerm_resource_group.infra.location | ||
| virtual_machine_name = "ghaf-builder-aarch64-${count.index}-${local.ws}" | ||
| virtual_machine_size = local.opts[local.conf].vm_size_builder_aarch64 | ||
| virtual_machine_osdisk_size = local.opts[local.conf].osdisk_size_builder | ||
|
|
||
| virtual_machine_custom_data = join("\n", ["#cloud-config", yamlencode({ | ||
| users = [{ | ||
| name = "remote-build" | ||
| ssh_authorized_keys = [ | ||
| "${data.azurerm_key_vault_secret.ssh_remote_build_pub.value}" | ||
| ] | ||
| }] | ||
| write_files = [ | ||
| { | ||
| content = "AZURE_STORAGE_ACCOUNT_NAME=${data.azurerm_storage_account.binary_cache.name}", | ||
| "path" = "/var/lib/rclone-http/env" | ||
| } | ||
| ], | ||
| })]) | ||
|
|
||
| subnet_id = azurerm_subnet.builders.id | ||
| } | ||
|
|
||
| # Allow inbound SSH from the jenkins subnet (only) | ||
| resource "azurerm_network_interface_security_group_association" "arm_builder_vm" { | ||
| count = local.arm_num_builders | ||
|
|
||
| network_interface_id = module.arm_builder_vm[count.index].virtual_machine_network_interface_id | ||
| network_security_group_id = azurerm_network_security_group.arm_builder_vm[count.index].id | ||
| } | ||
|
|
||
| resource "azurerm_network_security_group" "arm_builder_vm" { | ||
| count = local.arm_num_builders | ||
|
|
||
| name = "arm-builder-vm-${count.index}" | ||
| resource_group_name = azurerm_resource_group.infra.name | ||
| location = azurerm_resource_group.infra.location | ||
|
|
||
| security_rule { | ||
| name = "AllowSSHFromJenkins" | ||
| priority = 400 | ||
| direction = "Inbound" | ||
| access = "Allow" | ||
| protocol = "Tcp" | ||
| source_port_range = "*" | ||
| destination_port_ranges = [22] | ||
| source_address_prefix = azurerm_subnet.jenkins.address_prefixes[0] | ||
| destination_address_prefix = "*" | ||
| } | ||
| } | ||
|
|
||
| # Allow the VMs to read from the binary cache bucket | ||
| resource "azurerm_role_assignment" "arm_builder_access_binary_cache" { | ||
| count = local.arm_num_builders | ||
| scope = data.azurerm_storage_container.binary_cache_1.resource_manager_id | ||
| role_definition_name = "Storage Blob Data Reader" | ||
| principal_id = module.arm_builder_vm[count.index].virtual_machine_identity_principal_id | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,21 @@ | ||
| <!-- | ||
| SPDX-FileCopyrightText: 2022-2024 TII (SSRC) and the Ghaf contributors | ||
| SPDX-License-Identifier: Apache-2.0 | ||
| --> | ||
|
|
||
| # arm-builder-vm | ||
|
|
||
| Terraform module spinning up a Azure aarch64 VM with ubuntu and nix. | ||
|
|
||
| Modified from `azurerm-linux-vm` | ||
|
|
||
| This uses the `azurerm_virtual_machine` resource to spin up the VM, as it allows | ||
| data disks to be attached on boot. | ||
|
|
||
| This is due to | ||
| <https://github.com/hashicorp/terraform-provider-azurerm/issues/6117> | ||
|
|
||
| - with `azurerm_linux_virtual_machine` and | ||
| `azurerm_virtual_machine_data_disk_attachment` the disk only gets attached once | ||
| the VM is booted up, and the VM can't boot up if it waits for the data disk | ||
| to appear. |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,147 @@ | ||
| #!/usr/bin/env bash | ||
|
|
||
| # SPDX-FileCopyrightText: 2022-2024 TII (SSRC) and the Ghaf contributors | ||
| # SPDX-License-Identifier: Apache-2.0 | ||
|
|
||
| set -x # debug | ||
|
|
||
| ################################################################################ | ||
|
|
||
| # Assume root if HOME and USER are unset | ||
| [ -z "${HOME}" ] && export HOME="/root" | ||
| [ -z "${USER}" ] && export USER="root" | ||
|
|
||
| ################################################################################ | ||
|
|
||
| apt_update() { | ||
| sudo apt-get update -y | ||
| sudo apt-get upgrade -y | ||
| sudo apt-get install -y ca-certificates curl xz-utils | ||
| } | ||
|
|
||
| install_nix() { | ||
| type="$1" | ||
| if [ "$type" = "single" ]; then | ||
| # Single-user | ||
| sh <(curl -L https://nixos.org/nix/install) --yes --no-daemon | ||
| elif [ "$type" = "multi" ]; then | ||
| # Multi-user | ||
| sh <(curl -L https://nixos.org/nix/install) --yes --daemon | ||
| else | ||
| echo "Error: unknown installation type: '$type'" | ||
| exit 1 | ||
| fi | ||
| # Fix https://github.com/nix-community/home-manager/issues/3734: | ||
| sudo mkdir -m 0755 -p /nix/var/nix/{profiles,gcroots}/per-user/"$USER" | ||
| sudo chown -R "$USER:nixbld" "/nix/var/nix/profiles/per-user/$USER" | ||
| # Enable flakes | ||
| extra_nix_conf="experimental-features = nix-command flakes" | ||
| sudo sh -c "printf '$extra_nix_conf\n'>>/etc/nix/nix.conf" | ||
| # https://github.com/NixOS/nix/issues/1078#issuecomment-1019327751 | ||
| for f in /nix/var/nix/profiles/default/bin/nix*; do | ||
| sudo ln -fs "$f" "/usr/bin/$(basename "$f")" | ||
| done | ||
| } | ||
|
|
||
| configure_builder() { | ||
| # Add user: remote-build | ||
| # Extra nix config for the builder, | ||
| # for detailed description of each of the below options see: | ||
| # https://nixos.org/manual/nix/stable/command-ref/conf-file | ||
| extra_nix_conf=" | ||
| # 20 GB (20*1024*1024*1024) | ||
| min-free = 21474836480 | ||
| # 500 GB (500*1024*1024*1024) | ||
| # osdisk size for prod builders | ||
| max-free = 536870912000 | ||
| system-features = nixos-test benchmark big-parallel kvm | ||
| trusted-users = remote-build | ||
| substituters = https://ghaf-dev.cachix.org?priority=20 https://cache.vedenemo.dev https://cache.nixos.org | ||
| trusted-public-keys = ghaf-dev.cachix.org-1:S3M8x3no8LFQPBfHw1jl6nmP8A7cVWKntoMKN3IsEQY= cache.vedenemo.dev:8NhplARANhClUSWJyLVk4WMyy1Wb4rhmWW2u8AejH9E= cache.nixos.org-1:6NCHdD59X431o0gWypbMrAURkbJ16ZPMQFGspcDShjY=" | ||
| sudo sh -c "printf '$extra_nix_conf\n'>>/etc/nix/nix.conf" | ||
| } | ||
|
|
||
| restart_nix_daemon() { | ||
| # Re-start nix-daemon | ||
| if systemctl list-units | grep -iq "nix-daemon"; then | ||
| sudo systemctl restart nix-daemon | ||
| if ! systemctl status nix-daemon; then | ||
| echo "Error: nix-daemon failed to start" | ||
| exit 1 | ||
| fi | ||
| fi | ||
| } | ||
|
|
||
| uninstall_nix() { | ||
| # https://github.com/NixOS/nix/issues/1402 | ||
| if grep -q nixbld /etc/passwd; then | ||
| grep nixbld /etc/passwd | awk -F ":" '{print $1}' | xargs -t -n 1 sudo userdel -r | ||
| fi | ||
| if grep -q nixbld /etc/group; then | ||
| sudo groupdel nixbld | ||
| fi | ||
| rm -rf "$HOME/"{.nix-channels,.nix-defexpr,.nix-profile,.config/nixpkgs,.config/nix,.config/home-manager,.local/state/nix,.local/state/home-manager} | ||
| sudo rm -rf /etc/profile.d/nix.sh | ||
| if [ -d "/nix" ]; then | ||
| sudo rm -rf /nix | ||
| fi | ||
| if [ -d "/etc/nix" ]; then | ||
| sudo rm -fr /etc/nix | ||
| fi | ||
| sudo find /etc -iname "*backup-before-nix*" -delete | ||
| sudo find -L /usr/bin -iname "nix*" -delete | ||
| [ -f "$HOME/.profile" ] && sed -i "/\/nix/d" "$HOME/.profile" | ||
| [ -f "$HOME/.bash_profile" ] && sed -i "/\/nix/d" "$HOME/.bash_profile" | ||
| [ -f "$HOME/.bashrc" ] && sed -i "/\/nix/d" "$HOME/.bashrc" | ||
| if systemctl list-units | grep -iq "nix-daemon"; then | ||
| sudo systemctl stop nix-daemon nix-daemon.socket | ||
| sudo systemctl disable nix-daemon nix-daemon.socket | ||
| sudo find /etc/systemd -iname "*nix-daemon*" -delete | ||
| sudo find /usr/lib/systemd -iname "*nix-daemon*" -delete | ||
| sudo systemctl daemon-reload | ||
| sudo systemctl reset-failed | ||
| fi | ||
| unset NIX_PATH | ||
| } | ||
|
|
||
| outro() { | ||
| set +x | ||
| echo "" | ||
| nixpkgs_ver=$(nix-instantiate --eval -E '(import <nixpkgs> {}).lib.version' 2>/dev/null) | ||
| if [ -n "$nixpkgs_ver" ]; then | ||
| echo "Installed nixpkgs version: $nixpkgs_ver" | ||
| else | ||
| echo "Failed reading installed nixpkgs version" | ||
| exit 1 | ||
| fi | ||
| echo "" | ||
| echo "Open a new terminal for the changes to take impact" | ||
| echo "" | ||
| } | ||
|
|
||
| exit_unless_command_exists() { | ||
| if ! command -v "$1" 2>/dev/null; then | ||
| echo "Error: command '$1' is not installed" >&2 | ||
| exit 1 | ||
| fi | ||
| } | ||
|
|
||
| ################################################################################ | ||
|
|
||
| main() { | ||
| exit_unless_command_exists "apt-get" | ||
| exit_unless_command_exists "systemctl" | ||
| apt_update | ||
| uninstall_nix | ||
| install_nix "multi" | ||
| configure_builder | ||
| restart_nix_daemon | ||
| exit_unless_command_exists "nix-shell" | ||
| outro | ||
| } | ||
|
|
||
| ################################################################################ | ||
|
|
||
| main "$@" | ||
|
|
||
| ################################################################################ |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,42 @@ | ||
| # SPDX-FileCopyrightText: 2022-2024 TII (SSRC) and the Ghaf contributors | ||
| # SPDX-License-Identifier: Apache-2.0 | ||
|
|
||
| variable "resource_group_name" { | ||
| type = string | ||
| } | ||
|
|
||
| variable "location" { | ||
| type = string | ||
| } | ||
|
|
||
| variable "virtual_machine_name" { | ||
| type = string | ||
| } | ||
|
|
||
| variable "virtual_machine_size" { | ||
| type = string | ||
| } | ||
|
|
||
| variable "virtual_machine_osdisk_size" { | ||
| type = string | ||
| } | ||
|
|
||
| variable "virtual_machine_custom_data" { | ||
| type = string | ||
| default = "" | ||
| } | ||
|
|
||
| variable "allocate_public_ip" { | ||
| type = bool | ||
| default = false | ||
| } | ||
|
|
||
| variable "subnet_id" { | ||
| type = string | ||
| description = "The subnet ID to attach to the VM and allocate an IP from" | ||
| } | ||
|
|
||
| variable "data_disks" { | ||
| description = "List of dict containing keys of the storage_data_disk block" | ||
| default = [] | ||
| } |
Oops, something went wrong.