Merge azure-images to main

.reuse/dep5

@@ -2,4 +2,4 @@ Format: https://www.debian.org/doc/packaging-manuals/copyright-format/1.0/
 
 Copyright: 2023 Technology Innovation Institute (TII)
 License: Apache-2.0
-Files: *.lock *.png *.svg *.csv *.yaml
+Files: *.lock *.png *.svg *.csv *.yaml *.pub

.sops.yaml

@@ -3,33 +3,18 @@
 # SPDX-License-Identifier: Apache-2.0
 
 keys:
-  - &build01 age1tcp86swx4c8y8ej666k27lwca60j0x5tf4mcnw459ccec4am9vqqg2ht9d
   - &flokli age1lvpj49ewyx9a4uxevl05wfevmqld2d25juc65acjqpmerzdpc9kq2kxdgs
-  - &ghafhydra age1qnufx7gvz5kmm48nvdma4chxd4p0lca88f5fsyce8lrae6gp2a8sul692y
   - &hrosten age1hc6hszepd5xezxkgd3yx74pn3scxjm5w6px48m4rq9yj7w6rke7q72zhgn
   - &karim age122lvqyrdqz30fkfututykl0yle9u63u2em6e4aut7e5draws83ns3npt3a
   - &jrautiola age15jq5gjjd7ypsdlqfjtqy4red57v8ggqq9na6u3xffznu678nydpsuuwjg0
   - &binarycache age1s47a3y44j695gemcl0kqgjlxxvaa50de9s69jy2l6vc8xtmk5pcskhpknl
   - &monitoring age17s9sc2cgt9t30cyl65zya8p4zmwnndrx2r896e7gzgl08sjn0qmq3t6shs
 creation_rules:
-  - path_regex: terraform/secrets.yaml$
+  - path_regex: terraform/azarm/secrets.yaml$
     key_groups:
     - age:
-      - *flokli
       - *hrosten
       - *karim
-  - path_regex: hosts/ghafhydra/secrets.yaml$
-    key_groups:
-    - age:
-      - *flokli
-      - *hrosten
-      - *ghafhydra
-  - path_regex: hosts/build01/secrets.yaml$
-    key_groups:
-    - age:
-      - *flokli
-      - *hrosten
-      - *build01
   - path_regex: hosts/binarycache/secrets.yaml$
     key_groups:
     - age:

docs/adapting-to-new-environments.md

@@ -164,7 +164,6 @@ $ cat hosts/mytarget/configuration.nix
   # Define the services you want to run on your target, as well as the users
   # who can access the target with ssh:
   imports = [
-    inputs.nix-serve-ng.nixosModules.default
     inputs.sops-nix.nixosModules.sops
     inputs.disko.nixosModules.disko
     ../generic-disk-config.nix

flake.nix

@@ -19,7 +19,8 @@
     # Binary cache with nix-serve-ng
     nix-serve-ng = {
       url = "github:aristanetworks/nix-serve-ng";
-      inputs.nixpkgs.follows = "nixpkgs";
+      # Broken with 23.11, base32 misses text >=2.0 && <2.1
+      # inputs.nixpkgs.follows = "nixpkgs";
     };
     # Disko for disk partitioning
     disko = {

hosts/azure-common.nix

@@ -1,41 +1,47 @@
 # SPDX-FileCopyrightText: 2023 Technology Innovation Institute (TII)
 #
 # SPDX-License-Identifier: Apache-2.0
-{inputs, ...}: {
-  require = [
-    "${inputs.nixpkgs}/nixos/modules/virtualisation/azure-agent.nix"
+#
+# Profile to import for Azure VMs. Imports azure-common.nix from nixpkgs,
+# and configures cloud-init.
+{
+  modulesPath,
+  pkgs,
+  ...
+}: {
+  imports = [
+    "${modulesPath}/virtualisation/azure-config.nix"
   ];
-  virtualisation.azure.agent.enable = true;
-  boot.kernelParams = ["console=ttyS0" "earlyprintk=ttyS0" "rootdelay=300" "panic=1" "boot.panic_on_fail"];
-  boot.initrd.kernelModules = ["hv_vmbus" "hv_netvsc" "hv_utils" "hv_storvsc"];
-  boot.loader.systemd-boot.enable = true;
-  boot.loader.efi.canTouchEfiVariables = true;
-  boot.loader.timeout = 0;
-  boot.loader.grub.configurationLimit = 0;
-  boot.growPartition = true;
 
-  # Ref:
-  # - https://github.com/NixOS/nixpkgs/blob/8efd5d1e283604f75a808a20e6cde0ef313d07d4/nixos/modules/virtualisation/azure-common.nix#L44
-  # - https://learn.microsoft.com/en-us/troubleshoot/azure/virtual-machines/troubleshoot-device-names-problems
-  services.udev.extraRules = ''
-    ENV{DEVTYPE}=="disk", KERNEL!="sda" SUBSYSTEM=="block", SUBSYSTEMS=="scsi", KERNELS=="?:0:0:0", ATTR{removable}=="0", SYMLINK+="disk/by-lun/0",
-    ENV{DEVTYPE}=="disk", KERNEL!="sda" SUBSYSTEM=="block", SUBSYSTEMS=="scsi", KERNELS=="?:0:0:1", ATTR{removable}=="0", SYMLINK+="disk/by-lun/1",
-    ENV{DEVTYPE}=="disk", KERNEL!="sda" SUBSYSTEM=="block", SUBSYSTEMS=="scsi", KERNELS=="?:0:0:2", ATTR{removable}=="0", SYMLINK+="disk/by-lun/2"
-    ENV{DEVTYPE}=="disk", KERNEL!="sda" SUBSYSTEM=="block", SUBSYSTEMS=="scsi", KERNELS=="?:0:0:3", ATTR{removable}=="0", SYMLINK+="disk/by-lun/3"
+  nix = {
+    settings = {
+      # Enable flakes and 'nix' command
+      experimental-features = "nix-command flakes";
+    };
+  };
 
-    ENV{DEVTYPE}=="disk", KERNEL!="sda" SUBSYSTEM=="block", SUBSYSTEMS=="scsi", KERNELS=="?:0:0:4", ATTR{removable}=="0", SYMLINK+="disk/by-lun/4"
-    ENV{DEVTYPE}=="disk", KERNEL!="sda" SUBSYSTEM=="block", SUBSYSTEMS=="scsi", KERNELS=="?:0:0:5", ATTR{removable}=="0", SYMLINK+="disk/by-lun/5"
-    ENV{DEVTYPE}=="disk", KERNEL!="sda" SUBSYSTEM=="block", SUBSYSTEMS=="scsi", KERNELS=="?:0:0:6", ATTR{removable}=="0", SYMLINK+="disk/by-lun/6"
-    ENV{DEVTYPE}=="disk", KERNEL!="sda" SUBSYSTEM=="block", SUBSYSTEMS=="scsi", KERNELS=="?:0:0:7", ATTR{removable}=="0", SYMLINK+="disk/by-lun/7"
+  # Enable azure agent
+  virtualisation.azure.agent.enable = true;
 
-    ENV{DEVTYPE}=="disk", KERNEL!="sda" SUBSYSTEM=="block", SUBSYSTEMS=="scsi", KERNELS=="?:0:0:8", ATTR{removable}=="0", SYMLINK+="disk/by-lun/8"
-    ENV{DEVTYPE}=="disk", KERNEL!="sda" SUBSYSTEM=="block", SUBSYSTEMS=="scsi", KERNELS=="?:0:0:9", ATTR{removable}=="0", SYMLINK+="disk/by-lun/9"
-    ENV{DEVTYPE}=="disk", KERNEL!="sda" SUBSYSTEM=="block", SUBSYSTEMS=="scsi", KERNELS=="?:0:0:10", ATTR{removable}=="0", SYMLINK+="disk/by-lun/10"
-    ENV{DEVTYPE}=="disk", KERNEL!="sda" SUBSYSTEM=="block", SUBSYSTEMS=="scsi", KERNELS=="?:0:0:11", ATTR{removable}=="0", SYMLINK+="disk/by-lun/11"
+  # enable cloud-init, so instance metadata is set accordingly and we can use
+  # cloud-config for ssh key management.
+  services.cloud-init.enable = true;
+  systemd.services.cloud-config.after = ["mnt-resource.mount"];
+  systemd.services.cloud-config.requires = ["mnt-resource.mount"];
 
-    ENV{DEVTYPE}=="disk", KERNEL!="sda" SUBSYSTEM=="block", SUBSYSTEMS=="scsi", KERNELS=="?:0:0:12", ATTR{removable}=="0", SYMLINK+="disk/by-lun/12"
-    ENV{DEVTYPE}=="disk", KERNEL!="sda" SUBSYSTEM=="block", SUBSYSTEMS=="scsi", KERNELS=="?:0:0:13", ATTR{removable}=="0", SYMLINK+="disk/by-lun/13"
-    ENV{DEVTYPE}=="disk", KERNEL!="sda" SUBSYSTEM=="block", SUBSYSTEMS=="scsi", KERNELS=="?:0:0:14", ATTR{removable}=="0", SYMLINK+="disk/by-lun/14"
-    ENV{DEVTYPE}=="disk", KERNEL!="sda" SUBSYSTEM=="block", SUBSYSTEMS=="scsi", KERNELS=="?:0:0:15", ATTR{removable}=="0", SYMLINK+="disk/by-lun/15"
-  '';
+  # Use systemd-networkd for network configuration.
+  services.cloud-init.network.enable = true;
+  networking.useDHCP = false;
+  networking.useNetworkd = true;
+  # FUTUREWORK: Ideally, we'd keep systemd-resolved disabled too,
+  # but the way nixpkgs configures cloud-init prevents it from picking up DNS
+  # settings from elsewhere.
+  # services.resolved.enable = false;
+
+  # List packages installed in system profile
+  environment.systemPackages = with pkgs; [
+    git
+    vim
+    htop
+  ];
 }

hosts/azure-scratch-store-common.nix

@@ -0,0 +1,110 @@
+# SPDX-FileCopyrightText: 2023 Technology Innovation Institute (TII)
+#
+# SPDX-License-Identifier: Apache-2.0
+#
+# This configuration is currently not used, but kept here for reference.
+# The reason this isn't currently used is that the 'setup-resource-disk'
+# service that's setup in this file systematically fails on the first
+# boot-up, which then cascades other service failures.
+# It fails to mount the resource disk in initrd on the first boot.
+#
+# The changes from this file were originally introduced in the following PR
+# https://github.com/tiiuae/ghaf-infra/pull/35 in commit:
+# https://github.com/tiiuae/ghaf-infra/commit/7a7a1e40b24b6776c70f7e030c7608ed90b40e45
+# Later, the scratch disk was disabled due to the reason explained above
+# and worked-around by mounting /nix/store on the osdisk with the following change:
+# https://github.com/tiiuae/ghaf-infra/commit/f143ac92517a3588d038e88eda09f19471e42de3
+#
+# Note: if we decice to re-enable this config at some later time, it's worth
+# mentioning that originally this configuration did not work on nixos-23.11
+# as described here:
+# https://github.com/tiiuae/ghaf-infra/commit/e9b7db1c02c459c0b8d54a4d65aac1d400f4035d
+#
+# At the time of writing, ghaf-infra main branch follows 23.11:
+# https://github.com/tiiuae/ghaf-infra/pull/74/commits/dd42bf9191f8133aaedb65aebb5756d8b4d567af
+# which means these changes would not work without also changing the ghaf-infra
+# nixpkgs reference.
+#
+{
+  pkgs,
+  utils,
+  ...
+}: {
+  # Disable explicit resource disk handling in waagent.
+  # We want to take control over it in initrd already.
+  virtualisation.azure.agent.mountResourceDisk = false;
+
+  boot.initrd.systemd = {
+    # This requires systemd-in-initrd.
+    enable = true;
+
+    # We need the wipefs binary available in the initrd
+    extraBin = {
+      "wipefs" = "${pkgs.util-linux}/bin/wipefs";
+    };
+
+    # The resource disk comes pre-formatted with NTFS, not ext4.
+    # Wipe the superblock if it's NTFS (and only then, to not wipe on every reboot).
+    # Once we get `filesystems`-syntax to work again, we could delegate the mkfs
+    # part to systemd-makefs (and make this `wantedBy` and `before` that makefs
+    # unit).
+    services.wipe-resource-disk = {
+      description = "Wipe resource disk before makefs";
+      requires = ["${utils.escapeSystemdPath "dev/disk/azure/resource-part1"}.device"];
+      after = ["${utils.escapeSystemdPath "dev/disk/azure/resource-part1"}.device"];
+      wantedBy = ["${utils.escapeSystemdPath "sysroot/mnt/resource"}.mount"];
+      before = ["${utils.escapeSystemdPath "sysroot/mnt/resource"}.mount"];
+
+      script = ''
+        if [[ $(wipefs --output=TYPE -p /dev/disk/azure/resource-part1) == "ntfs" ]]; then
+          echo "wiping resource disk (was ntfs)"
+          wipefs -a /dev/disk/azure/resource-part1
+          mkfs.ext4 /dev/disk/azure/resource-part1
+        else
+          echo "skip wiping resource disk (not ntfs)"
+        fi
+      '';
+    };
+
+    # Once /sysroot/mnt/resource is mounted, ensure the two .rw-store/
+    # {work,store} directories that overlayfs is using are present.
+    # The kernel doesn't create them on its own and fails the mount if they're
+    # not present, so we set `wantedBy` and `before` to the .mount unit.
+    services.setup-resource-disk = {
+      description = "Setup resource disk after it's mounted";
+      unitConfig.RequiresMountsFor = "/sysroot/mnt/resource";
+      wantedBy = ["${utils.escapeSystemdPath "sysroot/nix/store"}.mount"];
+      before = ["${utils.escapeSystemdPath "sysroot/nix/store"}.mount"];
+
+      script = ''
+        mkdir -p /sysroot/mnt/resource/.rw-store/{work,store}
+      '';
+    };
+
+    # These describe the mountpoints inside the initrd
+    # (/sysroot/mnt/resource, /sysroot/nix/store).
+    # In the future, this should be moved to `filesystems`-syntax, so we can
+    # make use of systemd-makefs and can write some things more concisely.
+    mounts = [
+      {
+        where = "/sysroot/mnt/resource";
+        what = "/dev/disk/azure/resource-part1";
+        type = "ext4";
+      }
+      # describe the overlay mount
+      {
+        where = "/sysroot/nix/store";
+        what = "overlay";
+        type = "overlay";
+        options = "lowerdir=/sysroot/nix/store,upperdir=/sysroot/mnt/resource/.rw-store/store,workdir=/sysroot/mnt/resource/.rw-store/work";
+        wantedBy = ["initrd-fs.target"];
+        before = ["initrd-fs.target"];
+        requires = ["setup-resource-disk.service"];
+        after = ["setup-resource-disk.service"];
+        unitConfig.RequiresMountsFor = ["/sysroot" "/sysroot/mnt/resource"];
+      }
+    ];
+  };
+  # load the overlay kernel module
+  boot.initrd.kernelModules = ["overlay"];
+}