diff --git a/.github/workflows/test_zfs_root_guide.yml b/.github/workflows/test_zfs_root_guide.yml index 8e9fabf04..e2b25bedc 100644 --- a/.github/workflows/test_zfs_root_guide.yml +++ b/.github/workflows/test_zfs_root_guide.yml @@ -51,12 +51,15 @@ jobs: sudo apt install --yes git jq parted sudo apt install --yes whois curl sudo apt install --yes arch-install-scripts + - uses: cachix/install-nix-action@v20 + with: + nix_path: nixpkgs=channel:nixos-unstable + - name: Test NixOS guide + run: | + sudo PATH="${PATH}" NIX_PATH="${NIX_PATH}" ./scripts/zfs_root_guide_test.sh nixos - name: Test Alpine Linux guide run: | sudo ./scripts/zfs_root_guide_test.sh alpine - - name: Test Root on ZFS maintenance guide - run: | - sudo ./scripts/zfs_root_guide_test.sh maintenance - name: Test Arch Linux guide run: | sudo ./scripts/zfs_root_guide_test.sh archlinux @@ -66,14 +69,8 @@ jobs: - name: Test RHEL guide run: | sudo ./scripts/zfs_root_guide_test.sh rhel - - uses: cachix/install-nix-action@v20 - with: - nix_path: nixpkgs=channel:nixos-unstable - - name: Test NixOS guide - run: | - sudo PATH="${PATH}" NIX_PATH="${NIX_PATH}" ./scripts/zfs_root_guide_test.sh nixos - uses: actions/upload-artifact@v3 with: name: installation-scripts path: | - *.sh \ No newline at end of file + *.sh diff --git a/docs/Getting Started/Alpine Linux/Root on ZFS.rst b/docs/Getting Started/Alpine Linux/Root on ZFS.rst index e6d7fba2b..b7972b01e 100644 --- a/docs/Getting Started/Alpine Linux/Root on ZFS.rst +++ b/docs/Getting Started/Alpine Linux/Root on ZFS.rst @@ -24,11 +24,6 @@ Alpine Linux Root on ZFS **ZFSBootMenu** -This tutorial is based on the GRUB bootloader. Due to its independent -implementation of a read-only ZFS driver, GRUB only supports a subset -of ZFS features on the boot pool. [In general, bootloader treat disks -as read-only to minimize the risk of damaging on-disk data.] - `ZFSBootMenu `__ is an alternative bootloader free of such limitations and has support for boot environments. Do not follow instructions on this page if you plan to use ZBM, @@ -44,14 +39,18 @@ configuration before reboot. You should only use well-tested pool features. Avoid using new features if data integrity is paramount. See, for example, `this comment `__. +**UEFI support only** + +Only UEFI is supported by this guide. + Preparation --------------------------- #. Disable Secure Boot. ZFS modules can not be loaded if Secure Boot is enabled. #. Download latest extended variant of `Alpine Linux live image - `__, - verify `checksum `__ + `__, + verify `checksum `__ and boot from it. .. code-block:: sh @@ -64,8 +63,8 @@ Preparation # check whether the download page exists # alpine version must be in sync with ci/cd test chroot tarball - curl --head --fail https://dl-cdn.alpinelinux.org/alpine/v3.18/releases/x86_64/alpine-extended-3.18.4-x86_64.iso - curl --head --fail https://dl-cdn.alpinelinux.org/alpine/v3.18/releases/x86_64/alpine-extended-3.18.4-x86_64.iso.asc + curl --head --fail https://dl-cdn.alpinelinux.org/alpine/v3.19/releases/x86_64/alpine-extended-3.19.0-x86_64.iso + curl --head --fail https://dl-cdn.alpinelinux.org/alpine/v3.19/releases/x86_64/alpine-extended-3.19.0-x86_64.iso.asc #. Login as root user. There is no password. #. Configure Internet @@ -208,7 +207,7 @@ Preparation #. Install bootloader programs and partition tool :: - apk add grub-bios grub-efi parted e2fsprogs cryptsetup util-linux + apk add parted e2fsprogs cryptsetup util-linux System Installation --------------------------- @@ -226,14 +225,10 @@ System Installation parted --script --align=optimal "${disk}" -- \ mklabel gpt \ - mkpart EFI 2MiB 1GiB \ - mkpart bpool 1GiB 5GiB \ - mkpart rpool 5GiB -$((SWAPSIZE + RESERVE))GiB \ + mkpart EFI 1MiB 4GiB \ + mkpart rpool 4GiB -$((SWAPSIZE + RESERVE))GiB \ mkpart swap -$((SWAPSIZE + RESERVE))GiB -"${RESERVE}"GiB \ - mkpart BIOS 1MiB 2MiB \ set 1 esp on \ - set 5 bios_grub on \ - set 5 legacy_boot on partprobe "${disk}" } @@ -260,55 +255,26 @@ System Installation done done -#. Setup encrypted swap. This is useful if the available memory is - small:: + +#. Setup temporary encrypted swap for this installation only. This is + useful if the available memory is small:: for i in ${DISK}; do - cryptsetup open --type plain --key-file /dev/random "${i}"-part4 "${i##*/}"-part4 - mkswap /dev/mapper/"${i##*/}"-part4 - swapon /dev/mapper/"${i##*/}"-part4 + cryptsetup open --type plain --key-file /dev/random "${i}"-part3 "${i##*/}"-part3 + mkswap /dev/mapper/"${i##*/}"-part3 + swapon /dev/mapper/"${i##*/}"-part3 done + #. Load ZFS kernel module .. code-block:: sh modprobe zfs -#. Create boot pool - :: - - # shellcheck disable=SC2046 - zpool create -o compatibility=legacy \ - -o ashift=12 \ - -o autotrim=on \ - -O acltype=posixacl \ - -O canmount=off \ - -O devices=off \ - -O normalization=formD \ - -O relatime=on \ - -O xattr=sa \ - -O mountpoint=/boot \ - -R "${MNT}" \ - bpool \ - mirror \ - $(for i in ${DISK}; do - printf '%s ' "${i}-part2"; - done) - - If not using a multi-disk setup, remove ``mirror``. - - You should not need to customize any of the options for the boot pool. - - GRUB does not support all of the zpool features. See ``spa_feature_names`` - in `grub-core/fs/zfs/zfs.c - `__. - This step creates a separate boot pool for ``/boot`` with the features - limited to only those that GRUB supports, allowing the root pool to use - any/all features. - #. Create root pool - :: + + - Unencrypted:: # shellcheck disable=SC2046 zpool create \ @@ -317,233 +283,83 @@ System Installation -R "${MNT}" \ -O acltype=posixacl \ -O canmount=off \ - -O compression=zstd \ -O dnodesize=auto \ -O normalization=formD \ -O relatime=on \ -O xattr=sa \ - -O mountpoint=/ \ + -O mountpoint=none \ rpool \ mirror \ $(for i in ${DISK}; do - printf '%s ' "${i}-part3"; + printf '%s ' "${i}-part2"; done) - If not using a multi-disk setup, remove ``mirror``. - #. Create root system container: - - Unencrypted - :: - zfs create \ - -o canmount=off \ - -o mountpoint=none \ - rpool/alpinelinux - - - Encrypted: - - Avoid ZFS send/recv when using native encryption, see `a ZFS developer's comment on this issue`__ and `this spreadsheet of bugs`__. A LUKS-based guide has yet to be written. Once compromised, changing password will not keep your - data safe. See ``zfs-change-key(8)`` for more info - - .. code-block:: sh - - zfs create \ - -o canmount=off \ - -o mountpoint=none \ - -o encryption=on \ - -o keylocation=prompt \ - -o keyformat=passphrase \ - rpool/alpinelinux - - You can automate this step (insecure) with: ``echo POOLPASS | zfs create ...``. + zfs create -o canmount=noauto -o mountpoint=legacy rpool/root Create system datasets, manage mountpoints with ``mountpoint=legacy`` :: - zfs create -o canmount=noauto -o mountpoint=/ rpool/alpinelinux/root - zfs mount rpool/alpinelinux/root - zfs create -o mountpoint=legacy rpool/alpinelinux/home - mkdir "${MNT}"/home - mount -t zfs rpool/alpinelinux/home "${MNT}"/home - zfs create -o mountpoint=legacy rpool/alpinelinux/var - zfs create -o mountpoint=legacy rpool/alpinelinux/var/lib - zfs create -o mountpoint=legacy rpool/alpinelinux/var/log - zfs create -o mountpoint=none bpool/alpinelinux - zfs create -o mountpoint=legacy bpool/alpinelinux/root - mkdir "${MNT}"/boot - mount -t zfs bpool/alpinelinux/root "${MNT}"/boot - mkdir -p "${MNT}"/var/log - mkdir -p "${MNT}"/var/lib - mount -t zfs rpool/alpinelinux/var/lib "${MNT}"/var/lib - mount -t zfs rpool/alpinelinux/var/log "${MNT}"/var/log - -#. Format and mount ESP + zfs create -o mountpoint=legacy rpool/home + mount -o X-mount.mkdir -t zfs rpool/root "${MNT}" + mount -o X-mount.mkdir -t zfs rpool/home "${MNT}"/home + +#. Format and mount ESP. Only one of them is used as /boot, you need to set up mirroring afterwards :: for i in ${DISK}; do mkfs.vfat -n EFI "${i}"-part1 - mkdir -p "${MNT}"/boot/efis/"${i##*/}"-part1 - mount -t vfat -o iocharset=iso8859-1 "${i}"-part1 "${MNT}"/boot/efis/"${i##*/}"-part1 done - mkdir -p "${MNT}"/boot/efi - mount -t vfat -o iocharset=iso8859-1 "$(echo "${DISK}" | sed "s|^ *||" | cut -f1 -d' '|| true)"-part1 "${MNT}"/boot/efi + for i in ${DISK}; do + mount -t vfat -o fmask=0077,dmask=0077,iocharset=iso8859-1,X-mount.mkdir "${i}"-part1 "${MNT}"/boot + break + done System Configuration --------------------------- -#. Workaround for GRUB to recognize predictable disk names:: - - export ZPOOL_VDEV_NAME_PATH=YES - #. Install system to disk .. code-block:: sh - BOOTLOADER=grub setup-disk -k lts -v "${MNT}" + BOOTLOADER=none setup-disk -k lts -v "${MNT}" - GRUB installation will fail and will be reinstalled later. The error message about ZFS kernel module can be ignored. .. ifconfig:: zfs_root_test # lts kernel will pull in tons of firmware - BOOTLOADER=grub setup-disk -k virt -v "${MNT}" - -#. Allow EFI system partition to fail at boot:: - - sed -i "s|vfat.*rw|vfat rw,nofail|" "${MNT}"/etc/fstab + BOOTLOADER=none setup-disk -k virt -v "${MNT}" -#. Chroot +#. Install rEFInd boot loader:: - .. code-block:: sh - - for i in /dev /proc /sys; do mkdir -p "${MNT}"/"${i}"; mount --rbind "${i}" "${MNT}"/"${i}"; done - chroot "${MNT}" /usr/bin/env DISK="${DISK}" sh + # from http://www.rodsbooks.com/refind/getting.html + # use Binary Zip File option + apk add curl + curl -L http://sourceforge.net/projects/refind/files/0.14.0.2/refind-bin-0.14.0.2.zip/download --output refind.zip + unzip refind - .. ifconfig:: zfs_root_test + mkdir -p "${MNT}"/boot/EFI/BOOT + find ./refind-bin-0.14.0.2/ -name 'refind_x64.efi' -print0 \ + | xargs -0I{} mv {} "${MNT}"/boot/EFI/BOOT/BOOTX64.EFI + rm -rf refind.zip refind-bin-0.14.0.2 - :: +#. Add boot entry:: - for i in /dev /proc /sys; do mkdir -p "${MNT}"/"${i}"; mount --rbind "${i}" "${MNT}"/"${i}"; done - chroot "${MNT}" /usr/bin/env DISK="${DISK}" sh <<-'ZFS_ROOT_NESTED_CHROOT' + tee -a "${MNT}"/boot/refind-linux.conf <> /etc/profile.d/zpool_vdev_name_path.sh - # shellcheck disable=SC1091 - . /etc/profile.d/zpool_vdev_name_path.sh - - # GRUB fails to detect rpool name, hard code as "rpool" - sed -i "s|rpool=.*|rpool=rpool|" /etc/grub.d/10_linux - - # BusyBox stat does not recognize zfs, replace fs detection with ZFS - sed -i 's|stat -f -c %T /|echo zfs|' /usr/sbin/grub-mkconfig - - # grub-probe fails to identify fs mounted at /boot - BOOT_DEVICE=$(zpool status -P bpool | grep -- -part2 | head -n1 | sed "s|.*/dev*|/dev|" | sed "s|part2.*|part2|") - sed -i "s|GRUB_DEVICE_BOOT=.*|GRUB_DEVICE_BOOT=${BOOT_DEVICE}|" /usr/sbin/grub-mkconfig - - The ``sed`` workaround for ``grub-mkconfig`` needs to be applied - for every GRUB update, as the update will overwrite the changes. - -#. Install GRUB:: - - mkdir -p /boot/efi/alpine/grub-bootdir/i386-pc/ - mkdir -p /boot/efi/alpine/grub-bootdir/x86_64-efi/ - for i in ${DISK}; do - grub-install --target=i386-pc --boot-directory \ - /boot/efi/alpine/grub-bootdir/i386-pc/ "${i}" - done - grub-install --target x86_64-efi --boot-directory \ - /boot/efi/alpine/grub-bootdir/x86_64-efi/ --efi-directory \ - /boot/efi --bootloader-id alpine --removable - if test -d /sys/firmware/efi/efivars/; then - apk add efibootmgr - grub-install --target x86_64-efi --boot-directory \ - /boot/efi/alpine/grub-bootdir/x86_64-efi/ --efi-directory \ - /boot/efi --bootloader-id alpine - fi - -#. Generate GRUB menu:: - - mkdir -p /boot/grub - grub-mkconfig -o /boot/grub/grub.cfg - cp /boot/grub/grub.cfg \ - /boot/efi/alpine/grub-bootdir/x86_64-efi/grub/grub.cfg - cp /boot/grub/grub.cfg \ - /boot/efi/alpine/grub-bootdir/i386-pc/grub/grub.cfg - - .. ifconfig:: zfs_root_test - - :: - - find /boot/efis/ -name "grub.cfg" -print0 \ - | xargs -t -0I '{}' grub-script-check -v '{}' - -#. For both legacy and EFI booting: mirror ESP content:: - - espdir=$(mktemp -d) - find /boot/efi/ -maxdepth 1 -mindepth 1 -type d -print0 \ - | xargs -t -0I '{}' cp -r '{}' "${espdir}" - find "${espdir}" -maxdepth 1 -mindepth 1 -type d -print0 \ - | xargs -t -0I '{}' sh -vxc "find /boot/efis/ -maxdepth 1 -mindepth 1 -type d -print0 | xargs -t -0I '[]' cp -r '{}' '[]'" - - .. ifconfig:: zfs_root_test - - :: - - ################################################## - # - # - # MAINTENANCE SCRIPT ENTRY POINT - # DO NOT TOUCH - # - # - ################################################# - -#. Exit chroot - - .. code-block:: sh - - exit - - .. ifconfig:: zfs_root_test - - # nested chroot ends here - ZFS_ROOT_NESTED_CHROOT - - .. ifconfig:: zfs_root_test - - :: - - # list contents of boot dir to confirm - # that the mirroring succeeded - find "${MNT}"/boot/efis/ -type d > list_of_efi_dirs - for i in ${DISK}; do - if ! grep "${i##*/}-part1/efi\|${i##*/}-part1/EFI" list_of_efi_dirs; then - echo "disk ${i} not found in efi system partition, installation error"; - cat list_of_efi_dirs - exit 1 - fi - done - -#. Unmount filesystems and create initial system snapshot - You can later create a boot environment from this snapshot. - See `Root on ZFS maintenance page <../zfs_root_maintenance.html>`__. - :: +#. Unmount filesystems and create initial system snapshot:: umount -Rl "${MNT}" zfs snapshot -r rpool@initial-installation - zfs snapshot -r bpool@initial-installation zpool export -a #. Reboot @@ -552,10 +368,5 @@ System Configuration reboot - .. ifconfig:: zfs_root_test - - # chroot ends here - ZFS_ROOT_GUIDE_TEST - -.. _a ZFS developer's comment on this issue: https://ol.reddit.com/r/zfs/comments/10n8fsn/does_openzfs_have_a_new_developer_for_the_native/j6b8k1m/ -.. _this spreadsheet of bugs: https://docs.google.com/spreadsheets/d/1OfRSXibZ2nIE9DGK6swwBZXgXwdCPKgp4SbPZwTexCg/htmlview +#. Mount other EFI system partitions then set up a service for syncing + their contents. diff --git a/docs/Getting Started/Arch Linux/Root on ZFS.rst b/docs/Getting Started/Arch Linux/Root on ZFS.rst index f879ea605..593f572b9 100644 --- a/docs/Getting Started/Arch Linux/Root on ZFS.rst +++ b/docs/Getting Started/Arch Linux/Root on ZFS.rst @@ -29,11 +29,6 @@ Arch Linux Root on ZFS **ZFSBootMenu** -This tutorial is based on the GRUB bootloader. Due to its independent -implementation of a read-only ZFS driver, GRUB only supports a subset -of ZFS features on the boot pool. [In general, bootloader treat disks -as read-only to minimize the risk of damaging on-disk data.] - `ZFSBootMenu `__ is an alternative bootloader free of such limitations and has support for boot environments. Do not follow instructions on this page if you plan to use ZBM, @@ -49,6 +44,10 @@ configuration before reboot. You should only use well-tested pool features. Avoid using new features if data integrity is paramount. See, for example, `this comment `__. +**UEFI support only** + +Only UEFI is supported by this guide. + Preparation --------------------------- @@ -59,8 +58,8 @@ Preparation Download latest extended variant of `Alpine Linux live image - `__, - verify `checksum `__ + `__, + verify `checksum `__ and boot from it. .. code-block:: sh @@ -224,14 +223,10 @@ System Installation parted --script --align=optimal "${disk}" -- \ mklabel gpt \ - mkpart EFI 2MiB 1GiB \ - mkpart bpool 1GiB 5GiB \ - mkpart rpool 5GiB -$((SWAPSIZE + RESERVE))GiB \ + mkpart EFI 1MiB 4GiB \ + mkpart rpool 4GiB -$((SWAPSIZE + RESERVE))GiB \ mkpart swap -$((SWAPSIZE + RESERVE))GiB -"${RESERVE}"GiB \ - mkpart BIOS 1MiB 2MiB \ set 1 esp on \ - set 5 bios_grub on \ - set 5 legacy_boot on partprobe "${disk}" } @@ -242,7 +237,7 @@ System Installation .. ifconfig:: zfs_root_test - :: + :: # When working with GitHub chroot runners, we are using loop # devices as installation target. However, the alias support for @@ -258,55 +253,26 @@ System Installation done done -#. Setup encrypted swap. This is useful if the available memory is - small:: + +#. Setup temporary encrypted swap for this installation only. This is + useful if the available memory is small:: for i in ${DISK}; do - cryptsetup open --type plain --key-file /dev/random "${i}"-part4 "${i##*/}"-part4 - mkswap /dev/mapper/"${i##*/}"-part4 - swapon /dev/mapper/"${i##*/}"-part4 + cryptsetup open --type plain --key-file /dev/random "${i}"-part3 "${i##*/}"-part3 + mkswap /dev/mapper/"${i##*/}"-part3 + swapon /dev/mapper/"${i##*/}"-part3 done + #. Load ZFS kernel module .. code-block:: sh modprobe zfs -#. Create boot pool - :: - - # shellcheck disable=SC2046 - zpool create -o compatibility=legacy \ - -o ashift=12 \ - -o autotrim=on \ - -O acltype=posixacl \ - -O canmount=off \ - -O devices=off \ - -O normalization=formD \ - -O relatime=on \ - -O xattr=sa \ - -O mountpoint=/boot \ - -R "${MNT}" \ - bpool \ - mirror \ - $(for i in ${DISK}; do - printf '%s ' "${i}-part2"; - done) - - If not using a multi-disk setup, remove ``mirror``. - - You should not need to customize any of the options for the boot pool. - - GRUB does not support all of the zpool features. See ``spa_feature_names`` - in `grub-core/fs/zfs/zfs.c - `__. - This step creates a separate boot pool for ``/boot`` with the features - limited to only those that GRUB supports, allowing the root pool to use - any/all features. - #. Create root pool - :: + + - Unencrypted:: # shellcheck disable=SC2046 zpool create \ @@ -315,80 +281,42 @@ System Installation -R "${MNT}" \ -O acltype=posixacl \ -O canmount=off \ - -O compression=zstd \ -O dnodesize=auto \ -O normalization=formD \ -O relatime=on \ -O xattr=sa \ - -O mountpoint=/ \ + -O mountpoint=none \ rpool \ mirror \ $(for i in ${DISK}; do - printf '%s ' "${i}-part3"; + printf '%s ' "${i}-part2"; done) - If not using a multi-disk setup, remove ``mirror``. - #. Create root system container: - - Unencrypted - :: - zfs create \ - -o canmount=off \ - -o mountpoint=none \ - rpool/archlinux - - - Encrypted: - - Avoid ZFS send/recv when using native encryption, see `a ZFS developer's comment on this issue`__ and `this spreadsheet of bugs`__. A LUKS-based guide has yet to be written. Once compromised, changing password will not keep your - data safe. See ``zfs-change-key(8)`` for more info - - .. code-block:: sh - - zfs create \ - -o canmount=off \ - -o mountpoint=none \ - -o encryption=on \ - -o keylocation=prompt \ - -o keyformat=passphrase \ - rpool/archlinux - - You can automate this step (insecure) with: ``echo POOLPASS | zfs create ...``. + zfs create -o canmount=noauto -o mountpoint=legacy rpool/root Create system datasets, manage mountpoints with ``mountpoint=legacy`` :: - zfs create -o canmount=noauto -o mountpoint=/ rpool/archlinux/root - zfs mount rpool/archlinux/root - zfs create -o mountpoint=legacy rpool/archlinux/home - mkdir "${MNT}"/home - mount -t zfs rpool/archlinux/home "${MNT}"/home - zfs create -o mountpoint=legacy rpool/archlinux/var - zfs create -o mountpoint=legacy rpool/archlinux/var/lib - zfs create -o mountpoint=legacy rpool/archlinux/var/log - zfs create -o mountpoint=none bpool/archlinux - zfs create -o mountpoint=legacy bpool/archlinux/root - mkdir "${MNT}"/boot - mount -t zfs bpool/archlinux/root "${MNT}"/boot - mkdir -p "${MNT}"/var/log - mkdir -p "${MNT}"/var/lib - mount -t zfs rpool/archlinux/var/lib "${MNT}"/var/lib - mount -t zfs rpool/archlinux/var/log "${MNT}"/var/log - -#. Format and mount ESP + zfs create -o mountpoint=legacy rpool/home + mount -o X-mount.mkdir -t zfs rpool/root "${MNT}" + mount -o X-mount.mkdir -t zfs rpool/home "${MNT}"/home + +#. Format and mount ESP. Only one of them is used as /boot, you need to set up mirroring afterwards :: for i in ${DISK}; do mkfs.vfat -n EFI "${i}"-part1 - mkdir -p "${MNT}"/boot/efis/"${i##*/}"-part1 - mount -t vfat -o iocharset=iso8859-1 "${i}"-part1 "${MNT}"/boot/efis/"${i##*/}"-part1 done - mkdir -p "${MNT}"/boot/efi - mount -t vfat -o iocharset=iso8859-1 "$(echo "${DISK}" | sed "s|^ *||" | cut -f1 -d' '|| true)"-part1 "${MNT}"/boot/efi + for i in ${DISK}; do + mount -t vfat -o fmask=0077,dmask=0077,iocharset=iso8859-1,X-mount.mkdir "${i}"-part1 "${MNT}"/boot + break + done System Configuration --------------------------- @@ -398,10 +326,10 @@ System Configuration apk add curl curl --fail-early --fail -L \ - https://america.archive.pkgbuild.com/iso/2023.09.01/archlinux-bootstrap-x86_64.tar.gz \ + https://america.archive.pkgbuild.com/iso/2024.01.01/archlinux-bootstrap-x86_64.tar.gz \ -o rootfs.tar.gz curl --fail-early --fail -L \ - https://america.archive.pkgbuild.com/iso/2023.09.01/archlinux-bootstrap-x86_64.tar.gz.sig \ + https://america.archive.pkgbuild.com/iso/2024.01.01/archlinux-bootstrap-x86_64.tar.gz.sig \ -o rootfs.tar.gz.sig apk add gnupg @@ -493,7 +421,7 @@ System Configuration #. Install base packages:: pacman -Sy - pacman -S --noconfirm mg mandoc grub efibootmgr mkinitcpio + pacman -S --noconfirm mg mandoc efibootmgr mkinitcpio kernel_compatible_with_zfs="$(pacman -Si zfs-linux \ | grep 'Depends On' \ @@ -505,7 +433,6 @@ System Configuration pacman -S --noconfirm zfs-linux zfs-utils - #. Configure mkinitcpio:: sed -i 's|filesystems|zfs filesystems|' /etc/mkinitcpio.conf @@ -550,66 +477,24 @@ System Configuration Bootloader --------------------------- +#. Install rEFInd boot loader:: -#. Apply GRUB workaround - - :: - - echo 'export ZPOOL_VDEV_NAME_PATH=YES' >> /etc/profile.d/zpool_vdev_name_path.sh - # shellcheck disable=SC1091 - . /etc/profile.d/zpool_vdev_name_path.sh - - # GRUB fails to detect rpool name, hard code as "rpool" - sed -i "s|rpool=.*|rpool=rpool|" /etc/grub.d/10_linux - - This workaround needs to be applied for every GRUB update, as the - update will overwrite the changes. - -#. Install GRUB:: - - mkdir -p /boot/efi/archlinux/grub-bootdir/i386-pc/ - mkdir -p /boot/efi/archlinux/grub-bootdir/x86_64-efi/ - for i in ${DISK}; do - grub-install --target=i386-pc --boot-directory \ - /boot/efi/archlinux/grub-bootdir/i386-pc/ "${i}" - done - grub-install --target x86_64-efi --boot-directory \ - /boot/efi/archlinux/grub-bootdir/x86_64-efi/ --efi-directory \ - /boot/efi --bootloader-id archlinux --removable - if test -d /sys/firmware/efi/efivars/; then - grub-install --target x86_64-efi --boot-directory \ - /boot/efi/archlinux/grub-bootdir/x86_64-efi/ --efi-directory \ - /boot/efi --bootloader-id archlinux - fi - - -#. Import both bpool and rpool at boot:: + # from http://www.rodsbooks.com/refind/getting.html + # use Binary Zip File option + pacman -S --noconfirm unzip + curl -L http://sourceforge.net/projects/refind/files/0.14.0.2/refind-bin-0.14.0.2.zip/download --output refind.zip - echo 'GRUB_CMDLINE_LINUX="zfs_import_dir=/dev/"' >> /etc/default/grub + unzip refind.zip + mkdir -p /boot/EFI/BOOT + find ./refind-bin-0.14.0.2/ -name 'refind_x64.efi' -print0 \ + | xargs -0I{} mv {} /boot/EFI/BOOT/BOOTX64.EFI + rm -rf refind.zip refind-bin-0.14.0.2 -#. Generate GRUB menu:: +#. Add boot entry:: - mkdir -p /boot/grub - grub-mkconfig -o /boot/grub/grub.cfg - cp /boot/grub/grub.cfg \ - /boot/efi/archlinux/grub-bootdir/x86_64-efi/grub/grub.cfg - cp /boot/grub/grub.cfg \ - /boot/efi/archlinux/grub-bootdir/i386-pc/grub/grub.cfg - - .. ifconfig:: zfs_root_test - - :: - - find /boot/efis/ -name "grub.cfg" -print0 \ - | xargs -t -0I '{}' grub-script-check -v '{}' - -#. For both legacy and EFI booting: mirror ESP content:: - - espdir=$(mktemp -d) - find /boot/efi/ -maxdepth 1 -mindepth 1 -type d -print0 \ - | xargs -t -0I '{}' cp -r '{}' "${espdir}" - find "${espdir}" -maxdepth 1 -mindepth 1 -type d -print0 \ - | xargs -t -0I '{}' sh -vxc "find /boot/efis/ -maxdepth 1 -mindepth 1 -type d -print0 | xargs -t -0I '[]' cp -r '{}' '[]'" + tee -a /boot/refind-linux.conf < list_of_efi_dirs - for i in ${DISK}; do - if ! grep "${i##*/}-part1/efi\|${i##*/}-part1/EFI" list_of_efi_dirs; then - echo "disk ${i} not found in efi system partition, installation error"; - cat list_of_efi_dirs - exit 1 - fi - done #. Unmount filesystems and create initial system snapshot - You can later create a boot environment from this snapshot. - See `Root on ZFS maintenance page <../zfs_root_maintenance.html>`__. :: umount -Rl "${MNT}" zfs snapshot -r rpool@initial-installation - zfs snapshot -r bpool@initial-installation #. Export all pools @@ -668,5 +536,5 @@ Bootloader # chroot ends here ZFS_ROOT_GUIDE_TEST -.. _a ZFS developer's comment on this issue: https://ol.reddit.com/r/zfs/comments/10n8fsn/does_openzfs_have_a_new_developer_for_the_native/j6b8k1m/ -.. _this spreadsheet of bugs: https://docs.google.com/spreadsheets/d/1OfRSXibZ2nIE9DGK6swwBZXgXwdCPKgp4SbPZwTexCg/htmlview +#. Mount other EFI system partitions then set up a service for syncing + their contents. diff --git a/docs/Getting Started/Fedora/Root on ZFS.rst b/docs/Getting Started/Fedora/Root on ZFS.rst index 6b00e77f6..76d75c180 100644 --- a/docs/Getting Started/Fedora/Root on ZFS.rst +++ b/docs/Getting Started/Fedora/Root on ZFS.rst @@ -30,11 +30,6 @@ Fedora Root on ZFS **ZFSBootMenu** -This tutorial is based on the GRUB bootloader. Due to its independent -implementation of a read-only ZFS driver, GRUB only supports a subset -of ZFS features on the boot pool. [In general, bootloader treat disks -as read-only to minimize the risk of damaging on-disk data.] - `ZFSBootMenu `__ is an alternative bootloader free of such limitations and has support for boot environments. Do not follow instructions on this page if you plan to use ZBM, @@ -50,6 +45,10 @@ configuration before reboot. You should only use well-tested pool features. Avoid using new features if data integrity is paramount. See, for example, `this comment `__. +**UEFI support only** + +Only UEFI is supported by this guide. + Preparation --------------------------- @@ -60,8 +59,8 @@ Preparation Download latest extended variant of `Alpine Linux live image - `__, - verify `checksum `__ + `__, + verify `checksum `__ and boot from it. .. code-block:: sh @@ -228,14 +227,10 @@ System Installation parted --script --align=optimal "${disk}" -- \ mklabel gpt \ - mkpart EFI 2MiB 1GiB \ - mkpart bpool 1GiB 5GiB \ - mkpart rpool 5GiB -$((SWAPSIZE + RESERVE))GiB \ + mkpart EFI 1MiB 4GiB \ + mkpart rpool 4GiB -$((SWAPSIZE + RESERVE))GiB \ mkpart swap -$((SWAPSIZE + RESERVE))GiB -"${RESERVE}"GiB \ - mkpart BIOS 1MiB 2MiB \ set 1 esp on \ - set 5 bios_grub on \ - set 5 legacy_boot on partprobe "${disk}" } @@ -246,7 +241,7 @@ System Installation .. ifconfig:: zfs_root_test - :: + :: # When working with GitHub chroot runners, we are using loop # devices as installation target. However, the alias support for @@ -262,13 +257,14 @@ System Installation done done -#. Setup encrypted swap. This is useful if the available memory is - small:: + +#. Setup temporary encrypted swap for this installation only. This is + useful if the available memory is small:: for i in ${DISK}; do - cryptsetup open --type plain --key-file /dev/random "${i}"-part4 "${i##*/}"-part4 - mkswap /dev/mapper/"${i##*/}"-part4 - swapon /dev/mapper/"${i##*/}"-part4 + cryptsetup open --type plain --key-file /dev/random "${i}"-part3 "${i##*/}"-part3 + mkswap /dev/mapper/"${i##*/}"-part3 + swapon /dev/mapper/"${i##*/}"-part3 done @@ -278,40 +274,9 @@ System Installation modprobe zfs -#. Create boot pool - :: - - # shellcheck disable=SC2046 - zpool create -o compatibility=legacy \ - -o ashift=12 \ - -o autotrim=on \ - -O acltype=posixacl \ - -O canmount=off \ - -O devices=off \ - -O normalization=formD \ - -O relatime=on \ - -O xattr=sa \ - -O mountpoint=/boot \ - -R "${MNT}" \ - bpool \ - mirror \ - $(for i in ${DISK}; do - printf '%s ' "${i}-part2"; - done) - - If not using a multi-disk setup, remove ``mirror``. - - You should not need to customize any of the options for the boot pool. - - GRUB does not support all of the zpool features. See ``spa_feature_names`` - in `grub-core/fs/zfs/zfs.c - `__. - This step creates a separate boot pool for ``/boot`` with the features - limited to only those that GRUB supports, allowing the root pool to use - any/all features. - #. Create root pool - :: + + - Unencrypted:: # shellcheck disable=SC2046 zpool create \ @@ -320,80 +285,43 @@ System Installation -R "${MNT}" \ -O acltype=posixacl \ -O canmount=off \ - -O compression=zstd \ -O dnodesize=auto \ -O normalization=formD \ -O relatime=on \ -O xattr=sa \ - -O mountpoint=/ \ + -O mountpoint=none \ rpool \ mirror \ $(for i in ${DISK}; do - printf '%s ' "${i}-part3"; + printf '%s ' "${i}-part2"; done) - If not using a multi-disk setup, remove ``mirror``. - #. Create root system container: - - Unencrypted - :: - zfs create \ - -o canmount=off \ - -o mountpoint=none \ - rpool/fedora - - - Encrypted: - - Avoid ZFS send/recv when using native encryption, see `a ZFS developer's comment on this issue`__ and `this spreadsheet of bugs`__. A LUKS-based guide has yet to be written. Once compromised, changing password will not keep your - data safe. See ``zfs-change-key(8)`` for more info - - .. code-block:: sh - - zfs create \ - -o canmount=off \ - -o mountpoint=none \ - -o encryption=on \ - -o keylocation=prompt \ - -o keyformat=passphrase \ - rpool/fedora - - You can automate this step (insecure) with: ``echo POOLPASS | zfs create ...``. + # dracut demands system root dataset to have non-legacy mountpoint + zfs create -o canmount=noauto -o mountpoint=/ rpool/root Create system datasets, manage mountpoints with ``mountpoint=legacy`` :: - zfs create -o canmount=noauto -o mountpoint=/ rpool/fedora/root - zfs mount rpool/fedora/root - zfs create -o mountpoint=legacy rpool/fedora/home - mkdir "${MNT}"/home - mount -t zfs rpool/fedora/home "${MNT}"/home - zfs create -o mountpoint=legacy rpool/fedora/var - zfs create -o mountpoint=legacy rpool/fedora/var/lib - zfs create -o mountpoint=legacy rpool/fedora/var/log - zfs create -o mountpoint=none bpool/fedora - zfs create -o mountpoint=legacy bpool/fedora/root - mkdir "${MNT}"/boot - mount -t zfs bpool/fedora/root "${MNT}"/boot - mkdir -p "${MNT}"/var/log - mkdir -p "${MNT}"/var/lib - mount -t zfs rpool/fedora/var/lib "${MNT}"/var/lib - mount -t zfs rpool/fedora/var/log "${MNT}"/var/log - -#. Format and mount ESP + zfs create -o mountpoint=legacy rpool/home + zfs mount rpool/root + mount -o X-mount.mkdir -t zfs rpool/home "${MNT}"/home + +#. Format and mount ESP. Only one of them is used as /boot, you need to set up mirroring afterwards :: for i in ${DISK}; do mkfs.vfat -n EFI "${i}"-part1 - mkdir -p "${MNT}"/boot/efis/"${i##*/}"-part1 - mount -t vfat -o iocharset=iso8859-1 "${i}"-part1 "${MNT}"/boot/efis/"${i##*/}"-part1 done - mkdir -p "${MNT}"/boot/efi - mount -t vfat -o iocharset=iso8859-1 "$(echo "${DISK}" | sed "s|^ *||" | cut -f1 -d' '|| true)"-part1 "${MNT}"/boot/efi + for i in ${DISK}; do + mount -t vfat -o fmask=0077,dmask=0077,iocharset=iso8859-1,X-mount.mkdir "${i}"-part1 "${MNT}"/boot + break + done System Configuration --------------------------- @@ -402,10 +330,10 @@ System Configuration apk add curl curl --fail-early --fail -L \ - https://dl.fedoraproject.org/pub/fedora/linux/releases/38/Container/x86_64/images/Fedora-Container-Base-38-1.6.x86_64.tar.xz \ + https://dl.fedoraproject.org/pub/fedora/linux/releases/39/Container/x86_64/images/Fedora-Container-Base-39-1.5.x86_64.tar.xz \ -o rootfs.tar.gz curl --fail-early --fail -L \ - https://dl.fedoraproject.org/pub/fedora/linux/releases/38/Container/x86_64/images/Fedora-Container-38-1.6-x86_64-CHECKSUM \ + https://dl.fedoraproject.org/pub/fedora/linux/releases/39/Container/x86_64/images/Fedora-Container-39-1.5-x86_64-CHECKSUM \ -o checksum # BusyBox sha256sum treats all lines in the checksum file @@ -464,16 +392,12 @@ System Configuration .. code-block:: sh - dnf -y install @core grub2-efi-x64 \ - grub2-pc grub2-pc-modules grub2-efi-x64-modules shim-x64 \ - efibootmgr kernel kernel-devel + dnf -y install @core kernel kernel-devel .. ifconfig:: zfs_root_test # no firmware for test - dnf -y install --setopt=install_weak_deps=False @core grub2-efi-x64 \ - grub2-pc grub2-pc-modules grub2-efi-x64-modules shim-x64 \ - efibootmgr + dnf -y install --setopt=install_weak_deps=False @core # kernel-core #. Install ZFS packages @@ -481,7 +405,7 @@ System Configuration .. code-block:: sh dnf -y install \ - https://zfsonlinux.org/fedora/zfs-release-2-3"$(rpm --eval "%{dist}"||true)".noarch.rpm + https://zfsonlinux.org/fedora/zfs-release-2-4"$(rpm --eval "%{dist}"||true)".noarch.rpm dnf -y install zfs zfs-dracut @@ -491,7 +415,7 @@ System Configuration # no need to test building in chroot dnf -y install \ - https://zfsonlinux.org/fedora/zfs-release-2-3"$(rpm --eval "%{dist}"||true)".noarch.rpm + https://zfsonlinux.org/fedora/zfs-release-2-4"$(rpm --eval "%{dist}"||true)".noarch.rpm #. Check whether ZFS modules are successfully built @@ -601,61 +525,24 @@ System Configuration Bootloader --------------------------- -#. Apply GRUB workaround - - :: - - echo 'export ZPOOL_VDEV_NAME_PATH=YES' >> /etc/profile.d/zpool_vdev_name_path.sh - # shellcheck disable=SC1091 - . /etc/profile.d/zpool_vdev_name_path.sh - - # GRUB fails to detect rpool name, hard code as "rpool" - sed -i "s|rpool=.*|rpool=rpool|" /etc/grub.d/10_linux - - This workaround needs to be applied for every GRUB update, as the - update will overwrite the changes. - -#. Fedora and RHEL uses Boot Loader Specification module for GRUB, - which does not support ZFS. Disable it:: +#. Install rEFInd boot loader:: - echo 'GRUB_ENABLE_BLSCFG=false' >> /etc/default/grub + # from http://www.rodsbooks.com/refind/getting.html + # use Binary Zip File option + curl -L http://sourceforge.net/projects/refind/files/0.14.0.2/refind-bin-0.14.0.2.zip/download --output refind.zip - This means that you need to regenerate GRUB menu and mirror them - after every kernel update, otherwise computer will still boot old - kernel on reboot. + dnf install -y unzip + unzip refind.zip + mkdir -p /boot/EFI/BOOT + find ./refind-bin-0.14.0.2/ -name 'refind_x64.efi' -print0 \ + | xargs -0I{} mv {} /boot/EFI/BOOT/BOOTX64.EFI + rm -rf refind.zip refind-bin-0.14.0.2 -#. Install GRUB:: +#. Add boot entry:: - mkdir -p /boot/efi/fedora/grub-bootdir/i386-pc/ - for i in ${DISK}; do - grub2-install --target=i386-pc --boot-directory \ - /boot/efi/fedora/grub-bootdir/i386-pc/ "${i}" - done - dnf reinstall -y grub2-efi-x64 shim-x64 - cp -r /usr/lib/grub/x86_64-efi/ /boot/efi/EFI/fedora/ - -#. Generate GRUB menu - - .. code-block:: sh - - mkdir -p /boot/grub2 - grub2-mkconfig -o /boot/grub2/grub.cfg - cp /boot/grub2/grub.cfg \ - /boot/efi/efi/fedora/grub.cfg - cp /boot/grub2/grub.cfg \ - /boot/efi/fedora/grub-bootdir/i386-pc/grub2/grub.cfg - - .. ifconfig:: zfs_root_test - - # GRUB menu can not be generated in test due to missing zfs programs - -#. For both legacy and EFI booting: mirror ESP content:: - - espdir=$(mktemp -d) - find /boot/efi/ -maxdepth 1 -mindepth 1 -type d -print0 \ - | xargs -t -0I '{}' cp -r '{}' "${espdir}" - find "${espdir}" -maxdepth 1 -mindepth 1 -type d -print0 \ - | xargs -t -0I '{}' sh -vxc "find /boot/efis/ -maxdepth 1 -mindepth 1 -type d -print0 | xargs -t -0I '[]' cp -r '{}' '[]'" + tee -a /boot/refind-linux.conf < list_of_efi_dirs - for i in ${DISK}; do - if ! grep "${i##*/}-part1/efi\|${i##*/}-part1/EFI" list_of_efi_dirs; then - echo "disk ${i} not found in efi system partition, installation error"; - cat list_of_efi_dirs - exit 1 - fi - done #. Unmount filesystems and create initial system snapshot You can later create a boot environment from this snapshot. @@ -690,7 +563,6 @@ Bootloader umount -Rl "${MNT}" zfs snapshot -r rpool@initial-installation - zfs snapshot -r bpool@initial-installation #. Export all pools @@ -709,30 +581,12 @@ Bootloader reboot -#. For BIOS-legacy boot users only: the GRUB bootloader installed - might be unusable. In this case, see Bootloader Recovery section - in `Root on ZFS maintenance page <../zfs_root_maintenance.html>`__. - - This issue is not related to Alpine Linux chroot, as Arch Linux - installed with this method does not have this issue. - - UEFI bootloader is not affected by this issue. .. ifconfig:: zfs_root_test # chroot ends here ZFS_ROOT_GUIDE_TEST -#. On first reboot, SELinux policies will be applied, albeit - incompletely. The computer will then reboot with incomplete - policies and fail to mount ``/run``, resulting in a failure. - - Workaround is to append ``enforcing=0`` to kernel command line in - the GRUB menu, as many times as necessary, until the system - complete one successful boot. The author of this guide has not - found out a way to solve this issue during installation. Help is - appreciated. - Post installaion --------------------------- @@ -745,5 +599,5 @@ Post installaion #. Add new user, configure swap. -.. _a ZFS developer's comment on this issue: https://ol.reddit.com/r/zfs/comments/10n8fsn/does_openzfs_have_a_new_developer_for_the_native/j6b8k1m/ -.. _this spreadsheet of bugs: https://docs.google.com/spreadsheets/d/1OfRSXibZ2nIE9DGK6swwBZXgXwdCPKgp4SbPZwTexCg/htmlview +#. Mount other EFI system partitions then set up a service for syncing + their contents. diff --git a/docs/Getting Started/NixOS/Root on ZFS.rst b/docs/Getting Started/NixOS/Root on ZFS.rst index fbdf8efb6..48c4a0ffd 100644 --- a/docs/Getting Started/NixOS/Root on ZFS.rst +++ b/docs/Getting Started/NixOS/Root on ZFS.rst @@ -14,29 +14,20 @@ NixOS Root on ZFS ======================================= -**Note for arm64**: - -Currently there is a bug with the grub installation script. See `here -`__ for details. - -**Note for Immutable Root**: - -Immutable root can be enabled or disabled by setting -``zfs-root.boot.immutable`` option inside per-host configuration. **Customization** Unless stated otherwise, it is not recommended to customize system configuration before reboot. -**Only use well-tested pool features** +**UEFI support only** -You should only use well-tested pool features. Avoid using new features if data integrity is paramount. See, for example, `this comment `__. +Only UEFI is supported by this guide. Make sure your computer is +booted in UEFI mode. Preparation --------------------------- -#. Disable Secure Boot. ZFS modules can not be loaded if Secure Boot is enabled. #. Download `NixOS Live Image `__ and boot from it. @@ -89,6 +80,9 @@ Preparation :: + # install installation tools + nix-env -f '' -iA nixos-install-tools + # for github test run, use chroot and loop devices DISK="$(losetup --all| grep nixos | cut -f1 -d: | xargs -t -I '{}' printf '{} ')" @@ -124,27 +118,6 @@ Preparation RESERVE=1 -#. Enable Nix Flakes functionality - :: - - mkdir -p ~/.config/nix - echo "experimental-features = nix-command flakes" >> ~/.config/nix/nix.conf - -#. Install programs needed for system installation - :: - - if ! command -v git; then nix-env -f '' -iA git; fi - if ! command -v partprobe; then nix-env -f '' -iA parted; fi - - .. ifconfig:: zfs_root_test - - :: - - # install missing packages in chroot - if (echo "${DISK}" | grep "/dev/loop"); then - nix-env -f '' -iA nixos-install-tools - fi - System Installation --------------------------- @@ -161,17 +134,12 @@ System Installation parted --script --align=optimal "${disk}" -- \ mklabel gpt \ - mkpart EFI 2MiB 1GiB \ - mkpart bpool 1GiB 5GiB \ - mkpart rpool 5GiB -$((SWAPSIZE + RESERVE))GiB \ + mkpart EFI 1MiB 4GiB \ + mkpart rpool 4GiB -$((SWAPSIZE + RESERVE))GiB \ mkpart swap -$((SWAPSIZE + RESERVE))GiB -"${RESERVE}"GiB \ - mkpart BIOS 1MiB 2MiB \ set 1 esp on \ - set 5 bios_grub on \ - set 5 legacy_boot on partprobe "${disk}" - udevadm settle } for i in ${DISK}; do @@ -196,58 +164,24 @@ System Installation done done -#. Setup encrypted swap. This is useful if the available memory is - small:: +#. Setup temporary encrypted swap for this installation only. This is + useful if the available memory is small:: for i in ${DISK}; do - cryptsetup open --type plain --key-file /dev/random "${i}"-part4 "${i##*/}"-part4 - mkswap /dev/mapper/"${i##*/}"-part4 - swapon /dev/mapper/"${i##*/}"-part4 + cryptsetup open --type plain --key-file /dev/random "${i}"-part3 "${i##*/}"-part3 + mkswap /dev/mapper/"${i##*/}"-part3 + swapon /dev/mapper/"${i##*/}"-part3 done + #. **LUKS only**: Setup encrypted LUKS container for root pool:: for i in ${DISK}; do # see PASSPHRASE PROCESSING section in cryptsetup(8) - printf "YOUR_PASSWD" | cryptsetup luksFormat --type luks2 "${i}"-part3 - - printf "YOUR_PASSWD" | cryptsetup luksOpen "${i}"-part3 luks-rpool-"${i##*/}"-part3 - + printf "YOUR_PASSWD" | cryptsetup luksFormat --type luks2 "${i}"-part2 - + printf "YOUR_PASSWD" | cryptsetup luksOpen "${i}"-part2 luks-rpool-"${i##*/}"-part2 - done -#. Create boot pool - :: - - # shellcheck disable=SC2046 - zpool create -o compatibility=legacy \ - -o ashift=12 \ - -o autotrim=on \ - -O acltype=posixacl \ - -O canmount=off \ - -O devices=off \ - -O normalization=formD \ - -O relatime=on \ - -O xattr=sa \ - -O mountpoint=/boot \ - -R "${MNT}" \ - bpool \ - mirror \ - $(for i in ${DISK}; do - printf '%s ' "${i}-part2"; - done) - - If not using a multi-disk setup, remove ``mirror``. - - You should not need to customize any of the options for the boot pool. - - GRUB does not support all of the zpool features. See ``spa_feature_names`` - in `grub-core/fs/zfs/zfs.c - `__. - This step creates a separate boot pool for ``/boot`` with the features - limited to only those that GRUB supports, allowing the root pool to use - any/all features. - - Features enabled with ``-o compatibility=grub2`` can be seen - `here `__. - #. Create root pool - Unencrypted @@ -261,16 +195,15 @@ System Installation -R "${MNT}" \ -O acltype=posixacl \ -O canmount=off \ - -O compression=zstd \ -O dnodesize=auto \ -O normalization=formD \ -O relatime=on \ -O xattr=sa \ - -O mountpoint=/ \ + -O mountpoint=none \ rpool \ mirror \ $(for i in ${DISK}; do - printf '%s ' "${i}-part3"; + printf '%s ' "${i}-part2"; done) - LUKS encrypted @@ -284,236 +217,92 @@ System Installation -R "${MNT}" \ -O acltype=posixacl \ -O canmount=off \ - -O compression=zstd \ -O dnodesize=auto \ -O normalization=formD \ -O relatime=on \ -O xattr=sa \ - -O mountpoint=/ \ + -O mountpoint=none \ rpool \ mirror \ $(for i in ${DISK}; do - printf '/dev/mapper/luks-rpool-%s ' "${i##*/}-part3"; + printf '/dev/mapper/luks-rpool-%s ' "${i##*/}-part2"; done) If not using a multi-disk setup, remove ``mirror``. #. Create root system container: - - Unencrypted - :: - zfs create \ - -o canmount=off \ - -o mountpoint=none \ - rpool/nixos - - - Encrypted: - - Avoid ZFS send/recv when using native encryption, see `a ZFS developer's comment on - this issue`__ and `this spreadsheet of bugs`__. In short, if you - care about your data, don't use native encryption. This section - has been removed, use LUKS encryption instead. + zfs create -o canmount=noauto -o mountpoint=legacy rpool/root Create system datasets, manage mountpoints with ``mountpoint=legacy`` :: - zfs create -o mountpoint=legacy rpool/nixos/root - mount -t zfs rpool/nixos/root "${MNT}"/ - zfs create -o mountpoint=legacy rpool/nixos/home - mkdir "${MNT}"/home - mount -t zfs rpool/nixos/home "${MNT}"/home - zfs create -o mountpoint=none rpool/nixos/var - zfs create -o mountpoint=legacy rpool/nixos/var/lib - zfs create -o mountpoint=legacy rpool/nixos/var/log - zfs create -o mountpoint=none bpool/nixos - zfs create -o mountpoint=legacy bpool/nixos/root - mkdir "${MNT}"/boot - mount -t zfs bpool/nixos/root "${MNT}"/boot - mkdir -p "${MNT}"/var/log - mkdir -p "${MNT}"/var/lib - mount -t zfs rpool/nixos/var/lib "${MNT}"/var/lib - mount -t zfs rpool/nixos/var/log "${MNT}"/var/log - zfs create -o mountpoint=legacy rpool/nixos/empty - zfs snapshot rpool/nixos/empty@start - -#. Format and mount ESP - :: - - for i in ${DISK}; do - mkfs.vfat -n EFI "${i}"-part1 - mkdir -p "${MNT}"/boot/efis/"${i##*/}"-part1 - mount -t vfat -o iocharset=iso8859-1 "${i}"-part1 "${MNT}"/boot/efis/"${i##*/}"-part1 - done - - -System Configuration ---------------------------- - -#. Clone template flake configuration - - .. code-block:: sh - - mkdir -p "${MNT}"/etc - git clone --depth 1 --branch openzfs-guide \ - https://github.com/ne9z/dotfiles-flake.git "${MNT}"/etc/nixos - - .. ifconfig:: zfs_root_test - - :: - - # Use vm branch of the template config for test run - mkdir -p "${MNT}"/etc - git clone --depth 1 --branch openzfs-guide-testvm \ - https://github.com/ne9z/dotfiles-flake.git "${MNT}"/etc/nixos - # for debugging: show template revision - git -C "${MNT}"/etc/nixos log -n1 - -#. From now on, the complete configuration of the system will be - tracked by git, set a user name and email address to continue - :: - - rm -rf "${MNT}"/etc/nixos/.git - git -C "${MNT}"/etc/nixos/ init -b main - git -C "${MNT}"/etc/nixos/ add "${MNT}"/etc/nixos/ - git -C "${MNT}"/etc/nixos config user.email "you@example.com" - git -C "${MNT}"/etc/nixos config user.name "Alice Q. Nixer" - git -C "${MNT}"/etc/nixos commit -asm 'initial commit' - -#. Customize configuration to your hardware + zfs create -o mountpoint=legacy rpool/home + mount -o X-mount.mkdir -t zfs rpool/root "${MNT}" + mount -o X-mount.mkdir -t zfs rpool/home "${MNT}"/home +#. Format and mount ESP. Only one of them is used as /boot, you need to set up mirroring afterwards :: for i in ${DISK}; do - sed -i \ - "s|/dev/disk/by-id/|${i%/*}/|" \ - "${MNT}"/etc/nixos/hosts/exampleHost/default.nix - break + mkfs.vfat -n EFI "${i}"-part1 done - diskNames="" for i in ${DISK}; do - diskNames="${diskNames} \"${i##*/}\"" + mount -t vfat -o fmask=0077,dmask=0077,iocharset=iso8859-1,X-mount.mkdir "${i}"-part1 "${MNT}"/boot + break done - sed -i "s|\"bootDevices_placeholder\"|${diskNames}|g" \ - "${MNT}"/etc/nixos/hosts/exampleHost/default.nix - - sed -i "s|\"abcd1234\"|\"$(head -c4 /dev/urandom | od -A none -t x4| sed 's| ||g' || true)\"|g" \ - "${MNT}"/etc/nixos/hosts/exampleHost/default.nix - sed -i "s|\"x86_64-linux\"|\"$(uname -m || true)-linux\"|g" \ - "${MNT}"/etc/nixos/flake.nix +System Configuration +--------------------------- -#. **LUKS only**: Enable LUKS support:: +#. Generate system configuration:: - sed -i 's|luks.enable = false|luks.enable = true|' "${MNT}"/etc/nixos/hosts/exampleHost/default.nix + nixos-generate-config --root "${MNT}" -#. Detect kernel modules needed for boot +#. Edit system configuration: .. code-block:: sh - cp "$(command -v nixos-generate-config || true)" ./nixos-generate-config - - chmod a+rw ./nixos-generate-config - - # shellcheck disable=SC2016 - echo 'print STDOUT $initrdAvailableKernelModules' >> ./nixos-generate-config - - kernelModules="$(./nixos-generate-config --show-hardware-config --no-filesystems | tail -n1 || true)" - - sed -i "s|\"kernelModules_placeholder\"|${kernelModules}|g" \ - "${MNT}"/etc/nixos/hosts/exampleHost/default.nix - - .. ifconfig:: zfs_root_test - - :: - - sed -i "s|\"kernelModules_placeholder\"|\"nvme\"|g" \ - "${MNT}"/etc/nixos/hosts/exampleHost/default.nix - - # show generated config - cat "${MNT}"/etc/nixos/hosts/exampleHost/default.nix + nano "${MNT}"/etc/nixos/hardware-configuration.nix -#. Set root password +#. Set networking.hostId: .. code-block:: sh - rootPwd=$(mkpasswd -m SHA-512) + networking.hostId = "abcd1234"; - .. ifconfig:: zfs_root_test - - :: - - # Use "test" for root password in test run - rootPwd=$(echo yourpassword | mkpasswd -m SHA-512 -) - - Declare password in configuration - :: +#. If using LUKS, add the output from following command to system + configuration - sed -i \ - "s|rootHash_placeholder|${rootPwd}|" \ - "${MNT}"/etc/nixos/configuration.nix - -#. You can enable NetworkManager for wireless networks and GNOME - desktop environment in ``configuration.nix``. - -#. Commit changes to local repo - :: + .. code-block:: sh - git -C "${MNT}"/etc/nixos commit -asm 'initial installation' + tee <`__. +#. Set up networking, desktop and swap. + +#. Mount other EFI system partitions then set up a service for syncing + their contents. diff --git a/docs/Getting Started/RHEL-based distro/Root on ZFS.rst b/docs/Getting Started/RHEL-based distro/Root on ZFS.rst index a26064d55..e3f7605eb 100644 --- a/docs/Getting Started/RHEL-based distro/Root on ZFS.rst +++ b/docs/Getting Started/RHEL-based distro/Root on ZFS.rst @@ -27,11 +27,6 @@ Rocky Linux Root on ZFS **ZFSBootMenu** -This tutorial is based on the GRUB bootloader. Due to its independent -implementation of a read-only ZFS driver, GRUB only supports a subset -of ZFS features on the boot pool. [In general, bootloader treat disks -as read-only to minimize the risk of damaging on-disk data.] - `ZFSBootMenu `__ is an alternative bootloader free of such limitations and has support for boot environments. Do not follow instructions on this page if you plan to use ZBM, @@ -47,6 +42,10 @@ configuration before reboot. You should only use well-tested pool features. Avoid using new features if data integrity is paramount. See, for example, `this comment `__. +**UEFI support only** + +Only UEFI is supported by this guide. + Preparation --------------------------- @@ -57,8 +56,8 @@ Preparation Download latest extended variant of `Alpine Linux live image - `__, - verify `checksum `__ + `__, + verify `checksum `__ and boot from it. .. code-block:: sh @@ -223,14 +222,10 @@ System Installation parted --script --align=optimal "${disk}" -- \ mklabel gpt \ - mkpart EFI 2MiB 1GiB \ - mkpart bpool 1GiB 5GiB \ - mkpart rpool 5GiB -$((SWAPSIZE + RESERVE))GiB \ + mkpart EFI 1MiB 4GiB \ + mkpart rpool 4GiB -$((SWAPSIZE + RESERVE))GiB \ mkpart swap -$((SWAPSIZE + RESERVE))GiB -"${RESERVE}"GiB \ - mkpart BIOS 1MiB 2MiB \ set 1 esp on \ - set 5 bios_grub on \ - set 5 legacy_boot on partprobe "${disk}" } @@ -241,7 +236,7 @@ System Installation .. ifconfig:: zfs_root_test - :: + :: # When working with GitHub chroot runners, we are using loop # devices as installation target. However, the alias support for @@ -257,55 +252,26 @@ System Installation done done -#. Setup encrypted swap. This is useful if the available memory is - small:: + +#. Setup temporary encrypted swap for this installation only. This is + useful if the available memory is small:: for i in ${DISK}; do - cryptsetup open --type plain --key-file /dev/random "${i}"-part4 "${i##*/}"-part4 - mkswap /dev/mapper/"${i##*/}"-part4 - swapon /dev/mapper/"${i##*/}"-part4 + cryptsetup open --type plain --key-file /dev/random "${i}"-part3 "${i##*/}"-part3 + mkswap /dev/mapper/"${i##*/}"-part3 + swapon /dev/mapper/"${i##*/}"-part3 done + #. Load ZFS kernel module .. code-block:: sh modprobe zfs -#. Create boot pool - :: - - # shellcheck disable=SC2046 - zpool create -o compatibility=legacy \ - -o ashift=12 \ - -o autotrim=on \ - -O acltype=posixacl \ - -O canmount=off \ - -O devices=off \ - -O normalization=formD \ - -O relatime=on \ - -O xattr=sa \ - -O mountpoint=/boot \ - -R "${MNT}" \ - bpool \ - mirror \ - $(for i in ${DISK}; do - printf '%s ' "${i}-part2"; - done) - - If not using a multi-disk setup, remove ``mirror``. - - You should not need to customize any of the options for the boot pool. - - GRUB does not support all of the zpool features. See ``spa_feature_names`` - in `grub-core/fs/zfs/zfs.c - `__. - This step creates a separate boot pool for ``/boot`` with the features - limited to only those that GRUB supports, allowing the root pool to use - any/all features. - #. Create root pool - :: + + - Unencrypted:: # shellcheck disable=SC2046 zpool create \ @@ -314,80 +280,43 @@ System Installation -R "${MNT}" \ -O acltype=posixacl \ -O canmount=off \ - -O compression=zstd \ -O dnodesize=auto \ -O normalization=formD \ -O relatime=on \ -O xattr=sa \ - -O mountpoint=/ \ + -O mountpoint=none \ rpool \ mirror \ $(for i in ${DISK}; do - printf '%s ' "${i}-part3"; + printf '%s ' "${i}-part2"; done) - If not using a multi-disk setup, remove ``mirror``. - #. Create root system container: - - Unencrypted - :: - zfs create \ - -o canmount=off \ - -o mountpoint=none \ - rpool/rhel - - - Encrypted: - - Avoid ZFS send/recv when using native encryption, see `a ZFS developer's comment on this issue`__ and `this spreadsheet of bugs`__. A LUKS-based guide has yet to be written. Once compromised, changing password will not keep your - data safe. See ``zfs-change-key(8)`` for more info - - .. code-block:: sh - - zfs create \ - -o canmount=off \ - -o mountpoint=none \ - -o encryption=on \ - -o keylocation=prompt \ - -o keyformat=passphrase \ - rpool/rhel - - You can automate this step (insecure) with: ``echo POOLPASS | zfs create ...``. + # dracut demands system root dataset to have non-legacy mountpoint + zfs create -o canmount=noauto -o mountpoint=/ rpool/root Create system datasets, manage mountpoints with ``mountpoint=legacy`` :: - zfs create -o canmount=noauto -o mountpoint=/ rpool/rhel/root - zfs mount rpool/rhel/root - zfs create -o mountpoint=legacy rpool/rhel/home - mkdir "${MNT}"/home - mount -t zfs rpool/rhel/home "${MNT}"/home - zfs create -o mountpoint=legacy rpool/rhel/var - zfs create -o mountpoint=legacy rpool/rhel/var/lib - zfs create -o mountpoint=legacy rpool/rhel/var/log - zfs create -o mountpoint=none bpool/rhel - zfs create -o mountpoint=legacy bpool/rhel/root - mkdir "${MNT}"/boot - mount -t zfs bpool/rhel/root "${MNT}"/boot - mkdir -p "${MNT}"/var/log - mkdir -p "${MNT}"/var/lib - mount -t zfs rpool/rhel/var/lib "${MNT}"/var/lib - mount -t zfs rpool/rhel/var/log "${MNT}"/var/log - -#. Format and mount ESP + zfs create -o mountpoint=legacy rpool/home + zfs mount rpool/root + mount -o X-mount.mkdir -t zfs rpool/home "${MNT}"/home + +#. Format and mount ESP. Only one of them is used as /boot, you need to set up mirroring afterwards :: for i in ${DISK}; do mkfs.vfat -n EFI "${i}"-part1 - mkdir -p "${MNT}"/boot/efis/"${i##*/}"-part1 - mount -t vfat -o iocharset=iso8859-1 "${i}"-part1 "${MNT}"/boot/efis/"${i##*/}"-part1 done - mkdir -p "${MNT}"/boot/efi - mount -t vfat -o iocharset=iso8859-1 "$(echo "${DISK}" | sed "s|^ *||" | cut -f1 -d' '|| true)"-part1 "${MNT}"/boot/efi + for i in ${DISK}; do + mount -t vfat -o fmask=0077,dmask=0077,iocharset=iso8859-1,X-mount.mkdir "${i}"-part1 "${MNT}"/boot + break + done System Configuration --------------------------- @@ -396,10 +325,10 @@ System Configuration apk add curl curl --fail-early --fail -L \ - https://dl.rockylinux.org/pub/rocky/9.2/images/x86_64/Rocky-9-Container-Base-9.2-20230513.0.x86_64.tar.xz \ + https://dl.rockylinux.org/vault/rocky/9.3/images/x86_64/Rocky-9-Container-Base-9.3-20231119.0.x86_64.tar.xz \ -o rootfs.tar.gz curl --fail-early --fail -L \ - https://dl.rockylinux.org/pub/rocky/9.2/images/x86_64/Rocky-9-Container-Base-9.2-20230513.0.x86_64.tar.xz.CHECKSUM \ + https://dl.rockylinux.org/vault/rocky/9.3/images/x86_64/Rocky-9-Container-Base-9.3-20231119.0.x86_64.tar.xz.CHECKSUM \ -o checksum # BusyBox sha256sum treats all lines in the checksum file @@ -453,17 +382,13 @@ System Configuration .. code-block:: sh - dnf -y install --allowerasing @core grub2-efi-x64 \ - grub2-pc grub2-pc-modules grub2-efi-x64-modules shim-x64 \ - efibootmgr kernel-core + dnf -y install --allowerasing @core kernel-core .. ifconfig:: zfs_root_test # skip installing firmware in test dnf -y install --allowerasing --setopt=install_weak_deps=False \ - @core grub2-efi-x64 \ - grub2-pc grub2-pc-modules grub2-efi-x64-modules shim-x64 \ - efibootmgr kernel-core + @core kernel-core #. Install ZFS packages:: @@ -528,63 +453,24 @@ System Configuration Bootloader --------------------------- +#. Install rEFInd boot loader:: -#. Apply GRUB workaround - - :: - - echo 'export ZPOOL_VDEV_NAME_PATH=YES' >> /etc/profile.d/zpool_vdev_name_path.sh - # shellcheck disable=SC1091 - . /etc/profile.d/zpool_vdev_name_path.sh - - # GRUB fails to detect rpool name, hard code as "rpool" - sed -i "s|rpool=.*|rpool=rpool|" /etc/grub.d/10_linux - - This workaround needs to be applied for every GRUB update, as the - update will overwrite the changes. + # from http://www.rodsbooks.com/refind/getting.html + # use Binary Zip File option + curl -L http://sourceforge.net/projects/refind/files/0.14.0.2/refind-bin-0.14.0.2.zip/download --output refind.zip -#. RHEL uses Boot Loader Specification module for GRUB, - which does not support ZFS. Disable it:: + dnf install -y unzip + unzip refind.zip + mkdir -p /boot/EFI/BOOT + find ./refind-bin-0.14.0.2/ -name 'refind_x64.efi' -print0 \ + | xargs -0I{} mv {} /boot/EFI/BOOT/BOOTX64.EFI + rm -rf refind.zip refind-bin-0.14.0.2 - echo 'GRUB_ENABLE_BLSCFG=false' >> /etc/default/grub +#. Add boot entry:: - This means that you need to regenerate GRUB menu and mirror them - after every kernel update, otherwise computer will still boot old - kernel on reboot. - -#. Install GRUB:: - - mkdir -p /boot/efi/rocky/grub-bootdir/i386-pc/ - for i in ${DISK}; do - grub2-install --target=i386-pc --boot-directory \ - /boot/efi/rocky/grub-bootdir/i386-pc/ "${i}" - done - dnf reinstall -y grub2-efi-x64 shim-x64 - cp -r /usr/lib/grub/x86_64-efi/ /boot/efi/EFI/rocky/ - -#. Generate GRUB menu:: - - mkdir -p /boot/grub2 - grub2-mkconfig -o /boot/grub2/grub.cfg - cp /boot/grub2/grub.cfg \ - /boot/efi/efi/rocky/grub.cfg - cp /boot/grub2/grub.cfg \ - /boot/efi/rocky/grub-bootdir/i386-pc/grub2/grub.cfg - - .. ifconfig:: zfs_root_test - - :: - - find /boot/efis/ -name "grub.cfg" -print0 \ - | xargs -t -0I '{}' grub2-script-check -v '{}' - -#. For both legacy and EFI booting: mirror ESP content:: - - espdir=$(mktemp -d) - find /boot/efi/ -maxdepth 1 -mindepth 1 -type d -print0 \ - | xargs -t -0I '{}' cp -r '{}' "${espdir}" - find "${espdir}" -maxdepth 1 -mindepth 1 -type d -print0 \ - | xargs -t -0I '{}' sh -vxc "find /boot/efis/ -maxdepth 1 -mindepth 1 -type d -print0 | xargs -t -0I '[]' cp -r '{}' '[]'" + tee -a /boot/refind-linux.conf < list_of_efi_dirs - for i in ${DISK}; do - if ! grep "${i##*/}-part1/efi\|${i##*/}-part1/EFI" list_of_efi_dirs; then - echo "disk ${i} not found in efi system partition, installation error"; - cat list_of_efi_dirs - exit 1 - fi - done - #. Unmount filesystems and create initial system snapshot You can later create a boot environment from this snapshot. See `Root on ZFS maintenance page <../zfs_root_maintenance.html>`__. @@ -619,7 +490,6 @@ Bootloader umount -Rl "${MNT}" zfs snapshot -r rpool@initial-installation - zfs snapshot -r bpool@initial-installation #. Export all pools @@ -638,15 +508,6 @@ Bootloader reboot -#. For BIOS-legacy boot users only: the GRUB bootloader installed - might be unusable. In this case, see Bootloader Recovery section - in `Root on ZFS maintenance page <../zfs_root_maintenance.html>`__. - - This issue is not related to Alpine Linux chroot, as Arch Linux - installed with this method does not have this issue. - - UEFI bootloader is not affected by this issue. - .. ifconfig:: zfs_root_test # chroot ends here @@ -664,5 +525,5 @@ Post installaion #. Add new user, configure swap. -.. _a ZFS developer's comment on this issue: https://ol.reddit.com/r/zfs/comments/10n8fsn/does_openzfs_have_a_new_developer_for_the_native/j6b8k1m/ -.. _this spreadsheet of bugs: https://docs.google.com/spreadsheets/d/1OfRSXibZ2nIE9DGK6swwBZXgXwdCPKgp4SbPZwTexCg/htmlview +#. Mount other EFI system partitions then set up a service for syncing + their contents. diff --git a/docs/Getting Started/index.rst b/docs/Getting Started/index.rst index 176750bd3..6103495f0 100644 --- a/docs/Getting Started/index.rst +++ b/docs/Getting Started/index.rst @@ -21,4 +21,3 @@ documentation `__ openSUSE/index RHEL-based distro/index Ubuntu/index - zfs_root_maintenance diff --git a/docs/Getting Started/zfs_root_maintenance.rst b/docs/Getting Started/zfs_root_maintenance.rst deleted file mode 100644 index 341aad692..000000000 --- a/docs/Getting Started/zfs_root_maintenance.rst +++ /dev/null @@ -1,311 +0,0 @@ -.. highlight:: sh - -Root on ZFS maintenance -======================== - -Boot Environment ----------------- - -This section is compatible with Alpine, Arch, Fedora and RHEL guides. -Not necessary for NixOS. Incompatible with Ubuntu and Debian guides. - -Note: boot environments as described below are intended only for -system recovery purposes, that is, you boot into the alternate boot -environment once to perform system recovery on the default datasets: - -.. code-block:: sh - - rpool/distro/root - bpool/distro/root - -then reboot to those datasets once you have successfully recovered the -system. - -Switching the default boot environment complicates bootloader recovery -and other maintenance operations and is thus currently not supported. - -#. If you want to use the ``@initial-installation`` snapshot created - during installation, set ``my_boot_env=initial-installation`` and - skip Step 3 and 4. - -#. Identify which dataset is currently mounted as root - ``/`` and boot ``/boot`` - :: - - set -x - boot_dataset=$(df -P /boot | tail -n1 | cut -f1 -d' ' || true ) - root_dataset=$(df -P / | tail -n1 | cut -f1 -d' ' || true ) - -#. Choose a name for the new boot environment - :: - - my_boot_env=backup - -#. Take snapshots of the ``/`` and ``/boot`` datasets - - :: - - zfs snapshot "${boot_dataset}"@"${my_boot_env}" - zfs snapshot "${root_dataset}"@"${my_boot_env}" - -#. Create clones from read-only snapshots - - :: - - new_root_dataset="${root_dataset%/*}"/"${my_boot_env}" - new_boot_dataset="${boot_dataset%/*}"/"${my_boot_env}" - - zfs clone -o canmount=noauto \ - -o mountpoint=/ \ - "${root_dataset}"@"${my_boot_env}" \ - "${new_root_dataset}" - - zfs clone -o canmount=noauto \ - -o mountpoint=legacy \ - "${boot_dataset}"@"${my_boot_env}" \ - "${new_boot_dataset}" - -#. Mount clone and update file system table (fstab) - :: - - MNT=$(mktemp -d) - mount -t zfs -o zfsutil "${new_root_dataset}" "${MNT}" - mount -t zfs "${new_boot_dataset}" "${MNT}"/boot - - sed -i s,"${root_dataset}","${new_root_dataset}",g "${MNT}"/etc/fstab - sed -i s,"${boot_dataset}","${new_boot_dataset}",g "${MNT}"/etc/fstab - - if test -f "${MNT}"/boot/grub/grub.cfg; then - is_grub2=n - sed -i s,"${boot_dataset#bpool/}","${new_boot_dataset#bpool/}",g "${MNT}"/boot/grub/grub.cfg - elif test -f "${MNT}"/boot/grub2/grub.cfg; then - is_grub2=y - sed -i s,"${boot_dataset#bpool/}","${new_boot_dataset#bpool/}",g "${MNT}"/boot/grub2/grub.cfg - else - echo "ERROR: no grub menu found!" - exit 1 - fi - - Do not proceed if no grub menu was found! - -#. Unmount clone - :: - - umount -Rl "${MNT}" - -#. Add new boot environment as GRUB menu entry - :: - - echo "# ${new_boot_dataset}" > new_boot_env_entry_"${new_boot_dataset##*/}" - printf '\n%s' "menuentry 'Boot environment ${new_boot_dataset#bpool/} from ${boot_dataset#bpool/}' " \ - >> new_boot_env_entry_"${new_boot_dataset##*/}" - if [ "${is_grub2}" = y ]; then - # shellcheck disable=SC2016 - printf '{ search --set=drive1 --label bpool; configfile ($drive1)/%s@/grub2/grub.cfg; }' \ - "${new_boot_dataset#bpool/}" >> new_boot_env_entry_"${new_boot_dataset##*/}" - else - # shellcheck disable=SC2016 - printf '{ search --set=drive1 --label bpool; configfile ($drive1)/%s@/grub/grub.cfg; }' \ - "${new_boot_dataset#bpool/}" >> new_boot_env_entry_"${new_boot_dataset##*/}" - fi - - find /boot/efis/ -name "grub.cfg" -print0 \ - | xargs -t -0I '{}' sh -vxc "tail -n1 new_boot_env_entry_${new_boot_dataset##*/} >> '{}'" - - .. ifconfig:: zfs_root_test - - :: - - find /boot/efis/ -name "grub.cfg" -print0 \ - | xargs -t -0I '{}' grub-script-check -v '{}' - -#. Do not delete ``new_boot_env_entry_"${new_boot_dataset##*/}"`` file. It - is needed when you want to remove the new boot environment from - GRUB menu later. - -#. After reboot, select boot environment entry from GRUB - menu to boot from the clone. Press ESC inside - submenu to return to the previous menu. - -#. Steps above can also be used to create a new clone - from an existing snapshot. - -#. To delete the boot environment, first store its name in a - variable:: - - my_boot_env=backup - -#. Ensure that the boot environment is not - currently used - :: - - set -x - boot_dataset=$(df -P /boot | tail -n1 | cut -f1 -d' ' || true ) - root_dataset=$(df -P / | tail -n1 | cut -f1 -d' ' || true ) - new_boot_dataset="${boot_dataset%/*}"/"${my_boot_env}" - rm_boot_dataset=$(head -n1 new_boot_env_entry_"${new_boot_dataset##*/}" | sed 's|^# *||' || true ) - - if [ "${boot_dataset}" = "${rm_boot_dataset}" ]; then - echo "ERROR: the dataset you want to delete is the current root! abort!" - exit 1 - fi - -#. Then check the origin snapshot - :: - - rm_root_dataset=rpool/"${rm_boot_dataset#bpool/}" - - rm_boot_dataset_origin=$(zfs get -H origin "${rm_boot_dataset}"|cut -f3 || true ) - rm_root_dataset_origin=$(zfs get -H origin "${rm_root_dataset}"|cut -f3 || true ) - -#. Finally, destroy clone (boot environment) and its - origin snapshot - :: - - zfs destroy "${rm_root_dataset}" - zfs destroy "${rm_root_dataset_origin}" - zfs destroy "${rm_boot_dataset}" - zfs destroy "${rm_boot_dataset_origin}" - -#. Remove GRUB entry - :: - - new_entry_escaped=$(tail -n1 new_boot_env_entry_"${new_boot_dataset##*/}" | sed -e 's/[\/&]/\\&/g' || true ) - find /boot/efis/ -name "grub.cfg" -print0 | xargs -t -0I '{}' sed -i "/${new_entry_escaped}/d" '{}' - - .. ifconfig:: zfs_root_test - - :: - - find /boot/efis/ -name "grub.cfg" -print0 \ - | xargs -t -0I '{}' grub-script-check -v '{}' - -Disk replacement ----------------- - -When a disk fails in a mirrored setup, the disk can be replaced with -the following procedure. - -#. Shutdown the computer. - -#. Replace the failed disk with another disk. The replacement should - be at least the same size or larger than the failed disk. - -#. Boot the computer. - - When a disk fails, the system will boot, albeit several minutes - slower than normal. - - For NixOS, this is due to the initrd and systemd designed to only - import a pool in degraded state after a 90s timeout. - - Swap partition on that disk will also fail. - -#. Install GNU ``parted`` with your distribution package manager. - -#. Identify the bad disk and a working old disk - - .. code-block:: sh - - ZPOOL_VDEV_NAME_PATH=1 zpool status - - pool: bpool - status: DEGRADED - action: Replace the device using 'zpool replace'. - ... - config: bpool - mirror-0 - 2387489723748 UNAVAIL 0 0 0 was /dev/disk/by-id/ata-BAD-part2 - /dev/disk/by-id/ata-disk_known_good-part2 ONLINE 0 0 0 - -#. Store the bad disk and a working old disk in a variable, omit the partition number ``-partN`` - - .. code-block:: sh - - disk_to_replace=/dev/disk/by-id/ata-disk_to_replace - disk_known_good=/dev/disk/by-id/ata-disk_known_good - -#. Identify the new disk - - .. code-block:: sh - - find /dev/disk/by-id/ - - /dev/disk/by-id/ata-disk_known_good-part1 - /dev/disk/by-id/ata-disk_known_good-part2 - ... - /dev/disk/by-id/ata-disk_known_good-part5 - /dev/disk/by-id/ata-disk_new <-- new disk w/o partition table - -#. Store the new disk in a variable - - .. code-block:: sh - - disk_new=/dev/disk/by-id/ata-disk_new - -#. Create partition table on ``"${disk_new}"``, refer to respective - installation pages for details. - -#. Format and mount EFI system partition, refer to respective - installation pages for details. - -#. Replace failed disk in ZFS pool - - .. code-block:: sh - - zpool offline bpool "${disk_to_replace}"-part2 - zpool offline rpool "${disk_to_replace}"-part3 - zpool replace bpool "${disk_to_replace}"-part2 "${disk_new}"-part2 - zpool replace rpool "${disk_to_replace}"-part3 "${disk_new}"-part3 - zpool online bpool "${disk_new}"-part2 - zpool online rpool "${disk_new}"-part3 - - Let the new disk resilver. Check status with ``zpool status``. - -#. Reinstall and mirror bootloader, refer to respective installation - pages for details. - - If you are using NixOS, see below. - -#. For NixOS, replace bad disk with new disk inside per-host - configuration file. - - .. code-block:: sh - - sed -i "s|"${disk_to_replace##*/}"|"${disk_new##*/}"|" /etc/nixos/hosts/exampleHost/default.nix - -#. Commit and apply the changed configuration, reinstall bootloader, then reboot - - .. code-block:: sh - - git -C /etc/nixos commit -asm "replace "${disk_to_replace##*/}" with "${disk_new##*/}"." - - nixos-rebuild boot --install-bootloader - - reboot - -Bootloader Recovery -------------------- - -This section is compatible with Alpine, Arch, Fedora, RHEL and NixOS -root on ZFS guides. - -Sometimes the GRUB bootloader might be accidentally overwritten, -rendering the system inaccessible. However, as long as the disk -partitions where boot pool and root pool resides remain untouched, the -system can still be booted easily. - -#. Download GRUB rescue image from `this repo - `__. - - You can also build the image yourself if you are familiar with Nix - package manager. - -#. Extract either x86_64-efi or i386-pc image from the archive. - -#. Write the image to a disk. - -#. Boot the computer from the GRUB rescue disk. Select your distro in - GRUB menu. - -#. Reinstall bootloader. See respective installation pages for details. diff --git a/scripts/zfs_root_guide_test.sh b/scripts/zfs_root_guide_test.sh index 08cd76131..2d06ccc25 100755 --- a/scripts/zfs_root_guide_test.sh +++ b/scripts/zfs_root_guide_test.sh @@ -22,8 +22,8 @@ losetup --detach-all # it is easier to install rhel with Alpine Linux live media # which has native zfs support if ! test -f rootfs.tar.gz; then - curl --fail-early --fail -Lo rootfs.tar.gz https://dl-cdn.alpinelinux.org/alpine/v3.18/releases/x86_64/alpine-minirootfs-3.18.4-x86_64.tar.gz - curl --fail-early --fail -Lo rootfs.tar.gz.sig https://dl-cdn.alpinelinux.org/alpine/v3.18/releases/x86_64/alpine-minirootfs-3.18.4-x86_64.tar.gz.asc + curl --fail-early --fail -Lo rootfs.tar.gz https://dl-cdn.alpinelinux.org/alpine/v3.19/releases/x86_64/alpine-minirootfs-3.19.0-x86_64.tar.gz + curl --fail-early --fail -Lo rootfs.tar.gz.sig https://dl-cdn.alpinelinux.org/alpine/v3.19/releases/x86_64/alpine-minirootfs-3.19.0-x86_64.tar.gz.asc gpg --auto-key-retrieve --keyserver hkps://keyserver.ubuntu.com --verify rootfs.tar.gz.sig fi mkdir rootfs-"${distro}"