Automate rootdisk xfs uuid rewriting.

src/fc/qemu/hazmat/ceph.py

@@ -74,6 +74,7 @@ def ensure_root_volume(self):
             self.log.info("create-vm")
             cmd(self.CREATE_VM.format(**self.cfg), self.log)
         self.root.lock()
+        self.root.regen_xfs_uuid()
 
     def ensure_swap_volume(self):
         self.log.info("ensure-swap")

src/fc/qemu/hazmat/volume.py

@@ -13,7 +13,8 @@
 
 import rbd
 
-from ..util import cmd, remove_empty_dirs
+from ..timeout import TimeOut, TimeoutError
+from ..util import cmd, parse_export_format, remove_empty_dirs
 
 
 class Image(object):
@@ -26,6 +27,7 @@ def __init__(self, ceph, name):
         self.ceph = ceph
         self.ioctx = ceph.ioctx
         self.name = name
+        self.log = ceph.log.bind(image=self.name)
         self.rbd = rbd.RBD()
 
     def __str__(self):
@@ -45,6 +47,12 @@ def part1dev(self):
             return None
         return self.device + "-part1"
 
+    def wait_for_part1dev(self):
+        timeout = TimeOut(5, interval=0.1, raise_on_timeout=True, log=self.log)
+        while timeout.tick():
+            if os.path.exists(self.part1dev):
+                break
+
     def map(self):
         if self.device is not None:
             return
@@ -53,8 +61,10 @@ def map(self):
                 self.ceph.CEPH_CONF, self.ceph.CEPH_CLIENT, self.fullname
             )
         )
-        time.sleep(0.1)
-        self.device = "/dev/rbd/" + self.fullname
+        device = "/dev/rbd/" + self.fullname
+        while not os.path.exists(device):
+            time.sleep(0.1)
+        self.device = device
 
     def unmap(self):
         if self.device is None:
@@ -69,6 +79,11 @@ def unmap(self):
     @contextlib.contextmanager
     def mapped(self):
         """Maps the image to a block device and yields the device name."""
+        if self.device:
+            # Re-entrant version - do not map/unmap
+            yield self.device
+            return
+        # Non-reentrant version: actually do the work
         self.map()
         try:
             yield self.device
@@ -87,6 +102,7 @@ def mount(self):
             os.makedirs(mountpoint)
         except OSError:  # pragma: no cover
             pass
+        self.wait_for_part1dev()
         self.cmd('mount "{}" "{}"'.format(self.part1dev, mountpoint))
         self.mountpoint = mountpoint
 
@@ -351,16 +367,44 @@ def mkfs(self, fstype="xfs", gptbios=False):
                 )
             )
         self.cmd("partprobe {}".format(self.device))
-        time.sleep(0.2)
-        while not p.exists(self.part1dev):  # pragma: no cover
-            time.sleep(0.1)
+        self.wait_for_part1dev()
         options = getattr(self.ceph, "MKFS_" + fstype.upper())
         self.cmd(
             self.MKFS_CMD[fstype].format(
                 options=options, device=self.part1dev, label=self.label
             )
         )
 
+    def regen_xfs_uuid(self):
+        """Regenerate the UUID of the XFS filesystem on partition 1."""
+        with self.mapped():
+            try:
+                self.wait_for_part1dev()
+            except TimeoutError:
+                self.log.warn(
+                    "regenerate-xfs-uuid",
+                    status="skipped",
+                    reason="no partition found",
+                )
+                return
+            output = self.cmd(f"blkid {self.part1dev} -o export")
+            values = parse_export_format(output)
+            fs_type = values.get("TYPE")
+            if fs_type != "xfs":
+                self.log.info(
+                    "regenerate-xfs-uuid",
+                    device=self.part1dev,
+                    status="skipped",
+                    fs_type=fs_type,
+                    reason="filesystem type != xfs",
+                )
+                return
+            with self.mounted():
+                # Mount once to ensure a clean log.
+                pass
+            self.log.info("regenerate-xfs-uuid", device=self.part1dev)
+            self.cmd(f"xfs_admin -U generate {self.part1dev}")
+
     def seed(self, enc, generation):
         self.log.info("seed")
         with self.mounted() as target:

src/fc/qemu/tests/test_util.py

@@ -0,0 +1,15 @@
+from fc.qemu.util import parse_export_format
+
+
+def test_export_format():
+    data = """DEVNAME="test"
+    UUID='a5370804-c9b1-4610-8f99-02a7841b8393'
+    BLOCK_SIZE=512
+    TYPE=xfs
+    """
+    assert parse_export_format(data) == {
+        "BLOCK_SIZE": "512",
+        "TYPE": "xfs",
+        "UUID": "a5370804-c9b1-4610-8f99-02a7841b8393",
+        "DEVNAME": "test",
+    }

src/fc/qemu/tests/vm_test.py

@@ -82,6 +82,30 @@ def test_simple_vm_lifecycle_start_stop(vm):
 fc-create-vm> --------
 /nix/store/.../bin/fc-create-vm machine=simplevm returncode=0 subsystem=ceph
 lock machine=simplevm subsystem=ceph volume=rbd.ssd/simplevm.root
+rbd args=-c "/etc/ceph/ceph.conf" --id "host1" map "rbd.ssd/simplevm.root" machine=simplevm subsystem=ceph volume=rbd.ssd/simplevm.root
+rbd> /dev/rbd0
+rbd machine=simplevm returncode=0 subsystem=ceph volume=rbd.ssd/simplevm.root
+waiting interval=0 machine=simplevm remaining=4 subsystem=ceph volume=rbd.ssd/simplevm.root
+blkid args=/dev/rbd/rbd.ssd/simplevm.root-part1 -o export machine=simplevm subsystem=ceph volume=rbd.ssd/simplevm.root
+blkid> DEVNAME=/dev/rbd/rbd.ssd/simplevm.root-part1
+blkid> UUID=...
+blkid> BLOCK_SIZE=512
+blkid> TYPE=xfs
+blkid> PARTLABEL=ROOT
+blkid> PARTUUID=...
+blkid machine=simplevm returncode=0 subsystem=ceph volume=rbd.ssd/simplevm.root
+mount args="/dev/rbd/rbd.ssd/simplevm.root-part1" "/mnt/rbd/rbd.ssd/simplevm.root" machine=simplevm subsystem=ceph volume=rbd.ssd/simplevm.root
+mount machine=simplevm returncode=0 subsystem=ceph volume=rbd.ssd/simplevm.root
+umount args="/mnt/rbd/rbd.ssd/simplevm.root" machine=simplevm subsystem=ceph volume=rbd.ssd/simplevm.root
+umount machine=simplevm returncode=0 subsystem=ceph volume=rbd.ssd/simplevm.root
+regenerate-xfs-uuid device=/dev/rbd/rbd.ssd/simplevm.root-part1 machine=simplevm subsystem=ceph volume=rbd.ssd/simplevm.root
+xfs_admin args=-U generate /dev/rbd/rbd.ssd/simplevm.root-part1 machine=simplevm subsystem=ceph volume=rbd.ssd/simplevm.root
+xfs_admin> Clearing log and setting UUID
+xfs_admin> writing all SBs
+xfs_admin> new UUID = ...
+xfs_admin machine=simplevm returncode=0 subsystem=ceph volume=rbd.ssd/simplevm.root
+rbd args=-c "/etc/ceph/ceph.conf" --id "host1" unmap "/dev/rbd/rbd.ssd/simplevm.root" machine=simplevm subsystem=ceph volume=rbd.ssd/simplevm.root
+rbd machine=simplevm returncode=0 subsystem=ceph volume=rbd.ssd/simplevm.root
 ensure-tmp machine=simplevm subsystem=ceph
 lock machine=simplevm subsystem=ceph volume=rbd.ssd/simplevm.tmp
 rbd args=-c "/etc/ceph/ceph.conf" --id "host1" map "rbd.ssd/simplevm.tmp" machine=simplevm subsystem=ceph volume=rbd.ssd/simplevm.tmp
@@ -99,6 +123,7 @@ def test_simple_vm_lifecycle_start_stop(vm):
 sgdisk machine=simplevm returncode=0 subsystem=ceph volume=rbd.ssd/simplevm.tmp
 partprobe args=/dev/rbd/rbd.ssd/simplevm.tmp machine=simplevm subsystem=ceph volume=rbd.ssd/simplevm.tmp
 partprobe machine=simplevm returncode=0 subsystem=ceph volume=rbd.ssd/simplevm.tmp
+waiting interval=0 machine=simplevm remaining=4 subsystem=ceph volume=rbd.ssd/simplevm.tmp
 mkfs.xfs args=-q -f -K -m crc=1,finobt=1 -d su=4m,sw=1 -L "tmp" "/dev/rbd/rbd.ssd/simplevm.tmp-part1" machine=simplevm subsystem=ceph volume=rbd.ssd/simplevm.tmp
 mkfs.xfs> mkfs.xfs: Specified data stripe unit 8192 is not the same as the volume stripe unit 128
 mkfs.xfs> mkfs.xfs: Specified data stripe width 8192 is not the same as the volume stripe width 128

src/fc/qemu/util.py

@@ -8,6 +8,7 @@
 import subprocess
 import sys
 import time
+from typing import Dict
 
 from structlog import get_logger
 
@@ -145,3 +146,26 @@ def ensure_separate_cgroup():
             # Keep going.
     with open("{}/cgroup.procs".format(CGROUP), "w") as f:
         f.write(str(os.getpid()))
+
+
+def parse_export_format(data: str) -> Dict[str, str]:
+    """Parses formats intended for shell exports into a dict.
+
+    ASDF=foo
+    BSDF=bar
+
+    Introduced to support output from `blkid`.
+
+    """
+    result = {}
+    for line in data.splitlines():
+        try:
+            k, v = line.strip().split("=")
+        except ValueError:
+            continue
+        k = k.strip()
+        if not k:
+            continue
+        v = v.strip("'\"")
+        result[k] = v
+    return result