From 5a81fdf106ce9c4da08d688f3b5baf3c9cf11505 Mon Sep 17 00:00:00 2001 From: jbtrystram Date: Thu, 23 Nov 2023 12:07:09 +0100 Subject: [PATCH] Add a testiso scenario installing coreOS to an iscsi target then boot it I reused the same approach as other liveISO tests, wrapping all the setup in a butane config. Systemd units do all the steps, taking care of the ordering. It is more verbose compared to imperatively executing commands to the VM through SSH but the avaiable tooling to do SSH commands in testiso is not as complete as in regular kola tests, so doing SSH would end up in a lot of error handling code. A future work is to merge the testiso scenarios with regular kola tests. Extra storage mounted into /var to avoir running out of space in the liveiso. When installing to the iscsi target with coreos-install we pass an ignition config to write to a serial device when the boot is complete. This serial device is passed through to the kola completion serial device, so it completes the test. --- .../kola/resources/iscsi_butane_setup.yaml | 142 ++++++++++++++++++ mantle/cmd/kola/testiso.go | 84 +++++++++++ 2 files changed, 226 insertions(+) create mode 100644 mantle/cmd/kola/resources/iscsi_butane_setup.yaml diff --git a/mantle/cmd/kola/resources/iscsi_butane_setup.yaml b/mantle/cmd/kola/resources/iscsi_butane_setup.yaml new file mode 100644 index 0000000000..62abc0f56c --- /dev/null +++ b/mantle/cmd/kola/resources/iscsi_butane_setup.yaml @@ -0,0 +1,142 @@ +variant: fcos +version: 1.5.0 +storage: + filesystems: + - path: /var + device: /dev/disk/by-id/virtio-var + format: ext4 + wipe_filesystem: true + label: var + with_mount_unit: true + files: + - path: /etc/containers/systemd/target.container + contents: + inline: | + [Unit] + Description=Targetd container + Documentation=https://github.com/jbtrystram/targetcli-containers + After=local-fs.target network-online.target After=nss-lookup.target dev-disk-by\x2did-virtio\x2dtarget.device + Wants=network-online.target + OnFailure=emergency.target + [Container] + Image=quay.io/jbtrystram/targetcli:latest + ContainerName=target + Network=host + Volume=/dev/disk/by-id/virtio-target:/dev/disk/by-id/virtio-target + Volume=/lib/modules:/lib/modules + Volume=/sys/kernel/config:/sys/kernel/config + PodmanArgs=--privileged + [Install] + # Start by default on boot + WantedBy=multi-user.target + - path: /usr/local/bin/targetcli_script + mode: 0755 + contents: + inline: | + #!/bin/bash + set -xeuo pipefail + podman exec target bash -exc " + targetcli /backstores/block create name=coreos dev=/dev/disk/by-id/virtio-target + targetcli iscsi/ create iqn.2023-10.coreos.target.vm:coreos + targetcli iscsi/iqn.2023-10.coreos.target.vm:coreos/tpg1/luns create /backstores/block/coreos + targetcli iscsi/iqn.2023-10.coreos.target.vm:coreos/tpg1/ set attribute authentication=0 demo_mode_write_protect=0 generate_node_acls=1 cache_dynamic_acls=1 + " + # Will return 0 if the discovery yield a valid portal + iscsiadm -m discovery -p 127.0.0.1 -t st | grep iqn.2023-10.coreos.target.vm:coreos + - path: /mnt/workdir-tmp/boot.ipxe + mode: 0644 + contents: + inline: | + #!ipxe + set initiator-iqn iqn.2023-11.coreos.diskless:testsetup + sanboot iscsi:10.0.2.15::::iqn.2023-10.coreos.target.vm:coreos + - path: /usr/local/bin/install-coreos-iscsi + mode: 0755 + contents: + inline: | + #!/bin/bash + set -euxo + # Mount the iscsi target + iscsiadm -m discovery -t st -p 127.0.0.1 + iscsiadm -m node -T iqn.2023-10.coreos.target.vm:coreos -l + # Give a bit of time to udev to create the persistent names paths + sleep 2 + # Install coreos + coreos-installer install \ + /dev/disk/by-path/ip-127.0.0.1\:3260-iscsi-iqn.2023-10.coreos.target.vm\:coreos-lun-0 \ + --append-karg rd.iscsi.firmware=1 --append-karg ip=ibft \ + --console ttyS0 \ + -i /mnt/workdir-tmp/nested-ign.json + # Unmount the disk + iscsiadm --mode node --logoutall=all + - path: /etc/containers/systemd/coreos-iscsi-vm.container + contents: + inline: | + [Unit] + Description=Boot VM over iSCSI + After=network-online.target After=nss-lookup.target install-coreos-to-iscsi-target.service + Wants=network-online.target install-coreos-to-iscsi-target.service + Requires=install-coreos-to-iscsi-target.service + OnFailure=emergency.target + [Container] + Image=quay.io/coreos-assembler/coreos-assembler + ContainerName=iscsiboot + Volume=/mnt/workdir-tmp/:/mnt/workdir-tmp/ + Volume=/dev/virtio-ports/testiscsicompletion:/mnt/serial + PodmanArgs=--privileged + Network=host + Exec=shell -- kola qemuexec --netboot /mnt/workdir-tmp/boot.ipxe --usernet-addr 10.0.3.0/24 -- -device virtio-serial -chardev file,id=iscsi-completion-virtio,path=/mnt/serial,append=on -device virtserialport,chardev=iscsi-completion-virtio,name=testiscsicompletion + [Install] + # Start by default on boot + WantedBy=multi-user.target + [Service] + # fix permissions on the serial device before passing it as a volume + ExecStartPre=chmod 777 /dev/virtio-ports/testiscsicompletion + - path: /mnt/workdir-tmp/nested-ign.json + contents: + inline: | + { + "ignition": { + "version": "3.1.0" + }, + "systemd": { + "units": [ + { + "contents": "[Unit]\nDescription=iSCSI Boot Signal Completion\nAfter=multi-user.target\nOnFailureJobMode=isolate\n[Service]\nType=oneshot\nRemainAfterExit=yes\nExecStart=/bin/sh -c '/usr/bin/echo \"iscsi-boot-ok\" \u003e/dev/virtio-ports/testiscsicompletion \u0026\u0026 systemctl poweroff'\n[Install]\nRequiredBy=multi-user.target\n", + "enabled": true, + "name": "successful-boot-signal.service" + } + ] + } + } +systemd: + units: + - name: setup-targetcli.service + enabled: true + contents: | + [Unit] + Description=Setup targetcli + Requires=target.service + After=target.service + ConditionFirstBoot=true + OnFailure=emergency.target + [Service] + Type=oneshot + RemainAfterExit=yes + ExecStart=/usr/local/bin/targetcli_script + [Install] + WantedBy=multi-user.target + - name: install-coreos-to-iscsi-target.service + enabled: true + contents: | + [Unit] + Description=Mount an iscsi target and install coreOS into it + Requires=setup-targetcli.service + After=setup-targetcli.service + OnFailure=emergency.target + [Service] + Type=oneshot + RemainAfterExit=yes + ExecStart=/usr/local/bin/install-coreos-iscsi + [Install] + WantedBy=multi-user.target diff --git a/mantle/cmd/kola/testiso.go b/mantle/cmd/kola/testiso.go index 30bed3c856..d4e04aa553 100644 --- a/mantle/cmd/kola/testiso.go +++ b/mantle/cmd/kola/testiso.go @@ -22,6 +22,7 @@ package main import ( "bufio" "context" + _ "embed" "fmt" "io" "os" @@ -88,6 +89,7 @@ var ( "iso-offline-install.bios", "iso-offline-install.mpath.bios", "iso-offline-install-fromram.4k.uefi", + "iso-offline-install-iscsi.bios", "miniso-install.bios", "miniso-install.nm.bios", "miniso-install.4k.uefi", @@ -109,6 +111,7 @@ var ( "miniso-install.s390fw", "miniso-install.nm.s390fw", "miniso-install.4k.nm.s390fw", + "iso-offline-install-iscsi.bios", } tests_ppc64le = []string{ "iso-live-login.ppcfw", @@ -121,6 +124,7 @@ var ( "miniso-install.4k.nm.ppcfw", "pxe-online-install.ppcfw", "pxe-offline-install.4k.ppcfw", + "iso-offline-install-iscsi.bios", } tests_aarch64 = []string{ "iso-live-login.uefi", @@ -136,6 +140,7 @@ var ( "pxe-offline-install.4k.uefi", "pxe-online-install.uefi", "pxe-online-install.4k.uefi", + "iso-offline-install-iscsi.bios", } ) @@ -319,6 +324,9 @@ RequiredBy=coreos-installer.target # for target system RequiredBy=multi-user.target`, nmConnectionId, nmConnectionFile) +//go:embed resources/iscsi_butane_setup.yaml +var iscsi_butane_config string + func init() { cmdTestIso.Flags().BoolVarP(&instInsecure, "inst-insecure", "S", false, "Do not verify signature on metal image") cmdTestIso.Flags().BoolVar(&console, "console", false, "Connect qemu console to terminal, turn off automatic initramfs failure checking") @@ -587,6 +595,8 @@ func runTestIso(cmd *cobra.Command, args []string) (err error) { duration, err = testLiveIso(ctx, inst, filepath.Join(outputDir, test), false) case "miniso-install": duration, err = testLiveIso(ctx, inst, filepath.Join(outputDir, test), true) + case "iso-offline-install-iscsi": + duration, err = testLiveInstalliscsi(ctx, inst, filepath.Join(outputDir, test)) default: plog.Fatalf("Unknown test name:%s", test) } @@ -955,3 +965,77 @@ func testAsDisk(ctx context.Context, outdir string) (time.Duration, error) { return awaitCompletion(ctx, mach, outdir, completionChannel, nil, []string{liveOKSignal}) } + +// iscsi_butane_setup.yaml contain the full butane config but here is an overview of the setup +// 1 - Boot a live ISO with two extra 10G disks with labels "target" and "var" +// - Format and mount `virtio-var` to var +// +// 2 - target.container -> start an iscsi target, using quay.io/jbtrystram/targetcli +// 3 - setup-targetcli.service calls /usr/local/bin/targetcli_script: +// - instructs targetcli to serve /dev/disk/by-id/virtio-target as an iscsi target +// - disables authentication +// - verifies the iscsi service is active and reachable +// +// 4 - install-coreos-to-iscsi-target.service calls /usr/local/bin/install-coreos-iscsi: +// - mount iscsi target +// - run coreos-installer on the mounted block device +// - unmount iscsi +// +// 5 - coreos-iscsi-vm.container start a coreos-assemble: +// - launch cosa qemuexec instructing it to boot from an iPXE script +// wich in turns mount the iscsi target and load kernel +// - note the virtserial port device: we pass through the serial port that was created by kola for test completion +// +// 6 - /mnt/workdir-tmp/nested-ign.json contains an ignition config: +// - when the system is booted, write a success string to /dev/virtio-ports/testiscsicompletion +// - As this serial device is mapped to the host serial device, the test concludes +func testLiveInstalliscsi(ctx context.Context, inst platform.Install, outdir string) (time.Duration, error) { + + builddir := kola.CosaBuild.Dir + isopath := filepath.Join(builddir, kola.CosaBuild.Meta.BuildArtifacts.LiveIso.Path) + builder, err := newBaseQemuBuilder(outdir) + if err != nil { + return 0, err + } + defer builder.Close() + if err := builder.AddIso(isopath, "", false); err != nil { + return 0, err + } + + completionChannel, err := builder.VirtioChannelRead("testiscsicompletion") + if err != nil { + return 0, err + } + + // empty disk to use as an iscsi target to install coreOS on and subseqently boot + // Also add a 10G disk that we will mount on /var, to increase space available when pulling containers + err = builder.AddDisksFromSpecs([]string{"10G:serial=target", "10G:serial=var"}) + if err != nil { + return 0, err + } + + // We need more memory to start another VM within ! + builder.MemoryMiB = 4096 + + var iscsiTargetConfig = conf.Butane(iscsi_butane_config) + + config, err := iscsiTargetConfig.Render(conf.FailWarnings) + if err != nil { + return 0, err + } + // Add a failure target to stop the test if something go wrong rather than waiting for the 10min timeout + config.AddSystemdUnit("coreos-test-entered-emergency-target.service", signalFailureUnit, conf.Enable) + + // enable network + builder.EnableUsermodeNetworking([]platform.HostForwardPort{}, "") + + builder.SetConfig(config) + + mach, err := builder.Exec() + if err != nil { + return 0, errors.Wrapf(err, "running iso") + } + defer mach.Destroy() + + return awaitCompletion(ctx, mach, outdir, completionChannel, nil, []string{"iscsi-boot-ok"}) +}