diff --git a/mantle/cmd/kola/resources/iscsi_butane_setup.yaml b/mantle/cmd/kola/resources/iscsi_butane_setup.yaml new file mode 100644 index 0000000000..62abc0f56c --- /dev/null +++ b/mantle/cmd/kola/resources/iscsi_butane_setup.yaml @@ -0,0 +1,142 @@ +variant: fcos +version: 1.5.0 +storage: + filesystems: + - path: /var + device: /dev/disk/by-id/virtio-var + format: ext4 + wipe_filesystem: true + label: var + with_mount_unit: true + files: + - path: /etc/containers/systemd/target.container + contents: + inline: | + [Unit] + Description=Targetd container + Documentation=https://github.com/jbtrystram/targetcli-containers + After=local-fs.target network-online.target After=nss-lookup.target dev-disk-by\x2did-virtio\x2dtarget.device + Wants=network-online.target + OnFailure=emergency.target + [Container] + Image=quay.io/jbtrystram/targetcli:latest + ContainerName=target + Network=host + Volume=/dev/disk/by-id/virtio-target:/dev/disk/by-id/virtio-target + Volume=/lib/modules:/lib/modules + Volume=/sys/kernel/config:/sys/kernel/config + PodmanArgs=--privileged + [Install] + # Start by default on boot + WantedBy=multi-user.target + - path: /usr/local/bin/targetcli_script + mode: 0755 + contents: + inline: | + #!/bin/bash + set -xeuo pipefail + podman exec target bash -exc " + targetcli /backstores/block create name=coreos dev=/dev/disk/by-id/virtio-target + targetcli iscsi/ create iqn.2023-10.coreos.target.vm:coreos + targetcli iscsi/iqn.2023-10.coreos.target.vm:coreos/tpg1/luns create /backstores/block/coreos + targetcli iscsi/iqn.2023-10.coreos.target.vm:coreos/tpg1/ set attribute authentication=0 demo_mode_write_protect=0 generate_node_acls=1 cache_dynamic_acls=1 + " + # Will return 0 if the discovery yield a valid portal + iscsiadm -m discovery -p 127.0.0.1 -t st | grep iqn.2023-10.coreos.target.vm:coreos + - path: /mnt/workdir-tmp/boot.ipxe + mode: 0644 + contents: + inline: | + #!ipxe + set initiator-iqn iqn.2023-11.coreos.diskless:testsetup + sanboot iscsi:10.0.2.15::::iqn.2023-10.coreos.target.vm:coreos + - path: /usr/local/bin/install-coreos-iscsi + mode: 0755 + contents: + inline: | + #!/bin/bash + set -euxo + # Mount the iscsi target + iscsiadm -m discovery -t st -p 127.0.0.1 + iscsiadm -m node -T iqn.2023-10.coreos.target.vm:coreos -l + # Give a bit of time to udev to create the persistent names paths + sleep 2 + # Install coreos + coreos-installer install \ + /dev/disk/by-path/ip-127.0.0.1\:3260-iscsi-iqn.2023-10.coreos.target.vm\:coreos-lun-0 \ + --append-karg rd.iscsi.firmware=1 --append-karg ip=ibft \ + --console ttyS0 \ + -i /mnt/workdir-tmp/nested-ign.json + # Unmount the disk + iscsiadm --mode node --logoutall=all + - path: /etc/containers/systemd/coreos-iscsi-vm.container + contents: + inline: | + [Unit] + Description=Boot VM over iSCSI + After=network-online.target After=nss-lookup.target install-coreos-to-iscsi-target.service + Wants=network-online.target install-coreos-to-iscsi-target.service + Requires=install-coreos-to-iscsi-target.service + OnFailure=emergency.target + [Container] + Image=quay.io/coreos-assembler/coreos-assembler + ContainerName=iscsiboot + Volume=/mnt/workdir-tmp/:/mnt/workdir-tmp/ + Volume=/dev/virtio-ports/testiscsicompletion:/mnt/serial + PodmanArgs=--privileged + Network=host + Exec=shell -- kola qemuexec --netboot /mnt/workdir-tmp/boot.ipxe --usernet-addr 10.0.3.0/24 -- -device virtio-serial -chardev file,id=iscsi-completion-virtio,path=/mnt/serial,append=on -device virtserialport,chardev=iscsi-completion-virtio,name=testiscsicompletion + [Install] + # Start by default on boot + WantedBy=multi-user.target + [Service] + # fix permissions on the serial device before passing it as a volume + ExecStartPre=chmod 777 /dev/virtio-ports/testiscsicompletion + - path: /mnt/workdir-tmp/nested-ign.json + contents: + inline: | + { + "ignition": { + "version": "3.1.0" + }, + "systemd": { + "units": [ + { + "contents": "[Unit]\nDescription=iSCSI Boot Signal Completion\nAfter=multi-user.target\nOnFailureJobMode=isolate\n[Service]\nType=oneshot\nRemainAfterExit=yes\nExecStart=/bin/sh -c '/usr/bin/echo \"iscsi-boot-ok\" \u003e/dev/virtio-ports/testiscsicompletion \u0026\u0026 systemctl poweroff'\n[Install]\nRequiredBy=multi-user.target\n", + "enabled": true, + "name": "successful-boot-signal.service" + } + ] + } + } +systemd: + units: + - name: setup-targetcli.service + enabled: true + contents: | + [Unit] + Description=Setup targetcli + Requires=target.service + After=target.service + ConditionFirstBoot=true + OnFailure=emergency.target + [Service] + Type=oneshot + RemainAfterExit=yes + ExecStart=/usr/local/bin/targetcli_script + [Install] + WantedBy=multi-user.target + - name: install-coreos-to-iscsi-target.service + enabled: true + contents: | + [Unit] + Description=Mount an iscsi target and install coreOS into it + Requires=setup-targetcli.service + After=setup-targetcli.service + OnFailure=emergency.target + [Service] + Type=oneshot + RemainAfterExit=yes + ExecStart=/usr/local/bin/install-coreos-iscsi + [Install] + WantedBy=multi-user.target diff --git a/mantle/cmd/kola/testiso.go b/mantle/cmd/kola/testiso.go index 30bed3c856..d4e04aa553 100644 --- a/mantle/cmd/kola/testiso.go +++ b/mantle/cmd/kola/testiso.go @@ -22,6 +22,7 @@ package main import ( "bufio" "context" + _ "embed" "fmt" "io" "os" @@ -88,6 +89,7 @@ var ( "iso-offline-install.bios", "iso-offline-install.mpath.bios", "iso-offline-install-fromram.4k.uefi", + "iso-offline-install-iscsi.bios", "miniso-install.bios", "miniso-install.nm.bios", "miniso-install.4k.uefi", @@ -109,6 +111,7 @@ var ( "miniso-install.s390fw", "miniso-install.nm.s390fw", "miniso-install.4k.nm.s390fw", + "iso-offline-install-iscsi.bios", } tests_ppc64le = []string{ "iso-live-login.ppcfw", @@ -121,6 +124,7 @@ var ( "miniso-install.4k.nm.ppcfw", "pxe-online-install.ppcfw", "pxe-offline-install.4k.ppcfw", + "iso-offline-install-iscsi.bios", } tests_aarch64 = []string{ "iso-live-login.uefi", @@ -136,6 +140,7 @@ var ( "pxe-offline-install.4k.uefi", "pxe-online-install.uefi", "pxe-online-install.4k.uefi", + "iso-offline-install-iscsi.bios", } ) @@ -319,6 +324,9 @@ RequiredBy=coreos-installer.target # for target system RequiredBy=multi-user.target`, nmConnectionId, nmConnectionFile) +//go:embed resources/iscsi_butane_setup.yaml +var iscsi_butane_config string + func init() { cmdTestIso.Flags().BoolVarP(&instInsecure, "inst-insecure", "S", false, "Do not verify signature on metal image") cmdTestIso.Flags().BoolVar(&console, "console", false, "Connect qemu console to terminal, turn off automatic initramfs failure checking") @@ -587,6 +595,8 @@ func runTestIso(cmd *cobra.Command, args []string) (err error) { duration, err = testLiveIso(ctx, inst, filepath.Join(outputDir, test), false) case "miniso-install": duration, err = testLiveIso(ctx, inst, filepath.Join(outputDir, test), true) + case "iso-offline-install-iscsi": + duration, err = testLiveInstalliscsi(ctx, inst, filepath.Join(outputDir, test)) default: plog.Fatalf("Unknown test name:%s", test) } @@ -955,3 +965,77 @@ func testAsDisk(ctx context.Context, outdir string) (time.Duration, error) { return awaitCompletion(ctx, mach, outdir, completionChannel, nil, []string{liveOKSignal}) } + +// iscsi_butane_setup.yaml contain the full butane config but here is an overview of the setup +// 1 - Boot a live ISO with two extra 10G disks with labels "target" and "var" +// - Format and mount `virtio-var` to var +// +// 2 - target.container -> start an iscsi target, using quay.io/jbtrystram/targetcli +// 3 - setup-targetcli.service calls /usr/local/bin/targetcli_script: +// - instructs targetcli to serve /dev/disk/by-id/virtio-target as an iscsi target +// - disables authentication +// - verifies the iscsi service is active and reachable +// +// 4 - install-coreos-to-iscsi-target.service calls /usr/local/bin/install-coreos-iscsi: +// - mount iscsi target +// - run coreos-installer on the mounted block device +// - unmount iscsi +// +// 5 - coreos-iscsi-vm.container start a coreos-assemble: +// - launch cosa qemuexec instructing it to boot from an iPXE script +// wich in turns mount the iscsi target and load kernel +// - note the virtserial port device: we pass through the serial port that was created by kola for test completion +// +// 6 - /mnt/workdir-tmp/nested-ign.json contains an ignition config: +// - when the system is booted, write a success string to /dev/virtio-ports/testiscsicompletion +// - As this serial device is mapped to the host serial device, the test concludes +func testLiveInstalliscsi(ctx context.Context, inst platform.Install, outdir string) (time.Duration, error) { + + builddir := kola.CosaBuild.Dir + isopath := filepath.Join(builddir, kola.CosaBuild.Meta.BuildArtifacts.LiveIso.Path) + builder, err := newBaseQemuBuilder(outdir) + if err != nil { + return 0, err + } + defer builder.Close() + if err := builder.AddIso(isopath, "", false); err != nil { + return 0, err + } + + completionChannel, err := builder.VirtioChannelRead("testiscsicompletion") + if err != nil { + return 0, err + } + + // empty disk to use as an iscsi target to install coreOS on and subseqently boot + // Also add a 10G disk that we will mount on /var, to increase space available when pulling containers + err = builder.AddDisksFromSpecs([]string{"10G:serial=target", "10G:serial=var"}) + if err != nil { + return 0, err + } + + // We need more memory to start another VM within ! + builder.MemoryMiB = 4096 + + var iscsiTargetConfig = conf.Butane(iscsi_butane_config) + + config, err := iscsiTargetConfig.Render(conf.FailWarnings) + if err != nil { + return 0, err + } + // Add a failure target to stop the test if something go wrong rather than waiting for the 10min timeout + config.AddSystemdUnit("coreos-test-entered-emergency-target.service", signalFailureUnit, conf.Enable) + + // enable network + builder.EnableUsermodeNetworking([]platform.HostForwardPort{}, "") + + builder.SetConfig(config) + + mach, err := builder.Exec() + if err != nil { + return 0, errors.Wrapf(err, "running iso") + } + defer mach.Destroy() + + return awaitCompletion(ctx, mach, outdir, completionChannel, nil, []string{"iscsi-boot-ok"}) +}