From 48aa60b5da0a29de25c4ba6f0b0c4cf7b42b2d84 Mon Sep 17 00:00:00 2001 From: Arthur Sengileyev Date: Sat, 13 Jan 2024 15:32:59 +0200 Subject: [PATCH 1/2] Improve cross platform support of QEMU machine Signed-off-by: Arthur Sengileyev --- pkg/machine/e2e/README.md | 9 ++++ pkg/machine/machine_windows.go | 4 +- pkg/machine/qemu/command/command.go | 1 - pkg/machine/qemu/machine.go | 81 +++++++++++++++++------------ pkg/machine/qemu/machine_unix.go | 45 +++++++++------- pkg/machine/qemu/machine_windows.go | 34 +++++++++--- 6 files changed, 114 insertions(+), 60 deletions(-) diff --git a/pkg/machine/e2e/README.md b/pkg/machine/e2e/README.md index 5a1e324a20..23373db7a8 100644 --- a/pkg/machine/e2e/README.md +++ b/pkg/machine/e2e/README.md @@ -36,3 +36,12 @@ Note: To run specific test files, add the test files to the end of the winmake c 1. `export CONTAINERS_MACHINE_PROVIDER="applehv"` 1. `export MACHINE_IMAGE="https://fedorapeople.org/groups/podman/testing/applehv/arm64/fedora-coreos-38.20230925.dev.0-applehv.aarch64.raw.gz"` 1. `make localmachine` (Add `FOCUS_FILE=basic_test.go` to only run basic test) + +### QEMU + +1. Install Podman and QEMU for MacOS bundle using latest release from https://github.com/containers/podman/releases +1. `make podman-remote` +1. `export CONTAINERS_MACHINE_PROVIDER="qemu"` +1. Add bundled QEMU to path `export PATH=/opt/podman/qemu/bin:$PATH` +1. Set search path to gvproxy from bundle `export CONTAINERS_HELPER_BINARY_DIR=/opt/podman/bin` +1. `make localmachine` (Add `FOCUS_FILE=basic_test.go` to only run basic test) diff --git a/pkg/machine/machine_windows.go b/pkg/machine/machine_windows.go index 87c9ae8807..46cd19f99a 100644 --- a/pkg/machine/machine_windows.go +++ b/pkg/machine/machine_windows.go @@ -157,7 +157,7 @@ func StopWinProxy(name string, vmtype define.VMType) error { if err != nil { return nil } - sendQuit(tid) + SendQuit(tid) _ = waitTimeout(proc, 20*time.Second) _ = os.Remove(tidFile) @@ -200,7 +200,7 @@ func waitTimeout(proc *os.Process, timeout time.Duration) bool { return ret } -func sendQuit(tid uint32) { +func SendQuit(tid uint32) { user32 := syscall.NewLazyDLL("user32.dll") postMessage := user32.NewProc("PostThreadMessageW") postMessage.Call(uintptr(tid), WM_QUIT, 0, 0) diff --git a/pkg/machine/qemu/command/command.go b/pkg/machine/qemu/command/command.go index 3619619ef3..a3c56b7bea 100644 --- a/pkg/machine/qemu/command/command.go +++ b/pkg/machine/qemu/command/command.go @@ -53,7 +53,6 @@ func (q *QemuCmd) SetNetwork() { *q = append(*q, "-netdev", "socket,id=vlan,fd=3", "-device", "virtio-net-pci,netdev=vlan,mac=5a:94:ef:e4:0c:ee") } -// SetNetwork adds a network device to the machine func (q *QemuCmd) SetUSBHostPassthrough(usbs []USBConfig) { if len(usbs) == 0 { return diff --git a/pkg/machine/qemu/machine.go b/pkg/machine/qemu/machine.go index ea37098de5..588284b723 100644 --- a/pkg/machine/qemu/machine.go +++ b/pkg/machine/qemu/machine.go @@ -44,6 +44,10 @@ const ( MountType9p = "9p" dockerSock = "/var/run/docker.sock" dockerConnectTimeout = 5 * time.Second + retryCountForStop = 5 + windowsFallbackUID = 501 + baseBackoff = 500 * time.Millisecond + maxStartupBackoffs = 6 ) type MachineVM struct { @@ -146,6 +150,9 @@ func (v *MachineVM) Init(opts machine.InitOptions) (bool, error) { } v.UID = os.Getuid() + if v.UID == -1 { + v.UID = windowsFallbackUID // Used on Windows to match FCOS image + } // Add location of bootable image v.CmdLine.SetBootableImage(v.getImageFile()) @@ -422,7 +429,7 @@ func (v *MachineVM) qemuPid() (int, error) { logrus.Warnf("Reading QEMU pidfile: %v", err) return -1, nil } - return findProcess(pid) + return pingProcess(pid) } // Start executes the qemu command line and forks it @@ -433,9 +440,6 @@ func (v *MachineVM) Start(name string, opts machine.StartOptions) error { qemuSocketConn net.Conn ) - defaultBackoff := 500 * time.Millisecond - maxBackoffs := 6 - v.lock.Lock() defer v.lock.Unlock() @@ -489,7 +493,7 @@ func (v *MachineVM) Start(name string, opts machine.StartOptions) error { logrus.Errorf("machine %q is incompatible with this release of podman and needs to be recreated, starting for recovery only", v.Name) } - forwardSock, forwardState, err := v.startHostNetworking() + forwardSock, forwardState, _, err := v.startHostNetworking(&v.QMPMonitor.Address) if err != nil { return fmt.Errorf("unable to start host networking: %q", err) } @@ -513,7 +517,7 @@ func (v *MachineVM) Start(name string, opts machine.StartOptions) error { return err } - qemuSocketConn, err = sockets.DialSocketWithBackoffs(maxBackoffs, defaultBackoff, v.QMPMonitor.Address.Path) + qemuSocketConn, err = sockets.DialSocketWithBackoffs(maxStartupBackoffs, baseBackoff, v.QMPMonitor.Address.Path) if err != nil { return fmt.Errorf("failed to connect to qemu monitor socket: %w", err) } @@ -532,9 +536,6 @@ func (v *MachineVM) Start(name string, opts machine.StartOptions) error { defer dnr.Close() defer dnw.Close() - attr := new(os.ProcAttr) - files := []*os.File{dnr, dnw, dnw, fd} - attr.Files = files cmdLine := v.CmdLine cmdLine.SetPropagatedHostEnvs() @@ -551,12 +552,15 @@ func (v *MachineVM) Start(name string, opts machine.StartOptions) error { // actually run the command that starts the virtual machine cmd := &exec.Cmd{ - Args: cmdLine, - Path: cmdLine[0], - Stdin: dnr, - Stdout: dnw, - Stderr: stderrBuf, - ExtraFiles: []*os.File{fd}, + Args: cmdLine, + Path: cmdLine[0], + Stdin: dnr, + Stdout: dnw, + Stderr: stderrBuf, + } + // Forward FD if one was allocated + if fd != nil { + cmd.ExtraFiles = []*os.File{fd} } if err := runStartVMCommand(cmd); err != nil { @@ -569,7 +573,7 @@ func (v *MachineVM) Start(name string, opts machine.StartOptions) error { fmt.Println("Waiting for VM ...") } - conn, err = sockets.DialSocketWithBackoffsAndProcCheck(maxBackoffs, defaultBackoff, v.ReadySocket.GetPath(), checkProcessStatus, "qemu", cmd.Process.Pid, stderrBuf) + conn, err = sockets.DialSocketWithBackoffsAndProcCheck(maxStartupBackoffs, baseBackoff, v.ReadySocket.GetPath(), checkProcessStatus, "qemu", cmd.Process.Pid, stderrBuf) if err != nil { return err } @@ -603,7 +607,7 @@ func (v *MachineVM) Start(name string, opts machine.StartOptions) error { return nil } - connected, sshError, err := v.conductVMReadinessCheck(name, maxBackoffs, defaultBackoff) + connected, sshError, err := v.conductVMReadinessCheck(name, maxStartupBackoffs, baseBackoff) if err != nil { return err } @@ -677,7 +681,7 @@ func (v *MachineVM) checkStatus(monitor *qmp.SocketMonitor) (define.Status, erro func (v *MachineVM) waitForMachineToStop() error { fmt.Println("Waiting for VM to stop running...") waitInternal := 250 * time.Millisecond - for i := 0; i < 5; i++ { + for i := 0; i < retryCountForStop; i++ { state, err := v.State(false) if err != nil { return err @@ -710,15 +714,15 @@ func (v *MachineVM) ProxyPID() (int, error) { return proxyPid, nil } -// cleanupVMProxyProcess kills the proxy process and removes the VM's pidfile -func (v *MachineVM) cleanupVMProxyProcess(proxyProc *os.Process) error { +// cleanupVMProxyProcess kills the proxy process +func (v *MachineVM) cleanupVMProxyProcess(proxyPid int) error { // Kill the process - if err := proxyProc.Kill(); err != nil { + if err := killProcess(proxyPid, false); err != nil { return err } // Remove the pidfile if err := v.PidFilePath.Delete(); err != nil { - return err + logrus.Debugf("Error while removing proxy pidfile: %v", err) } return nil } @@ -762,7 +766,7 @@ func (v *MachineVM) Stop(_ string, _ machine.StopOptions) error { return stopErr } - if err := sigKill(qemuPid); err != nil { + if err := killProcess(qemuPid, true); err != nil { if stopErr == nil { return err } @@ -837,7 +841,7 @@ func (v *MachineVM) stopLocked() error { return err } - if err := v.cleanupVMProxyProcess(proxyProc); err != nil { + if err := v.cleanupVMProxyProcess(proxyPid); err != nil { return err } @@ -869,8 +873,18 @@ func (v *MachineVM) stopLocked() error { } fmt.Println("Waiting for VM to exit...") - for isProcessAlive(vmPid) { - time.Sleep(500 * time.Millisecond) + retries := 60 + for { + alive, _ := isProcessAlive(vmPid) + if retries <= 0 { + logrus.Warning("Giving up on waiting for VM to exit. VM process might still terminate") + break + } + if !alive { + break + } + time.Sleep(baseBackoff) + retries-- } return nil @@ -1101,18 +1115,18 @@ func getDiskSize(path string) (uint64, error) { // startHostNetworking runs a binary on the host system that allows users // to set up port forwarding to the podman virtual machine -func (v *MachineVM) startHostNetworking() (string, machine.APIForwardingState, error) { +func (v *MachineVM) startHostNetworking(vlanSocket *define.VMFile) (string, machine.APIForwardingState, *os.Process, error) { cfg, err := config.Default() if err != nil { - return "", machine.NoForwarding, err + return "", machine.NoForwarding, nil, err } binary, err := cfg.FindHelperBinary(machine.ForwarderBinaryName, false) if err != nil { - return "", machine.NoForwarding, err + return "", machine.NoForwarding, nil, err } cmd := gvproxy.NewGvproxyCommand() - cmd.AddQemuSocket(fmt.Sprintf("unix://%s", v.QMPMonitor.Address.GetPath())) + cmd.AddQemuSocket(fmt.Sprintf("unix://%s", filepath.ToSlash(vlanSocket.GetPath()))) cmd.PidFile = v.PidFilePath.GetPath() cmd.SSHPort = v.Port @@ -1127,11 +1141,13 @@ func (v *MachineVM) startHostNetworking() (string, machine.APIForwardingState, e logrus.Debug(cmd) } + cargs := cmd.ToCmdline() + logrus.Debugf("gvproxy cmd: %v", append([]string{binary}, cargs...)) c := cmd.Cmd(binary) if err := c.Start(); err != nil { - return "", 0, fmt.Errorf("unable to execute: %q: %w", cmd.ToCmdline(), err) + return "", 0, nil, fmt.Errorf("unable to execute: %q: %w", cmd.ToCmdline(), err) } - return forwardSock, state, nil + return forwardSock, state, c.Process, nil } func (v *MachineVM) setupAPIForwarding(cmd gvproxy.GvproxyCommand) (gvproxy.GvproxyCommand, string, machine.APIForwardingState) { @@ -1325,6 +1341,7 @@ func (v *MachineVM) Inspect() (*machine.InspectInfo, error) { return nil, err } connInfo.PodmanSocket = podmanSocket + connInfo.PodmanPipe = podmanPipe(v.Name) return &machine.InspectInfo{ ConfigPath: v.ConfigPath, ConnectionInfo: *connInfo, diff --git a/pkg/machine/qemu/machine_unix.go b/pkg/machine/qemu/machine_unix.go index 37ed1f6193..3f242501de 100644 --- a/pkg/machine/qemu/machine_unix.go +++ b/pkg/machine/qemu/machine_unix.go @@ -8,22 +8,41 @@ import ( "strings" "syscall" + "github.com/containers/podman/v4/pkg/machine/define" "golang.org/x/sys/unix" ) -func isProcessAlive(pid int) bool { +func isProcessAlive(pid int) (bool, error) { err := unix.Kill(pid, syscall.Signal(0)) if err == nil || err == unix.EPERM { - return true + return true, nil } - return false + return false, err +} + +func pingProcess(pid int) (int, error) { + alive, err := isProcessAlive(pid) + if !alive { + if err == unix.ESRCH { + return -1, nil + } + return -1, fmt.Errorf("pinging QEMU process: %w", err) + } + return pid, nil +} + +func killProcess(pid int, force bool) error { + if force { + return unix.Kill(pid, unix.SIGKILL) + } + return unix.Kill(pid, unix.SIGTERM) } func checkProcessStatus(processHint string, pid int, stderrBuf *bytes.Buffer) error { var status syscall.WaitStatus pid, err := syscall.Wait4(pid, &status, syscall.WNOHANG, nil) if err != nil { - return fmt.Errorf("failed to read qem%su process status: %w", processHint, err) + return fmt.Errorf("failed to read %s process status: %w", processHint, err) } if pid > 0 { // child exited @@ -32,6 +51,10 @@ func checkProcessStatus(processHint string, pid int, stderrBuf *bytes.Buffer) er return nil } +func podmanPipe(name string) *define.VMFile { + return nil +} + func pathsFromVolume(volume string) []string { return strings.SplitN(volume, ":", 3) } @@ -42,17 +65,3 @@ func extractTargetPath(paths []string) string { } return paths[0] } - -func sigKill(pid int) error { - return unix.Kill(pid, unix.SIGKILL) -} - -func findProcess(pid int) (int, error) { - if err := unix.Kill(pid, 0); err != nil { - if err == unix.ESRCH { - return -1, nil - } - return -1, fmt.Errorf("pinging QEMU process: %w", err) - } - return pid, nil -} diff --git a/pkg/machine/qemu/machine_windows.go b/pkg/machine/qemu/machine_windows.go index b31a4f1d10..86443f6a67 100644 --- a/pkg/machine/qemu/machine_windows.go +++ b/pkg/machine/qemu/machine_windows.go @@ -7,13 +7,27 @@ import ( "strings" "github.com/containers/podman/v4/pkg/machine" + "github.com/containers/podman/v4/pkg/machine/define" ) -func isProcessAlive(pid int) bool { +func isProcessAlive(pid int) (bool, error) { if checkProcessStatus("process", pid, nil) == nil { - return true + return true, nil } - return false + return false, nil +} + +func pingProcess(pid int) (int, error) { + alive, _ := isProcessAlive(pid) + if !alive { + return -1, nil + } + return pid, nil +} + +func killProcess(pid int, force bool) error { + machine.SendQuit(uint32(pid)) + return nil } func checkProcessStatus(processHint string, pid int, stderrBuf *bytes.Buffer) error { @@ -51,10 +65,16 @@ func extractTargetPath(paths []string) string { return dedup.ReplaceAllLiteralString("/"+target, "/") } -func sigKill(pid int) error { - return nil +func podmanPipe(name string) *define.VMFile { + return &define.VMFile{Path: `\\.\pipe\` + toPipeName(name)} } -func findProcess(pid int) (int, error) { - return -1, nil +func toPipeName(name string) string { + if !strings.HasPrefix(name, "qemu-podman") { + if !strings.HasPrefix(name, "podman") { + name = "podman-" + name + } + name = "qemu-" + name + } + return name } From 3913f3ed43277878b1fdf30b488bf8b24410b178 Mon Sep 17 00:00:00 2001 From: Arthur Sengileyev Date: Tue, 26 Sep 2023 14:52:38 +0300 Subject: [PATCH 2/2] Implement Unix domain socket support for VLAN This change adds support for new QEMU stream netdev added in 7.2.0. It is implemented as an opt-in mode for previously supported platforms and the only supported mode on Windows. Old FD netdev has changes only on podman side. Instead of previously used QMP socket address, it now has VLAN dedicated socket address to make both implementations more similar. As VLAN socket address has short lifespan (it exists only after forwarder has been started and before QEMU has finished startup), it is not promoted to persisted machine settings, but is rather calculated inside Start method. Signed-off-by: Arthur Sengileyev --- pkg/machine/e2e/README.md | 12 +- pkg/machine/qemu/command/command.go | 23 +++- pkg/machine/qemu/command/command_unix.go | 13 ++ pkg/machine/qemu/command/command_windows.go | 5 + pkg/machine/qemu/command/qemu_command_test.go | 69 +++++++++- pkg/machine/qemu/config.go | 15 +- pkg/machine/qemu/machine.go | 130 +++++++++++++----- pkg/machine/qemu/machine_unix.go | 5 + pkg/machine/qemu/machine_windows.go | 13 ++ 9 files changed, 243 insertions(+), 42 deletions(-) create mode 100644 pkg/machine/qemu/command/command_unix.go create mode 100644 pkg/machine/qemu/command/command_windows.go diff --git a/pkg/machine/e2e/README.md b/pkg/machine/e2e/README.md index 23373db7a8..eae1a8a697 100644 --- a/pkg/machine/e2e/README.md +++ b/pkg/machine/e2e/README.md @@ -37,11 +37,21 @@ Note: To run specific test files, add the test files to the end of the winmake c 1. `export MACHINE_IMAGE="https://fedorapeople.org/groups/podman/testing/applehv/arm64/fedora-coreos-38.20230925.dev.0-applehv.aarch64.raw.gz"` 1. `make localmachine` (Add `FOCUS_FILE=basic_test.go` to only run basic test) -### QEMU +### QEMU (fd vlan) + +1. Install Podman and QEMU for MacOS bundle using latest release from https://github.com/containers/podman/releases +1. `make podman-remote` +1. `export CONTAINERS_MACHINE_PROVIDER="qemu"` +1. Add bundled QEMU to path `export PATH=/opt/podman/qemu/bin:$PATH` +1. Set search path to gvproxy from bundle `export CONTAINERS_HELPER_BINARY_DIR=/opt/podman/bin` +1. `make localmachine` (Add `FOCUS_FILE=basic_test.go` to only run basic test) + +### QEMU (UNIX domain socket vlan) 1. Install Podman and QEMU for MacOS bundle using latest release from https://github.com/containers/podman/releases 1. `make podman-remote` 1. `export CONTAINERS_MACHINE_PROVIDER="qemu"` 1. Add bundled QEMU to path `export PATH=/opt/podman/qemu/bin:$PATH` 1. Set search path to gvproxy from bundle `export CONTAINERS_HELPER_BINARY_DIR=/opt/podman/bin` +1. `export CONTAINERS_USE_SOCKET_VLAN=true` 1. `make localmachine` (Add `FOCUS_FILE=basic_test.go` to only run basic test) diff --git a/pkg/machine/qemu/command/command.go b/pkg/machine/qemu/command/command.go index a3c56b7bea..2a86bd40e5 100644 --- a/pkg/machine/qemu/command/command.go +++ b/pkg/machine/qemu/command/command.go @@ -2,6 +2,7 @@ package command import ( "encoding/base64" + "errors" "fmt" "os" "path/filepath" @@ -14,6 +15,11 @@ import ( "github.com/containers/podman/v4/pkg/machine/define" ) +const ( + FdVlanNetdev = "socket,id=vlan,fd=3" + vlanMac = "5a:94:ef:e4:0c:ee" +) + // QemuCmd is an alias around a string slice to prevent the need to migrate the // MachineVM struct due to changes type QemuCmd []string @@ -47,10 +53,23 @@ func (q *QemuCmd) SetQmpMonitor(monitor Monitor) { } // SetNetwork adds a network device to the machine -func (q *QemuCmd) SetNetwork() { +func (q *QemuCmd) SetNetwork(vlanSocket *define.VMFile) error { // Right now the mac address is hardcoded so that the host networking gives it a specific IP address. This is // why we can only run one vm at a time right now - *q = append(*q, "-netdev", "socket,id=vlan,fd=3", "-device", "virtio-net-pci,netdev=vlan,mac=5a:94:ef:e4:0c:ee") + if UseFdVLan() { + *q = append(*q, []string{"-netdev", FdVlanNetdev}...) + } else { + if vlanSocket == nil { + return errors.New("vlanSocket is undefined") + } + *q = append(*q, []string{"-netdev", socketVlanNetdev(vlanSocket.GetPath())}...) + } + *q = append(*q, []string{"-device", "virtio-net-pci,netdev=vlan,mac=" + vlanMac}...) + return nil +} + +func socketVlanNetdev(path string) string { + return fmt.Sprintf("stream,id=vlan,server=off,addr.type=unix,addr.path=%s", path) } func (q *QemuCmd) SetUSBHostPassthrough(usbs []USBConfig) { diff --git a/pkg/machine/qemu/command/command_unix.go b/pkg/machine/qemu/command/command_unix.go new file mode 100644 index 0000000000..b05e0ca51d --- /dev/null +++ b/pkg/machine/qemu/command/command_unix.go @@ -0,0 +1,13 @@ +//go:build darwin || dragonfly || freebsd || linux || netbsd || openbsd +// +build darwin dragonfly freebsd linux netbsd openbsd + +package command + +import ( + "os" + "strings" +) + +func UseFdVLan() bool { + return strings.ToUpper(os.Getenv("CONTAINERS_USE_SOCKET_VLAN")) != "TRUE" +} diff --git a/pkg/machine/qemu/command/command_windows.go b/pkg/machine/qemu/command/command_windows.go new file mode 100644 index 0000000000..9fdea7e3f3 --- /dev/null +++ b/pkg/machine/qemu/command/command_windows.go @@ -0,0 +1,5 @@ +package command + +func UseFdVLan() bool { + return false +} diff --git a/pkg/machine/qemu/command/qemu_command_test.go b/pkg/machine/qemu/command/qemu_command_test.go index 5cfa6c0d5b..cb17641358 100644 --- a/pkg/machine/qemu/command/qemu_command_test.go +++ b/pkg/machine/qemu/command/qemu_command_test.go @@ -40,7 +40,8 @@ func TestQemuCmd(t *testing.T) { cmd.SetCPUs(4) cmd.SetIgnitionFile(*ignFile) cmd.SetQmpMonitor(monitor) - cmd.SetNetwork() + err = cmd.SetNetwork(nil) + assert.NoError(t, err) cmd.SetSerialPort(*readySocket, *vmPidFile, "test-machine") cmd.SetVirtfsMount("/tmp/path", "vol10", "none", true) cmd.SetBootableImage(bootableImagePath) @@ -64,3 +65,69 @@ func TestQemuCmd(t *testing.T) { require.Equal(t, cmd.Build(), expected) } + +func TestQemuCmdUnixVlanMissingSocket(t *testing.T) { + t.Setenv("CONTAINERS_USE_SOCKET_VLAN", "true") + cmd := NewQemuBuilder("/usr/bin/qemu-system-x86_64", []string{}) + err := cmd.SetNetwork(nil) + assert.Error(t, err) +} + +func TestQemuCmdUnixVlan(t *testing.T) { + t.Setenv("CONTAINERS_USE_SOCKET_VLAN", "true") + ignFile, err := define.NewMachineFile(t.TempDir()+"demo-ignition-file.ign", nil) + assert.NoError(t, err) + + machineAddrFile, err := define.NewMachineFile(t.TempDir()+"tmp.sock", nil) + assert.NoError(t, err) + + vlanSocket, err := define.NewMachineFile(t.TempDir()+"vlan.sock", nil) + assert.NoError(t, err) + + readySocket, err := define.NewMachineFile(t.TempDir()+"readySocket.sock", nil) + assert.NoError(t, err) + + vmPidFile, err := define.NewMachineFile(t.TempDir()+"vmpidfile.pid", nil) + assert.NoError(t, err) + + monitor := Monitor{ + Address: *machineAddrFile, + Network: "unix", + Timeout: 3, + } + ignPath := ignFile.GetPath() + addrFilePath := machineAddrFile.GetPath() + readySocketPath := readySocket.GetPath() + vmPidFilePath := vmPidFile.GetPath() + bootableImagePath := t.TempDir() + "test-machine_fedora-coreos-38.20230918.2.0-qemu.x86_64.qcow2" + + cmd := NewQemuBuilder("/usr/bin/qemu-system-x86_64", []string{}) + cmd.SetMemory(2048) + cmd.SetCPUs(4) + cmd.SetIgnitionFile(*ignFile) + cmd.SetQmpMonitor(monitor) + err = cmd.SetNetwork(vlanSocket) + assert.NoError(t, err) + cmd.SetSerialPort(*readySocket, *vmPidFile, "test-machine") + cmd.SetVirtfsMount("/tmp/path", "vol10", "none", true) + cmd.SetBootableImage(bootableImagePath) + cmd.SetDisplay("none") + + expected := []string{ + "/usr/bin/qemu-system-x86_64", + "-m", "2048", + "-smp", "4", + "-fw_cfg", fmt.Sprintf("name=opt/com.coreos/config,file=%s", ignPath), + "-qmp", fmt.Sprintf("unix:%s,server=on,wait=off", addrFilePath), + "-netdev", socketVlanNetdev(vlanSocket.GetPath()), + "-device", "virtio-net-pci,netdev=vlan,mac=5a:94:ef:e4:0c:ee", + "-device", "virtio-serial", + "-chardev", fmt.Sprintf("socket,path=%s,server=on,wait=off,id=atest-machine_ready", readySocketPath), + "-device", "virtserialport,chardev=atest-machine_ready,name=org.fedoraproject.port.0", + "-pidfile", vmPidFilePath, + "-virtfs", "local,path=/tmp/path,mount_tag=vol10,security_model=none,readonly", + "-drive", fmt.Sprintf("if=virtio,file=%s", bootableImagePath), + "-display", "none"} + + require.Equal(t, cmd.Build(), expected) +} diff --git a/pkg/machine/qemu/config.go b/pkg/machine/qemu/config.go index e15e7b0387..a47232fdd3 100644 --- a/pkg/machine/qemu/config.go +++ b/pkg/machine/qemu/config.go @@ -61,15 +61,22 @@ func (v *MachineVM) setQMPMonitorSocket() error { // setNewMachineCMD configure the CLI command that will be run to create the new // machine -func (v *MachineVM) setNewMachineCMD(qemuBinary string, cmdOpts *setNewMachineCMDOpts) { +func (v *MachineVM) setNewMachineCMD(qemuBinary string, cmdOpts *setNewMachineCMDOpts) error { v.CmdLine = command.NewQemuBuilder(qemuBinary, v.addArchOptions(cmdOpts)) v.CmdLine.SetMemory(v.Memory) v.CmdLine.SetCPUs(v.CPUs) v.CmdLine.SetIgnitionFile(v.IgnitionFile) v.CmdLine.SetQmpMonitor(v.QMPMonitor) - v.CmdLine.SetNetwork() + vlanSocket, err := machineSocket(v.Name, "vlan", "") + if err != nil { + return err + } + if err := v.CmdLine.SetNetwork(vlanSocket); err != nil { + return err + } v.CmdLine.SetSerialPort(v.ReadySocket, v.VMPidFilePath, v.Name) v.CmdLine.SetUSBHostPassthrough(v.USBs) + return nil } // NewMachine initializes an instance of a virtual machine based on the qemu @@ -146,7 +153,9 @@ func (p *QEMUVirtualization) NewMachine(opts machine.InitOptions) (machine.VM, e // configure command to run cmdOpts := setNewMachineCMDOpts{imageDir: dataDir} - vm.setNewMachineCMD(execPath, &cmdOpts) + if err := vm.setNewMachineCMD(execPath, &cmdOpts); err != nil { + return nil, err + } return vm, nil } diff --git a/pkg/machine/qemu/machine.go b/pkg/machine/qemu/machine.go index 588284b723..95461583e3 100644 --- a/pkg/machine/qemu/machine.go +++ b/pkg/machine/qemu/machine.go @@ -435,9 +435,9 @@ func (v *MachineVM) qemuPid() (int, error) { // Start executes the qemu command line and forks it func (v *MachineVM) Start(name string, opts machine.StartOptions) error { var ( - conn net.Conn - err error - qemuSocketConn net.Conn + conn net.Conn + err error + fd *os.File ) v.lock.Lock() @@ -493,11 +493,6 @@ func (v *MachineVM) Start(name string, opts machine.StartOptions) error { logrus.Errorf("machine %q is incompatible with this release of podman and needs to be recreated, starting for recovery only", v.Name) } - forwardSock, forwardState, _, err := v.startHostNetworking(&v.QMPMonitor.Address) - if err != nil { - return fmt.Errorf("unable to start host networking: %q", err) - } - rtPath, err := getRuntimeDir() if err != nil { return err @@ -511,23 +506,44 @@ func (v *MachineVM) Start(name string, opts machine.StartOptions) error { } } - // If the qemusocketpath exists and the vm is off/down, we should rm - // it before the dial as to avoid a segv - if err := v.QMPMonitor.Address.Delete(); err != nil { + vlanSocket, err := machineSocket(v.Name, "vlan", "") + if err != nil { + return fmt.Errorf("failed to connect to qemu monitor socket: %w", err) + } + err = vlanSocket.Delete() + if err != nil { return err } + isFdVlanVM := false + for _, c := range v.CmdLine { + if c == command.FdVlanNetdev { + isFdVlanVM = true + } + } - qemuSocketConn, err = sockets.DialSocketWithBackoffs(maxStartupBackoffs, baseBackoff, v.QMPMonitor.Address.Path) + forwardSock, forwardState, forwarderProcess, err := v.startHostNetworking(vlanSocket) if err != nil { - return fmt.Errorf("failed to connect to qemu monitor socket: %w", err) + return fmt.Errorf("unable to start host networking: %q", err) } - defer qemuSocketConn.Close() - fd, err := qemuSocketConn.(*net.UnixConn).File() - if err != nil { - return err + if isFdVlanVM { + qemuSocketConn, err := sockets.DialSocketWithBackoffs(maxStartupBackoffs, baseBackoff, vlanSocket.GetPath()) + if err != nil { + return err + } + defer qemuSocketConn.Close() + + fd, err = qemuSocketConn.(*net.UnixConn).File() + if err != nil { + return err + } + defer fd.Close() + } else { + err := waitForVlanReady(forwarderProcess.Pid, vlanSocket.GetPath()) + if err != nil { + return err + } } - defer fd.Close() dnr, dnw, err := machine.GetDevNullFiles() if err != nil { @@ -637,6 +653,28 @@ func (v *MachineVM) Start(name string, opts machine.StartOptions) error { return nil } +func waitForVlanReady(pid int, vlanPath string) error { + time.Sleep(baseBackoff) + backoff := baseBackoff + for i := 0; i < maxStartupBackoffs; i++ { + if i > 0 { + time.Sleep(backoff) + backoff *= 2 + } + // First need to verify that gvproxy is alive, + // because `.sock` file could belong to a different process + err := checkProcessStatus(machine.ForwarderBinaryName, pid, nil) + if err != nil { + return err + } + _, err = os.Stat(vlanPath) + if err == nil { + break + } + } + return nil +} + func (v *MachineVM) checkStatus(monitor *qmp.SocketMonitor) (define.Status, error) { // this is the format returned from the monitor // {"return": {"status": "running", "singlestep": false, "running": true}} @@ -892,32 +930,42 @@ func (v *MachineVM) stopLocked() error { // NewQMPMonitor creates the monitor subsection of our vm func NewQMPMonitor(network, name string, timeout time.Duration) (command.Monitor, error) { - rtDir, err := getRuntimeDir() + if timeout == 0 { + timeout = defaultQMPTimeout + } + address, err := machineSocket(name, "qmp", "") if err != nil { return command.Monitor{}, err } + monitor := command.Monitor{ + Network: network, + Address: *address, + Timeout: timeout, + } + return monitor, nil +} + +func machineSocket(name, prefix, suffix string) (*define.VMFile, error) { + rtDir, err := getRuntimeDir() + if err != nil { + return nil, err + } if isRootful() { rtDir = "/run" } rtDir = filepath.Join(rtDir, "podman") if _, err := os.Stat(rtDir); errors.Is(err, fs.ErrNotExist) { if err := os.MkdirAll(rtDir, 0755); err != nil { - return command.Monitor{}, err + return nil, err } } - if timeout == 0 { - timeout = defaultQMPTimeout - } - address, err := define.NewMachineFile(filepath.Join(rtDir, "qmp_"+name+".sock"), nil) - if err != nil { - return command.Monitor{}, err + if prefix != "" { + name = prefix + "_" + name } - monitor := command.Monitor{ - Network: network, - Address: *address, - Timeout: timeout, + if suffix != "" { + name = name + "_" + suffix } - return monitor, nil + return define.NewMachineFile(filepath.Join(rtDir, name+".sock"), nil) } // collectFilesToDestroy retrieves the files that will be destroyed by `Remove` @@ -1165,10 +1213,10 @@ func (v *MachineVM) setupAPIForwarding(cmd gvproxy.GvproxyCommand) (gvproxy.Gvpr forwardUser = "root" } - cmd.AddForwardSock(socket.GetPath()) - cmd.AddForwardDest(destSock) - cmd.AddForwardUser(forwardUser) - cmd.AddForwardIdentity(v.IdentityPath) + err = forwardSocketArgs(&cmd, v.Name, socket.GetPath(), destSock, v.IdentityPath, forwardUser) + if err != nil { + return cmd, "", machine.NoForwarding + } // The linking pattern is /var/run/docker.sock -> user global sock (link) -> machine sock (socket) // This allows the helper to only have to maintain one constant target to the user, which can be @@ -1408,6 +1456,18 @@ func (v *MachineVM) editCmdLine(flag string, value string) { } } +func forwardSocketArgs(cmd *gvproxy.GvproxyCommand, name string, path string, destPath string, identityPath string, user string) error { + err := forwardPipeArgs(cmd, name, destPath, identityPath, user) + if err != nil { + return err + } + cmd.AddForwardSock(path) + cmd.AddForwardDest(destPath) + cmd.AddForwardUser(user) + cmd.AddForwardIdentity(identityPath) + return nil +} + func isRootful() bool { // Rootless is not relevant on Windows. In the future rootless.IsRootless // could be switched to return true on Windows, and other codepaths migrated diff --git a/pkg/machine/qemu/machine_unix.go b/pkg/machine/qemu/machine_unix.go index 3f242501de..69cc8e09d0 100644 --- a/pkg/machine/qemu/machine_unix.go +++ b/pkg/machine/qemu/machine_unix.go @@ -8,6 +8,7 @@ import ( "strings" "syscall" + gvproxy "github.com/containers/gvisor-tap-vsock/pkg/types" "github.com/containers/podman/v4/pkg/machine/define" "golang.org/x/sys/unix" ) @@ -51,6 +52,10 @@ func checkProcessStatus(processHint string, pid int, stderrBuf *bytes.Buffer) er return nil } +func forwardPipeArgs(cmd *gvproxy.GvproxyCommand, name string, destPath string, identityPath string, user string) error { + return nil +} + func podmanPipe(name string) *define.VMFile { return nil } diff --git a/pkg/machine/qemu/machine_windows.go b/pkg/machine/qemu/machine_windows.go index 86443f6a67..1454c63e80 100644 --- a/pkg/machine/qemu/machine_windows.go +++ b/pkg/machine/qemu/machine_windows.go @@ -6,6 +6,7 @@ import ( "regexp" "strings" + gvproxy "github.com/containers/gvisor-tap-vsock/pkg/types" "github.com/containers/podman/v4/pkg/machine" "github.com/containers/podman/v4/pkg/machine/define" ) @@ -42,6 +43,18 @@ func checkProcessStatus(processHint string, pid int, stderrBuf *bytes.Buffer) er return nil } +func forwardPipeArgs(cmd *gvproxy.GvproxyCommand, name string, destPath string, identityPath string, user string) error { + machinePipe := toPipeName(name) + if !machine.PipeNameAvailable(machinePipe) { + return fmt.Errorf("could not start api proxy since expected pipe is not available: %s", machinePipe) + } + cmd.AddForwardSock(fmt.Sprintf("npipe:////./pipe/%s", machinePipe)) + cmd.AddForwardDest(destPath) + cmd.AddForwardUser(user) + cmd.AddForwardIdentity(identityPath) + return nil +} + func pathsFromVolume(volume string) []string { paths := strings.SplitN(volume, ":", 3) driveLetterMatcher := regexp.MustCompile(`^(?:\\\\[.?]\\)?[a-zA-Z]$`)