From 37d3bb57df3756df0eaee24c2c1db258232c157a Mon Sep 17 00:00:00 2001 From: Ganga Ram Date: Wed, 2 Oct 2024 10:34:48 +0400 Subject: [PATCH] Added shared memory multi-server feature Enabled memsocket to run on the host in addition to VMs Added support for multiple server instances, each managing a unique set of client IDs Enforced exclusivity of client IDs across all servers Renamed some Nix configuration options for clarity Swapped -c (server mode) and -s (client mode) options Added -l option in server mode to define the list of managed client IDs Signed-off-by: Jaroslaw Kurowski --- modules/hardware/common/shared-mem.nix | 30 ++++++++++--------- .../virtualization/microvm/common/waypipe.nix | 4 +-- .../0001-ivshmem-flat-memory-support.patch | 20 +++++++++---- packages/memsocket/default.nix | 8 ++--- packages/memsocket/module.nix | 10 +++---- 5 files changed, 41 insertions(+), 31 deletions(-) diff --git a/modules/hardware/common/shared-mem.nix b/modules/hardware/common/shared-mem.nix index 222894975..b0198cb01 100644 --- a/modules/hardware/common/shared-mem.nix +++ b/modules/hardware/common/shared-mem.nix @@ -27,7 +27,7 @@ in type = types.bool; default = false; description = mdDoc '' - Enables shared memory communication between virtual machines (VMs) + Enables shared memory communication between virtual machines (VMs) and the host ''; }; memSize = mkOption { @@ -83,7 +83,7 @@ in Enables the memsocket functionality on the host system ''; }; - instancesCount = mkOption { + shmSlots = mkOption { type = types.int; default = if cfg.enable_host then (builtins.length cfg.vms_enabled) + 1 else builtins.length cfg.vms_enabled; @@ -91,18 +91,18 @@ in Number of memory slots allocated in the shared memory region ''; }; - serverSocketPath = mkOption { + clientSocketPath = mkOption { type = types.path; - default = "/run/user/${builtins.toString config.ghaf.users.accounts.uid}/memsocket-server.sock"; + default = "/run/user/${builtins.toString config.ghaf.users.accounts.uid}/memsocket-client.sock"; description = mdDoc '' Specifies the path of the listening socket, which is used by Waypipe - or other server applications as the output socket in server mode for + or other server applications as the output socket in its server mode for data transmission ''; }; - clientSocketPath = mkOption { + serverSocketPath = mkOption { type = types.path; - default = "/run/user/${builtins.toString config.ghaf.users.accounts.uid}/memsocket-client.sock"; + default = "/run/user/${builtins.toString config.ghaf.users.accounts.uid}/memsocket-server.sock"; description = mdDoc '' Specifies the location of the output socket, which will connected to in order to receive data from AppVMs. This socket must be created by @@ -142,7 +142,7 @@ in } (mkIf cfg.enable_host { environment.systemPackages = [ - (pkgs.callPackage ../../../packages/memsocket { vms = cfg.instancesCount; }) + (pkgs.callPackage ../../../packages/memsocket { inherit (cfg) shmSlots; }) ]; }) { @@ -151,7 +151,7 @@ in pidFilePath = "/tmp/ivshmem-server.pid"; ivShMemSrv = let - vectors = toString (2 * cfg.instancesCount); + vectors = toString (2 * cfg.shmSlots); in pkgs.writeShellScriptBin "ivshmemsrv" '' if [ -S ${cfg.hostSocketPath} ]; then @@ -180,8 +180,8 @@ in { microvm.vms = let - memsocket = pkgs.callPackage ../../../packages/memsocket { vms = cfg.instancesCount; }; - vectors = toString (2 * cfg.instancesCount); + memsocket = pkgs.callPackage ../../../packages/memsocket { inherit (cfg) shmSlots; }; + vectors = toString (2 * cfg.shmSlots); makeAssignment = vmName: { ${vmName} = { config = { @@ -200,7 +200,7 @@ in boot.extraModulePackages = [ (pkgs.linuxPackages.callPackage ../../../packages/memsocket/module.nix { inherit (config.microvm.vms.${vmName}.config.config.boot.kernelPackages) kernel; - vmCount = cfg.instancesCount; + inherit (cfg) shmSlots; }) ]; services = { @@ -221,7 +221,9 @@ in after = [ "labwc.service" ]; serviceConfig = { Type = "simple"; - ExecStart = "${memsocket}/bin/memsocket -c ${cfg.clientSocketPath}"; + # option '-l -1': listen to all slots. If you want to run other servers + # for some slots, provide a list of handled slots, e.g.: '-l 1,3,5' + ExecStart = "${memsocket}/bin/memsocket -s ${cfg.serverSocketPath} -l -1"; Restart = "always"; RestartSec = "1"; }; @@ -237,7 +239,7 @@ in description = "memsocket"; serviceConfig = { Type = "simple"; - ExecStart = "${memsocket}/bin/memsocket -s ${cfg.serverSocketPath} ${builtins.toString vmIndex}"; + ExecStart = "${memsocket}/bin/memsocket -c ${cfg.clientSocketPath} ${builtins.toString vmIndex}"; Restart = "always"; RestartSec = "1"; }; diff --git a/modules/microvm/virtualization/microvm/common/waypipe.nix b/modules/microvm/virtualization/microvm/common/waypipe.nix index 3d5e1bafb..ed49baeac 100644 --- a/modules/microvm/virtualization/microvm/common/waypipe.nix +++ b/modules/microvm/virtualization/microvm/common/waypipe.nix @@ -25,7 +25,7 @@ let if configHost.ghaf.shm.display then '' #!${pkgs.runtimeShell} -e - ${pkgs.waypipe}/bin/waypipe -s ${configHost.ghaf.shm.serverSocketPath} server "$@" + ${pkgs.waypipe}/bin/waypipe -s ${configHost.ghaf.shm.clientSocketPath} server "$@" '' else '' @@ -76,7 +76,7 @@ in RestartSec = "1"; ExecStart = if configHost.ghaf.shm.display then - "${pkgs.waypipe}/bin/waypipe --secctx \"${vm.name}\" ${waypipeBorder} -s ${configHost.ghaf.shm.clientSocketPath} client" + "${pkgs.waypipe}/bin/waypipe --secctx \"${vm.name}\" ${waypipeBorder} -s ${configHost.ghaf.shm.serverSocketPath} client" else "${pkgs.waypipe}/bin/waypipe --vsock --secctx \"${vm.name}\" ${waypipeBorder} -s ${toString waypipePort} client"; }; diff --git a/overlays/custom-packages/qemu/0001-ivshmem-flat-memory-support.patch b/overlays/custom-packages/qemu/0001-ivshmem-flat-memory-support.patch index 690f15352..efb0ea790 100644 --- a/overlays/custom-packages/qemu/0001-ivshmem-flat-memory-support.patch +++ b/overlays/custom-packages/qemu/0001-ivshmem-flat-memory-support.patch @@ -1,17 +1,17 @@ -From 13229d9e11eaf15eb6679be036364b9d0256f1c1 Mon Sep 17 00:00:00 2001 +From e830291535405f73b9b8d32e97b273b806ed3097 Mon Sep 17 00:00:00 2001 From: Jaroslaw Kurowski -Date: Wed, 10 Jul 2024 15:21:07 +0400 +Date: Thu, 21 Nov 2024 14:19:07 +0400 Subject: ivshmem flat memory support --- - contrib/ivshmem-server/ivshmem-server.c | 6 +- + contrib/ivshmem-server/ivshmem-server.c | 7 ++- hw/i386/pc_q35.c | 2 + hw/misc/ivshmem.c | 74 ++++++++++++++++++++++--- include/hw/misc/ivshmem.h | 1 + - 4 files changed, 71 insertions(+), 12 deletions(-) + 4 files changed, 72 insertions(+), 12 deletions(-) diff --git a/contrib/ivshmem-server/ivshmem-server.c b/contrib/ivshmem-server/ivshmem-server.c -index 2f3c732..906c5d0 100644 +index 2f3c732..e7c7774 100644 --- a/contrib/ivshmem-server/ivshmem-server.c +++ b/contrib/ivshmem-server/ivshmem-server.c @@ -11,6 +11,7 @@ @@ -36,6 +36,14 @@ index 2f3c732..906c5d0 100644 g_free(filename); } +@@ -347,6 +347,7 @@ ivshmem_server_start(IvshmemServer *server) + + server->sock_fd = sock_fd; + server->shm_fd = shm_fd; ++ server->cur_id = 1; + + return 0; + diff --git a/hw/i386/pc_q35.c b/hw/i386/pc_q35.c index c7bc8a2..d76939e 100644 --- a/hw/i386/pc_q35.c @@ -254,5 +262,5 @@ index 433ef53..43aeab7 100644 #endif /* IVSHMEM_H */ -- -2.45.2 +2.47.0 diff --git a/packages/memsocket/default.nix b/packages/memsocket/default.nix index 968fbd6a3..167313c90 100644 --- a/packages/memsocket/default.nix +++ b/packages/memsocket/default.nix @@ -4,7 +4,7 @@ stdenv, lib, debug ? false, - vms, + shmSlots, fetchFromGitHub, ... }: @@ -14,11 +14,11 @@ stdenv.mkDerivation { src = fetchFromGitHub { owner = "tiiuae"; repo = "shmsockproxy"; - rev = "2357926b94ed12c050fdbfbfc0f248393a4c9ea1"; - sha256 = "sha256-9KlHuVbe5qvjRUXj7oyJ1X7CLvqj7/OoVGDWRqpIY2s="; + rev = "1e52f40df4b6a93cdbdf3ad43302cf1f7c313bfa"; + sha256 = "sha256-nWPDWMryM5yONOwQWcBu4VlHJ5Y0slT6FtNKbajxrmY="; }; - CFLAGS = "-O2 -DVM_COUNT=" + (toString vms) + (if debug then " -DDEBUG_ON" else ""); + CFLAGS = "-O2 -DSHM_SLOTS=" + (toString shmSlots) + (if debug then " -DDEBUG_ON" else ""); sourceRoot = "source/app"; installPhase = '' diff --git a/packages/memsocket/module.nix b/packages/memsocket/module.nix index e4ee0c06a..4e2c7d52b 100644 --- a/packages/memsocket/module.nix +++ b/packages/memsocket/module.nix @@ -5,18 +5,18 @@ lib, kernel, fetchFromGitHub, - vmCount, + shmSlots, ... }: stdenv.mkDerivation { - inherit vmCount; + inherit shmSlots; name = "ivshmem-driver-${kernel.version}"; src = fetchFromGitHub { owner = "tiiuae"; repo = "shmsockproxy"; - rev = "2357926b94ed12c050fdbfbfc0f248393a4c9ea1"; - sha256 = "sha256-9KlHuVbe5qvjRUXj7oyJ1X7CLvqj7/OoVGDWRqpIY2s="; + rev = "1e52f40df4b6a93cdbdf3ad43302cf1f7c313bfa"; + sha256 = "sha256-nWPDWMryM5yONOwQWcBu4VlHJ5Y0slT6FtNKbajxrmY="; }; sourceRoot = "source/module"; @@ -29,7 +29,7 @@ stdenv.mkDerivation { makeFlags = [ "KDIR=${kernel.dev}/lib/modules/${kernel.modDirVersion}/build" "MODULEDIR=$(out)/lib/modules/${kernel.modDirVersion}/kernel/drivers/char" - "CFLAGS_kvm_ivshmem.o=\"-DCONFIG_KVM_IVSHMEM_VM_COUNT=${builtins.toString vmCount}\"" + "CFLAGS_kvm_ivshmem.o=\"-DCONFIG_KVM_IVSHMEM_SHM_SLOTS=${builtins.toString shmSlots}\"" ]; meta = with lib; {