From 03e1767274040af38726bef11d5165c985e231f0 Mon Sep 17 00:00:00 2001 From: "Matthieu Baerts (NGI0)" Date: Wed, 27 Nov 2024 15:52:21 +0100 Subject: [PATCH 01/13] virtme-ng: use a locally modified version Useful for the development, when it is required to modify Virtme NG: VIRTME_NG_PATH="${PWD}/../virtme-ng" ./.virtme-clang.sh vm Signed-off-by: Matthieu Baerts (NGI0) --- Dockerfile | 2 +- run.sh | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 0010eb3..6af0297 100644 --- a/Dockerfile +++ b/Dockerfile @@ -95,7 +95,7 @@ RUN for i in /usr/lib/klibc/bin/*; do \ # CCache for quicker builds with default colours # Note: use 'ccache -M xG' to increase max size, default is 5GB -ENV PATH=/usr/lib/ccache:${PATH} +ENV PATH=/usr/lib/ccache:/opt/virtme-ng:${PATH} ENV CCACHE_COMPRESS=true ENV KBUILD_BUILD_TIMESTAMP="0" ENV GCC_COLORS=error=01;31:warning=01;35:note=01;36:caret=01;32:locus=01:quote=01 diff --git a/run.sh b/run.sh index 1ff1ff0..eba2cf5 100755 --- a/run.sh +++ b/run.sh @@ -17,6 +17,7 @@ docker run \ ${VIRTME_PACKETDRILL_PATH:+-v "${VIRTME_PACKETDRILL_PATH}:/opt/packetdrill:rw"} \ -v "${HOME_DIR}:/root" \ ${VIRTME_SYZKALLER_PATH:+ -v "${VIRTME_SYZKALLER_PATH}:/opt/syzkaller:rw"} \ + ${VIRTME_NG_PATH:+ -v "${VIRTME_NG_PATH}:/opt/virtme-ng:ro"} \ -w "${PWD}" \ -e "INPUT_CLANG" \ -e "INPUT_TRACE" \ From ee6b416dd13e9306ac1396e16db45c6bd0ee1f8c Mon Sep 17 00:00:00 2001 From: "Matthieu Baerts (NGI0)" Date: Wed, 27 Nov 2024 16:16:11 +0100 Subject: [PATCH 02/13] net: support bridges First step to be able to start multiple VMs talking to each others. Note: this requires virtme-ng dev version [1]. Link: https://github.com/arighi/virtme-ng/pull/194 Signed-off-by: Matthieu Baerts (NGI0) --- Dockerfile | 3 ++- entrypoint.sh | 50 +++++++++++++++++++++++++++++++++++++++++++++++++- run.sh | 1 + 3 files changed, 52 insertions(+), 2 deletions(-) diff --git a/Dockerfile b/Dockerfile index 6af0297..9c328bb 100644 --- a/Dockerfile +++ b/Dockerfile @@ -20,7 +20,8 @@ RUN apt-get update && \ pkg-config libmnl-dev \ clang clangd clang-tidy lld llvm llvm-dev libcap-dev \ gdb gdb-multiarch crash dwarves strace trace-cmd \ - iptables ebtables nftables vim psmisc bash-completion less jq \ + iptables ebtables nftables bridge-utils \ + vim psmisc bash-completion less jq \ gettext-base libevent-dev libtraceevent-dev libnewt0.52 libslang2 libutempter0 python3-newt tmux \ libdwarf-dev libbfd-dev libnuma-dev libzstd-dev libunwind-dev libdw-dev libslang2-dev python3-dev python3-setuptools binutils-dev libiberty-dev libbabeltrace-dev systemtap-sdt-dev libperl-dev python3-docutils \ libtap-formatter-junit-perl lcov libjson-xs-perl \ diff --git a/entrypoint.sh b/entrypoint.sh index 80e9864..063c0b8 100755 --- a/entrypoint.sh +++ b/entrypoint.sh @@ -54,6 +54,7 @@ set_trace_on : "${INPUT_CPUS:=""}" : "${INPUT_RAM:=""}" : "${INPUT_GCOV:=""}" +: "${INPUT_NET_BRIDGES:=""}" : "${INPUT_CI_RESULTS_DIR:=""}" : "${INPUT_CI_PRINT_EXIT_CODE:=1}" : "${INPUT_CI_TIMEOUT_SEC:=5400}" @@ -204,6 +205,45 @@ _get_results_dir() { echo "${RESULTS_DIR_BASE}/$(git rev-parse --short HEAD || echo "UNKNOWN")/${1}" } +# $1: bridge name +_add_bridge() { + local br="${1}" + local i="${br//[^0-9]/}" # only the numbers + + VIRTME_RUN_OPTS+=("--net=bridge=${br}") + echo "allow ${br}" >> /etc/qemu/bridge.conf + brctl addbr "${br}" + ip addr add "10.0.${i}.1/24" dev "${br}" + ip link set "${br}" up + + cat <<-EOF > "/tmp/udhcpd-${br}.conf" + start 10.0.${i}.2 + end 10.0.${i}.2 + interface ${br} + pidfile /var/run/udhcpd-${br}.pid + lease_file /var/lib/misc/udhcpd-${br}.leases + opt subnet 255.255.255.0 + EOF + + touch "/var/lib/misc/udhcpd-${br}.leases" + busybox udhcpd "/tmp/udhcpd-${br}.conf" +} + +_setup_bridges() { + chmod u+s /usr/lib/qemu/qemu-bridge-helper + mkdir -p /etc/qemu + touch /etc/qemu/bridge.conf + chmod 755 /etc/qemu/bridge.conf + sysctl -w net.bridge.bridge-nf-call-ip6tables=0 + sysctl -w net.bridge.bridge-nf-call-iptables=0 + sysctl -w net.bridge.bridge-nf-call-arptables=0 + + local br + for br in "${@}"; do + _add_bridge "${br}" + done +} + setup_env() { local mode mode="${1}" @@ -284,7 +324,15 @@ setup_env() { local mode : "${INPUT_GCOV:=0}" # add net support, can be useful, but delay the start of the tests (~1 sec?) - VIRTME_RUN_OPTS+=("--net") + if [ -z "${INPUT_NET_BRIDGES}" ]; then + VIRTME_RUN_OPTS+=("--net") + fi + fi + + if [ -n "${INPUT_NET_BRIDGES}" ]; then + local bridges + IFS=',' read -ra bridges <<< "${INPUT_NET_BRIDGES}" + _setup_bridges "${bridges[@]}" fi : "${INPUT_RAM:="$((2048 * (1 + INPUT_GCOV)))M"}" # More needed for GCOV, not to swap diff --git a/run.sh b/run.sh index eba2cf5..fe30b30 100755 --- a/run.sh +++ b/run.sh @@ -43,6 +43,7 @@ docker run \ -e "INPUT_CPUS" \ -e "INPUT_RAM" \ -e "INPUT_GCOV" \ + -e "INPUT_NET_BRIDGES" \ -e "VIRTME_ARCH" \ -e "COMPILER" \ --privileged \ From f819033e3a3caa6de70be1afdc54fea99fda42a7 Mon Sep 17 00:00:00 2001 From: "Matthieu Baerts (NGI0)" Date: Wed, 27 Nov 2024 16:18:48 +0100 Subject: [PATCH 03/13] net: bridge: add access to host via the first iface With this mode, and compared to the 'user' one, there is no proxy, so tests can be done from the VM to the outside world, e.g. $ mptcpize run curl https://check.mptcp.dev You are using MPTCP. Signed-off-by: Matthieu Baerts (NGI0) --- entrypoint.sh | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/entrypoint.sh b/entrypoint.sh index 063c0b8..fff1eb7 100755 --- a/entrypoint.sh +++ b/entrypoint.sh @@ -206,7 +206,7 @@ _get_results_dir() { } # $1: bridge name -_add_bridge() { +_add_bridge() { local router local br="${1}" local i="${br//[^0-9]/}" # only the numbers @@ -216,6 +216,11 @@ _add_bridge() { ip addr add "10.0.${i}.1/24" dev "${br}" ip link set "${br}" up + # one default address + if [ "${i}" = "0" ]; then + router="opt router 10.0.${i}.1" + fi + cat <<-EOF > "/tmp/udhcpd-${br}.conf" start 10.0.${i}.2 end 10.0.${i}.2 @@ -223,6 +228,7 @@ _add_bridge() { pidfile /var/run/udhcpd-${br}.pid lease_file /var/lib/misc/udhcpd-${br}.leases opt subnet 255.255.255.0 + ${router} EOF touch "/var/lib/misc/udhcpd-${br}.leases" @@ -237,6 +243,8 @@ _setup_bridges() { sysctl -w net.bridge.bridge-nf-call-ip6tables=0 sysctl -w net.bridge.bridge-nf-call-iptables=0 sysctl -w net.bridge.bridge-nf-call-arptables=0 + # only v4 for the moment + iptables -t nat -A POSTROUTING -s 10.0.0.0/16 -o eth0 -j MASQUERADE local br for br in "${@}"; do From a0f1495b2b81b88759135711aa45fef38a749b76 Mon Sep 17 00:00:00 2001 From: "Matthieu Baerts (NGI0)" Date: Wed, 27 Nov 2024 16:20:18 +0100 Subject: [PATCH 04/13] docker: add simple net tools - iperf3 and netperf to run tests - bmon and ifstat to quickly check the bandwidth Signed-off-by: Matthieu Baerts (NGI0) --- Dockerfile | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 9c328bb..a456900 100644 --- a/Dockerfile +++ b/Dockerfile @@ -30,7 +30,8 @@ RUN apt-get update && \ cscope \ bpftrace \ golang \ - mptcpize \ + mptcpize iperf3 netperf \ + bmon ifstat \ && \ apt-get clean From e63e1d285d7d73b1f3c97de4d11b9bb46ea0ae56 Mon Sep 17 00:00:00 2001 From: "Matthieu Baerts (NGI0)" Date: Fri, 29 Nov 2024 12:01:06 +0100 Subject: [PATCH 05/13] bridge: ability to change MAC address Needed to avoid conflicts with multiple VMs running in parallel. It requires a WIP version of virtme-ng [1]. Link: https://github.com/arighi/virtme-ng/pull/195 [1] Signed-off-by: Matthieu Baerts (NGI0) --- entrypoint.sh | 14 ++++++++++++-- run.sh | 1 + 2 files changed, 13 insertions(+), 2 deletions(-) diff --git a/entrypoint.sh b/entrypoint.sh index fff1eb7..7fa4895 100755 --- a/entrypoint.sh +++ b/entrypoint.sh @@ -55,6 +55,7 @@ set_trace_on : "${INPUT_RAM:=""}" : "${INPUT_GCOV:=""}" : "${INPUT_NET_BRIDGES:=""}" +: "${INPUT_MAC_ADDRESS_PREFIX:=""}" : "${INPUT_CI_RESULTS_DIR:=""}" : "${INPUT_CI_PRINT_EXIT_CODE:=1}" : "${INPUT_CI_TIMEOUT_SEC:=5400}" @@ -206,7 +207,7 @@ _get_results_dir() { } # $1: bridge name -_add_bridge() { local router +_add_bridge() { local router static local br="${1}" local i="${br//[^0-9]/}" # only the numbers @@ -221,14 +222,19 @@ _add_bridge() { local router router="opt router 10.0.${i}.1" fi + if [ -n "${INPUT_MAC_ADDRESS_PREFIX}" ]; then + static="static_lease ${INPUT_MAC_ADDRESS_PREFIX%=*}:0${i} 10.0.${i}.${INPUT_MAC_ADDRESS_PREFIX#*=}" + fi + cat <<-EOF > "/tmp/udhcpd-${br}.conf" start 10.0.${i}.2 - end 10.0.${i}.2 + end 10.0.${i}.254 interface ${br} pidfile /var/run/udhcpd-${br}.pid lease_file /var/lib/misc/udhcpd-${br}.leases opt subnet 255.255.255.0 ${router} + ${static} EOF touch "/var/lib/misc/udhcpd-${br}.leases" @@ -246,6 +252,10 @@ _setup_bridges() { # only v4 for the moment iptables -t nat -A POSTROUTING -s 10.0.0.0/16 -o eth0 -j MASQUERADE + if [ -n "${INPUT_MAC_ADDRESS_PREFIX}" ]; then + VIRTME_RUN_OPTS+=("--net-mac-address" "${INPUT_MAC_ADDRESS_PREFIX%=*}:00") + fi + local br for br in "${@}"; do _add_bridge "${br}" diff --git a/run.sh b/run.sh index fe30b30..fb54b05 100755 --- a/run.sh +++ b/run.sh @@ -44,6 +44,7 @@ docker run \ -e "INPUT_RAM" \ -e "INPUT_GCOV" \ -e "INPUT_NET_BRIDGES" \ + -e "INPUT_MAC_ADDRESS_PREFIX" \ -e "VIRTME_ARCH" \ -e "COMPILER" \ --privileged \ From 3440e1ce63489f5cddb66011b60e711c3a60b0e1 Mon Sep 17 00:00:00 2001 From: "Matthieu Baerts (NGI0)" Date: Fri, 29 Nov 2024 12:05:23 +0100 Subject: [PATCH 06/13] bridge: use var instead of duplicated text To avoid typo, easier to read. Signed-off-by: Matthieu Baerts (NGI0) --- entrypoint.sh | 27 +++++++++++++++------------ 1 file changed, 15 insertions(+), 12 deletions(-) diff --git a/entrypoint.sh b/entrypoint.sh index 7fa4895..158435e 100755 --- a/entrypoint.sh +++ b/entrypoint.sh @@ -209,36 +209,39 @@ _get_results_dir() { # $1: bridge name _add_bridge() { local router static local br="${1}" + local i="${br//[^0-9]/}" # only the numbers + local prefix="10.0.${i}" + local conf="/tmp/udhcpd-${br}.conf" + local pidfile="/var/run/udhcpd-${br}.pid" + local leases="/var/lib/misc/udhcpd-${br}.leases" VIRTME_RUN_OPTS+=("--net=bridge=${br}") echo "allow ${br}" >> /etc/qemu/bridge.conf brctl addbr "${br}" - ip addr add "10.0.${i}.1/24" dev "${br}" + ip addr add "${prefix}.1/24" dev "${br}" ip link set "${br}" up # one default address if [ "${i}" = "0" ]; then - router="opt router 10.0.${i}.1" + router="opt router ${prefix}.1" fi if [ -n "${INPUT_MAC_ADDRESS_PREFIX}" ]; then - static="static_lease ${INPUT_MAC_ADDRESS_PREFIX%=*}:0${i} 10.0.${i}.${INPUT_MAC_ADDRESS_PREFIX#*=}" - fi - - cat <<-EOF > "/tmp/udhcpd-${br}.conf" - start 10.0.${i}.2 - end 10.0.${i}.254 + static="static_lease ${INPUT_MAC_ADDRESS_PREFIX%=*}:0${i} ${prefix}.${INPUT_MAC_ADDRESS_PREFIX#*=}" + cat <<-EOF > "${conf}" + start ${prefix}.2 + end ${prefix}.254 interface ${br} - pidfile /var/run/udhcpd-${br}.pid - lease_file /var/lib/misc/udhcpd-${br}.leases + pidfile ${pidfile} + lease_file ${leases} opt subnet 255.255.255.0 ${router} ${static} EOF - touch "/var/lib/misc/udhcpd-${br}.leases" - busybox udhcpd "/tmp/udhcpd-${br}.conf" + touch "${leases}" # to avoid a warning + busybox udhcpd "${conf}" } _setup_bridges() { From 2a6690ef90251c808a07de7e4d0ea8964bb22d4f Mon Sep 17 00:00:00 2001 From: "Matthieu Baerts (NGI0)" Date: Fri, 29 Nov 2024 12:08:23 +0100 Subject: [PATCH 07/13] input: ability to change the hostname Useful when multiple VMs are started in parallel. Signed-off-by: Matthieu Baerts (NGI0) --- entrypoint.sh | 3 ++- run.sh | 1 + 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/entrypoint.sh b/entrypoint.sh index 158435e..4c9f8d0 100755 --- a/entrypoint.sh +++ b/entrypoint.sh @@ -51,6 +51,7 @@ set_trace_on : "${INPUT_SELFTESTS_MPTCP_LIB_EXPECT_ALL_FEATURES:=1}" : "${INPUT_SELFTESTS_MPTCP_LIB_OVERRIDE_FLAKY:=0}" : "${INPUT_SELFTESTS_MPTCP_LIB_COLOR_FORCE:=1}" +: "${INPUT_HOSTNAME:="mptcpdev"}" : "${INPUT_CPUS:=""}" : "${INPUT_RAM:=""}" : "${INPUT_GCOV:=""}" @@ -112,7 +113,7 @@ VIRTME_CONFIGKERNEL="virtme-configkernel" VIRTME_RUN="virtme-run" VIRTME_RUN_OPTS=( --arch "${VIRTME_ARCH}" - --name "mptcpdev" # hostname + --name "${INPUT_HOSTNAME}" --mods=auto --rw # Don't use "rwdir", it will use 9p ; in a container, we can use rw --pwd diff --git a/run.sh b/run.sh index fb54b05..45c2601 100755 --- a/run.sh +++ b/run.sh @@ -40,6 +40,7 @@ docker run \ -e "INPUT_PACKETDRILL_STABLE=${VIRTME_PACKETDRILL_STABLE:-0}" \ -e "INPUT_EXPECT_TIMEOUT" \ -e "INPUT_EXTRA_ENV" \ + -e "INPUT_HOSTNAME" \ -e "INPUT_CPUS" \ -e "INPUT_RAM" \ -e "INPUT_GCOV" \ From 970acaf0a50f7695fd9c62494f78cf2831175e96 Mon Sep 17 00:00:00 2001 From: "Matthieu Baerts (NGI0)" Date: Fri, 29 Nov 2024 12:10:47 +0100 Subject: [PATCH 08/13] virtme: clearer way to handle qemu-opts All the options have to be set at the end. More options are coming, it is easier to have a dedicated variable for that, instead of mixing them with the other options. Signed-off-by: Matthieu Baerts (NGI0) --- entrypoint.sh | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/entrypoint.sh b/entrypoint.sh index 4c9f8d0..c94e613 100755 --- a/entrypoint.sh +++ b/entrypoint.sh @@ -121,6 +121,7 @@ VIRTME_RUN_OPTS=( --verbose --show-boot-console --kopt mitigations=off ) +VIRTME_RUN_QEMU_OPTS=() # results dir RESULTS_DIR_BASE="${VIRTME_WORKDIR}/results" @@ -1270,7 +1271,7 @@ EOF run() { printinfo "Run the virtme script: manual" - "${VIRTME_RUN}" "${VIRTME_RUN_OPTS[@]}" + "${VIRTME_RUN}" "${VIRTME_RUN_OPTS[@]}" ${VIRTME_RUN_QEMU_OPTS:+--qemu-opts "${VIRTME_RUN_QEMU_OPTS[@]}"} } run_expect() { @@ -1287,7 +1288,8 @@ run_expect() { fi # force a stop in case of panic, but avoid a reboot in "expect" mode - VIRTME_RUN_OPTS+=(--kopt panic=-1 --qemu-opts -no-reboot) + VIRTME_RUN_OPTS+=(--kopt panic=-1) + VIRTME_RUN_QEMU_OPTS+=(-no-reboot) printinfo "Run the virtme script: expect (timeout: ${VIRTME_EXPECT_TEST_TIMEOUT})" @@ -1320,7 +1322,7 @@ EOF #! /bin/bash echo -e "$(log_section_start "Boot VM")" set -x -"${VIRTME_RUN}" ${VIRTME_RUN_OPTS[@]} 2>&1 | tr -d '\r' +"${VIRTME_RUN}" ${VIRTME_RUN_OPTS[@]} ${VIRTME_RUN_QEMU_OPTS:+--qemu-opts ${VIRTME_RUN_QEMU_OPTS[@]}} 2>&1 | tr -d '\r' EOF chmod +x "${VIRTME_RUN_SCRIPT}" From 970f5d1878342d3debca309bebe468db1e187b6c Mon Sep 17 00:00:00 2001 From: "Matthieu Baerts (NGI0)" Date: Fri, 29 Nov 2024 14:14:08 +0100 Subject: [PATCH 09/13] console: add VSock support To be able to connect to a VM without using the network. In the VM, we need to run: socat VSOCK-LISTEN:1024,reuseaddr,fork EXEC:"byobu",pty,stderr,setsid,sigint,sane,echo=1 & In the Docker: socat file:$(tty),raw,echo=0, VSOCK-CONNECT:${INPUT_VSOCK_CID}:1024 Where ${INPUT_VSOCK_CID} is 3 by default. Signed-off-by: Matthieu Baerts (NGI0) --- Dockerfile | 4 ++-- entrypoint.sh | 8 ++++++++ run.sh | 1 + 3 files changed, 11 insertions(+), 2 deletions(-) diff --git a/Dockerfile b/Dockerfile index a456900..4a8eabb 100644 --- a/Dockerfile +++ b/Dockerfile @@ -20,9 +20,9 @@ RUN apt-get update && \ pkg-config libmnl-dev \ clang clangd clang-tidy lld llvm llvm-dev libcap-dev \ gdb gdb-multiarch crash dwarves strace trace-cmd \ - iptables ebtables nftables bridge-utils \ + iptables ebtables nftables bridge-utils socat \ vim psmisc bash-completion less jq \ - gettext-base libevent-dev libtraceevent-dev libnewt0.52 libslang2 libutempter0 python3-newt tmux \ + gettext-base libevent-dev libtraceevent-dev libnewt0.52 libslang2 libutempter0 python3-newt tmux gawk \ libdwarf-dev libbfd-dev libnuma-dev libzstd-dev libunwind-dev libdw-dev libslang2-dev python3-dev python3-setuptools binutils-dev libiberty-dev libbabeltrace-dev systemtap-sdt-dev libperl-dev python3-docutils \ libtap-formatter-junit-perl lcov libjson-xs-perl \ zstd \ diff --git a/entrypoint.sh b/entrypoint.sh index c94e613..c53b597 100755 --- a/entrypoint.sh +++ b/entrypoint.sh @@ -57,6 +57,7 @@ set_trace_on : "${INPUT_GCOV:=""}" : "${INPUT_NET_BRIDGES:=""}" : "${INPUT_MAC_ADDRESS_PREFIX:=""}" +: "${INPUT_VSOCK_CID:="3"}" : "${INPUT_CI_RESULTS_DIR:=""}" : "${INPUT_CI_PRINT_EXIT_CODE:=1}" : "${INPUT_CI_TIMEOUT_SEC:=5400}" @@ -350,6 +351,10 @@ setup_env() { local mode if [ -z "${INPUT_NET_BRIDGES}" ]; then VIRTME_RUN_OPTS+=("--net") fi + + # In the VM, run (or "bash -i" instead of "byobu"): socat VSOCK-LISTEN:1024,reuseaddr,fork EXEC:"byobu",pty,stderr,setsid,sigint,sane,echo=1 & + # From the docker: socat file:$(tty),raw,echo=0, VSOCK-CONNECT:${INPUT_VSOCK_CID}:1024 + VIRTME_RUN_QEMU_OPTS+=("-device" "vhost-vsock-pci,guest-cid=${INPUT_VSOCK_CID}") fi if [ -n "${INPUT_NET_BRIDGES}" ]; then @@ -571,6 +576,9 @@ gen_kconfig() { local mode kconfig=() vck rc=0 # Useful to reproduce issue kconfig+=(-e NET_SCH_TBF) + # Useful to get access to VMs from host + kconfig+=(-e VSOCKETS -e VIRTIO_VSOCKETS) + # Disable retpoline to accelerate tests kconfig+=(-d RETPOLINE) diff --git a/run.sh b/run.sh index 45c2601..ddf13c2 100755 --- a/run.sh +++ b/run.sh @@ -46,6 +46,7 @@ docker run \ -e "INPUT_GCOV" \ -e "INPUT_NET_BRIDGES" \ -e "INPUT_MAC_ADDRESS_PREFIX" \ + -e "INPUT_VSOCK_CID" \ -e "VIRTME_ARCH" \ -e "COMPILER" \ --privileged \ From a1973b5e5c2edbdc485b490cd07e36a9bff0287d Mon Sep 17 00:00:00 2001 From: "Matthieu Baerts (NGI0)" Date: Fri, 29 Nov 2024 20:26:57 +0100 Subject: [PATCH 10/13] console: use VSock support from virtme-ng Cleaner to do that like that. Also added a new 'connect' parameter. Use it via connect.sh Note, it depends on a WIP version of virtme-ng [1] Link: https://github.com/arighi/virtme-ng/pull/197 [1] Signed-off-by: Matthieu Baerts (NGI0) --- README.md | 25 +++++++++++++++++++++++++ connect.sh | 5 +++++ entrypoint.sh | 27 ++++++++++++++++++++------- 3 files changed, 50 insertions(+), 7 deletions(-) create mode 100755 connect.sh diff --git a/README.md b/README.md index c8d2d3b..b6c77f4 100644 --- a/README.md +++ b/README.md @@ -37,6 +37,8 @@ When launching the docker image, you have to specify the mode you want to use: default). - `vm-auto`: Start the VM with what has already been built, then run the tests (`normal` mode by default). +- `connect`: connect to a VM's remote shell via a VSock. For multiple VMs + running in parallel, set a different CID, e.g. `INPUT_VSOCK_CID=42` - `lcov2html`: Generate HTML from LCOV file(s) (available when tests have been executed with `INPUT_GCOV=1`). - `src`: `source` a given script file. @@ -90,6 +92,29 @@ ln -s /PATH/TO/THIS/REPO/run-tests-dev-clang.sh .virtme-clang.sh Then simply call `./.virtme.sh` or `.virtme-clang.sh`. +### Remote Shell + +To connect to an existing VM with a remote shell, you can use this command: + +```bash +docker exec -it \ + "$(docker ps --filter "label=name=mptcp-upstream-virtme-docker" -l --format "{{.ID}}")" \ + /entrypoint.sh connect +``` + +(Or use the `connect.sh` script.) + +Note, if you want to run multiple VMs in parallel, you will need to set the +right container ID or name found with `docker ps`: + +``` +$ docker ps +CONTAINER ID IMAGE COMMAND +f055e439a1e7 mptcp/mptcp-upstream-virtme-docker:latest "/entrypoint.sh manual" (...) +50cd8ce27116 mptcp/mptcp-upstream-virtme-docker:latest "/entrypoint.sh manual" (...) +$ docker exec -it f055e439a1e7 /entrypoint.sh connect +``` + ## Extension ### Files diff --git a/connect.sh b/connect.sh new file mode 100755 index 0000000..530328c --- /dev/null +++ b/connect.sh @@ -0,0 +1,5 @@ +#! /bin/bash -x + +docker exec -it \ + "$(docker ps --filter "label=name=mptcp-upstream-virtme-docker" -l --format "{{.ID}}")" \ + /entrypoint.sh connect "${@}" diff --git a/entrypoint.sh b/entrypoint.sh index c53b597..ff5bca5 100755 --- a/entrypoint.sh +++ b/entrypoint.sh @@ -102,6 +102,7 @@ VIRTME_SCRIPT_UNEXPECTED_STOP="Unexpected stop of the VM" VIRTME_SCRIPT_TIMEOUT="${VIRTME_SCRIPTS_DIR}/tests.timeout" VIRTME_RUN_SCRIPT="${VIRTME_SCRIPTS_DIR}/virtme.sh" VIRTME_RUN_EXPECT="${VIRTME_SCRIPTS_DIR}/virtme.expect" +VIRTME_CONSOLE="${VIRTME_SCRIPTS_DIR}/console.sh" SELFTESTS_DIR="${INPUT_SELFTESTS_DIR:-tools/testing/selftests/net/mptcp}" SELFTESTS_CONFIG="${SELFTESTS_DIR}/config" @@ -352,9 +353,11 @@ setup_env() { local mode VIRTME_RUN_OPTS+=("--net") fi - # In the VM, run (or "bash -i" instead of "byobu"): socat VSOCK-LISTEN:1024,reuseaddr,fork EXEC:"byobu",pty,stderr,setsid,sigint,sane,echo=1 & - # From the docker: socat file:$(tty),raw,echo=0, VSOCK-CONNECT:${INPUT_VSOCK_CID}:1024 - VIRTME_RUN_QEMU_OPTS+=("-device" "vhost-vsock-pci,guest-cid=${INPUT_VSOCK_CID}") + # From the docker: vng --vsock-connect + # TODO: remove if condition when virtme-ng supports it + if virtme-run --help | grep -q vsock; then + VIRTME_RUN_OPTS+=("--vsock" "${VIRTME_CONSOLE}" "--vsock-cid" "${INPUT_VSOCK_CID}") + fi fi if [ -n "${INPUT_NET_BRIDGES}" ]; then @@ -576,9 +579,6 @@ gen_kconfig() { local mode kconfig=() vck rc=0 # Useful to reproduce issue kconfig+=(-e NET_SCH_TBF) - # Useful to get access to VMs from host - kconfig+=(-e VSOCKETS -e VIRTIO_VSOCKETS) - # Disable retpoline to accelerate tests kconfig+=(-d RETPOLINE) @@ -1834,7 +1834,7 @@ usage() { echo echo " - KConfig: optional kernel config: arguments for './scripts/config' or config file" echo - echo "Usage: ${0} | defconfig | selftests | bpftests | cmd | src | static | vm-manual | vm-auto | lcov2html >" + echo "Usage: ${0} | defconfig | selftests | bpftests | cmd | src | static | vm-manual | vm-auto | connect | lcov2html>" echo echo " - make: run the make command with optional parameters" echo " - make.cross: run Intel's make.cross command with optional parameters" @@ -1847,6 +1847,7 @@ usage() { echo " - static: run static analysis, with make W=1 C=1" echo " - vm-manual: start the VM with what has already been built ('normal' mode by default)" echo " - vm-auto: same, then run the tests as well ('normal' mode by default)" + echo " - connect: connect to a VM's remote shell via a VSock (set INPUT_VSOCK_CID for multiple VMs)." echo " - lcov2html: generate html from lcov file (required INPUT_GCOV=1)" echo echo "This script needs to be ran from the root of kernel source code." @@ -1968,6 +1969,18 @@ case "${INPUT_MODE}" in run_expect analyze "${@:-normal}" ;; + "connect") + read -r rows columns <<< "$(stty size)" + cat <<-EOF > "${VIRTME_CONSOLE}" + #! /bin/bash + stty rows ${rows} columns ${columns} + cd "\${virtme_chdir}" + HOME=/root + byobu + EOF + chmod +x "${VIRTME_CONSOLE}" + vng --vsock-connect --vsock-cid "${1:-${INPUT_VSOCK_CID}}" + ;; "lcov2html") setup_env "${@:-normal}" while [ -n "${1}" ] && [ ! -s "${1}" ]; do From 2c7145cb2f65545d55f69811b2002cc426448621 Mon Sep 17 00:00:00 2001 From: "Matthieu Baerts (NGI0)" Date: Fri, 29 Nov 2024 20:28:47 +0100 Subject: [PATCH 11/13] bridge: allow multiple VMs from the same container New step to be able to have multiple VMs started from the same container and talking to each others. Signed-off-by: Matthieu Baerts (NGI0) --- entrypoint.sh | 35 ++++++++++++++++++++++++++++++++--- 1 file changed, 32 insertions(+), 3 deletions(-) diff --git a/entrypoint.sh b/entrypoint.sh index ff5bca5..19a21e6 100755 --- a/entrypoint.sh +++ b/entrypoint.sh @@ -210,6 +210,19 @@ _get_results_dir() { echo "${RESULTS_DIR_BASE}/$(git rev-parse --short HEAD || echo "UNKNOWN")/${1}" } +# $1: pid +kill_wait() { + local pid="${1}" + if [ -z "${pid}" ]; then + return + fi + + kill "${pid}" + while [ -d "/proc/${pid}" ]; do + sleep 0.1 + done +} + # $1: bridge name _add_bridge() { local router static local br="${1}" @@ -221,6 +234,22 @@ _add_bridge() { local router static local leases="/var/lib/misc/udhcpd-${br}.leases" VIRTME_RUN_OPTS+=("--net=bridge=${br}") + + if [ -n "${INPUT_MAC_ADDRESS_PREFIX}" ]; then + static="static_lease ${INPUT_MAC_ADDRESS_PREFIX%=*}:0${i} ${prefix}.${INPUT_MAC_ADDRESS_PREFIX#*=}" + fi + + # already launched? + if grep -wq "${br}" /etc/qemu/bridge.conf; then + if [ -n "${static}" ]; then + echo "${static}" >> "${conf}" + kill_wait "$(<"${pidfile}")" + busybox udhcpd "${conf}" + fi + + return 0 + fi + echo "allow ${br}" >> /etc/qemu/bridge.conf brctl addbr "${br}" ip addr add "${prefix}.1/24" dev "${br}" @@ -231,8 +260,6 @@ _add_bridge() { local router static router="opt router ${prefix}.1" fi - if [ -n "${INPUT_MAC_ADDRESS_PREFIX}" ]; then - static="static_lease ${INPUT_MAC_ADDRESS_PREFIX%=*}:0${i} ${prefix}.${INPUT_MAC_ADDRESS_PREFIX#*=}" cat <<-EOF > "${conf}" start ${prefix}.2 end ${prefix}.254 @@ -257,7 +284,9 @@ _setup_bridges() { sysctl -w net.bridge.bridge-nf-call-iptables=0 sysctl -w net.bridge.bridge-nf-call-arptables=0 # only v4 for the moment - iptables -t nat -A POSTROUTING -s 10.0.0.0/16 -o eth0 -j MASQUERADE + if ! iptables -t nat -C POSTROUTING -s 10.0.0.0/16 -o eth0 -j MASQUERADE 2>/dev/null; then + iptables -t nat -A POSTROUTING -s 10.0.0.0/16 -o eth0 -j MASQUERADE + fi if [ -n "${INPUT_MAC_ADDRESS_PREFIX}" ]; then VIRTME_RUN_OPTS+=("--net-mac-address" "${INPUT_MAC_ADDRESS_PREFIX%=*}:00") From be18c2fe0c04634690fb939721fb6f45baffa7f1 Mon Sep 17 00:00:00 2001 From: "Matthieu Baerts (NGI0)" Date: Fri, 29 Nov 2024 20:29:25 +0100 Subject: [PATCH 12/13] prompt: add colour Because it is usually enabled by default, and helpful. Can still be turned off with INPUT_SELFTESTS_MPTCP_LIB_COLOR_FORCE=0. Signed-off-by: Matthieu Baerts (NGI0) --- entrypoint.sh | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/entrypoint.sh b/entrypoint.sh index 19a21e6..fd0e897 100755 --- a/entrypoint.sh +++ b/entrypoint.sh @@ -834,6 +834,13 @@ export KERNEL_SRC_DIR="${KERNEL_SRC}" export PATH="\${PATH}:${VIRTME_TOOLS_SBIN_DIR}" EOF + # add colours to the prompt if OK + if [ "${INPUT_SELFTESTS_MPTCP_LIB_COLOR_FORCE}" = 1 ]; then + # shellcheck disable=SC2016 # escaped on purpose + local ps1='${debian_chroot:+($debian_chroot)}\[\033[01;32m\]\u@\h\[\033[00m\]:\[\033[01;34m\]\w\[\033[00m\]\$ ' + echo "PS1='${ps1}'" >> "${BASH_PROFILE}" + fi + cat < "${VIRTME_SCRIPT}" #! /bin/bash From c4a2c65d6ea302bf2ee2483b5a232fef802289a3 Mon Sep 17 00:00:00 2001 From: "Matthieu Baerts (NGI0)" Date: Fri, 29 Nov 2024 20:29:58 +0100 Subject: [PATCH 13/13] docker: switch to Ubuntu 24.10 Mostly to have more recent toolchains and tools. Signed-off-by: Matthieu Baerts (NGI0) --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 4a8eabb..778ced3 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,5 +1,5 @@ # SPDX-License-Identifier: GPL-2.0 -FROM ubuntu:24.04 +FROM ubuntu:24.10 LABEL name=mptcp-upstream-virtme-docker