Add spdk bdevperf build, container and ci test

- https://spdk.io/doc/bdevperf.html - https://spdk.io/doc/nvmf_multipath_howto.html Env variables exposed by the `bdevperf` container BDEVPERF_TEST_DURATION="90" BDEVPERF_SOCKET="/tmp/bdevperf.sock" BDEVPERF_IO_QUEUE="128" BDEVPERF_IO_BS="4096" BDEVPERF_IO_TYPE="verify" Signed-off-by: Alexander Indenbaum <[email protected]>
ceph · Aug 24, 2023 · 42efb26 · 42efb26
1 parent f74509d
commit 42efb26
Show file tree

Hide file tree

Showing 4 changed files with 81 additions and 4 deletions.
diff --git a/.github/workflows/build-container.yml b/.github/workflows/build-container.yml
@@ -17,14 +17,15 @@ jobs:
           submodules: recursive
 
       - name: Build container images
-        run: make build SVC="spdk nvmeof nvmeof-cli ceph"
+        run: make build SVC="spdk bdevperf nvmeof nvmeof-cli ceph"
 
       - name: Save container images
         run: |
           . .env
           docker save quay.io/ceph/nvmeof:$NVMEOF_VERSION > nvmeof.tar
           docker save quay.io/ceph/nvmeof-cli:$NVMEOF_VERSION > nvmeof-cli.tar
           docker save quay.io/ceph/vstart-cluster:$CEPH_VERSION  > vstart-cluster.tar
+          docker save bdevperf > bdevperf.tar
 
       - name: Upload container images
         uses: actions/upload-artifact@v3
@@ -34,6 +35,7 @@ jobs:
             nvmeof.tar
             nvmeof-cli.tar
             vstart-cluster.tar
+            bdevperf.tar
 
       - name: Build stand-alone packages (RPMs and Python wheel)
         id: build-standalone-packages
@@ -124,7 +126,7 @@ jobs:
     needs: build
     runs-on: ubuntu-latest
     env:
-      HUGEPAGES: 256
+      HUGEPAGES: 512
     steps:
       - name: Checkout code
         uses: actions/checkout@v3
@@ -142,6 +144,7 @@ jobs:
           docker load < nvmeof.tar
           docker load < nvmeof-cli.tar
           docker load < vstart-cluster.tar
+          docker load < bdevperf.tar
 
       - name: Start containers
         run: |
@@ -175,6 +178,28 @@ jobs:
           eval $(make alias)
           nvmeof-cli get_subsystems
 
+      - name: Run bdevperf
+        run: |
+          # see https://spdk.io/doc/nvmf_multipath_howto.html
+          . .env
+          echo -n "ℹ️  Starting bdevperf container"
+          make up  SVC=bdevperf OPTS="--detach"
+          sleep 10
+          echo "ℹ️  bdevperf start up logs"
+          make logs SVC=bdevperf
+          eval $(make run SVC=bdevperf OPTS="--entrypoint=env" | grep BDEVPERF_SOCKET | tr -d '\n\r' )
+
+          rpc="/usr/libexec/spdk/scripts/rpc.py"
+          echo "ℹ️  bdevperf bdev_nvme_set_options"
+          make exec SVC=bdevperf OPTS=-T CMD="$rpc -v -s $BDEVPERF_SOCKET bdev_nvme_set_options -r -1"
+          echo "ℹ️  bdevperf tcp connect ip: $NVMEOF_IP_ADDRESS port: $NVMEOF_IO_PORT nqn: $NQN"
+          make exec SVC=bdevperf OPTS=-T CMD="$rpc -v -s $BDEVPERF_SOCKET bdev_nvme_attach_controller -b Nvme0 -t tcp -a $NVMEOF_IP_ADDRESS -s $NVMEOF_IO_PORT -f ipv4 -n $NQN -l -1 -o 10"
+          echo "ℹ️  bdevperf perform_tests"
+          eval $(make run SVC=bdevperf OPTS="--entrypoint=env" | grep BDEVPERF_TEST_DURATION | tr -d '\n\r' )
+          timeout=$(expr $BDEVPERF_TEST_DURATION \* 2)
+          bdevperf="/usr/libexec/spdk/scripts/bdevperf.py"
+          make exec SVC=bdevperf OPTS=-T CMD="$bdevperf -v -t $timeout -s $BDEVPERF_SOCKET perform_tests"
+
       #- name: Test mounting nvmeof device locally
       #  run: |
       #    . .env

diff --git a/Dockerfile.spdk b/Dockerfile.spdk
@@ -50,12 +50,15 @@ RUN \
     MAKEFLAGS=$SPDK_MAKEFLAGS \
     rpmbuild/rpm.sh $SPDK_CONFIGURE_ARGS
 
+# build bdevperf example, will not be a part of generated rpm
+RUN make -C ./examples/bdev/bdevperf
+
 #------------------------------------------------------------------------------
 FROM registry.access.redhat.com/ubi9/ubi AS rpm-export
 COPY --from=build /root/rpmbuild/rpm /rpm
 
 #------------------------------------------------------------------------------
-FROM registry.access.redhat.com/ubi9/ubi
+FROM registry.access.redhat.com/ubi9/ubi as spdk
 
 ARG SPDK_CEPH_VERSION \
     SPDK_VERSION
@@ -116,3 +119,32 @@ RUN \
 
 ENTRYPOINT [ "/usr/local/bin/nvmf_tgt" ]
 CMD [ "-u", "-r", "/var/tmp/spdk.sock" ]
+
+#------------------------------------------------------------------------------
+FROM spdk AS bdevperf
+
+# Default test run duration in secs
+ENV BDEVPERF_TEST_DURATION="90"
+
+# RPC socket path
+ENV BDEVPERF_SOCKET="/tmp/bdevperf.sock"
+
+# IO Queue depth
+ENV BDEVPERF_IO_QUEUE="128"
+
+# IO Block size
+ENV BDEVPERF_IO_BS="4096"
+
+# Type of I/O pattern, see https://spdk.io/doc/bdevperf.html
+# for available types
+ENV BDEVPERF_IO_TYPE="verify"
+
+COPY --from=build /src/build/examples/bdevperf /usr/local/bin/bdevperf
+COPY --from=build /src/examples/bdev/bdevperf/bdevperf.py /usr/libexec/spdk/scripts/bdevperf.py
+RUN echo "#!/bin/sh -x"                                                 > /entrypoint
+RUN echo "bdevperf -z -r \$BDEVPERF_SOCKET \\"                         >> /entrypoint
+RUN echo "         -q \$BDEVPERF_IO_QUEUE -o \$BDEVPERF_IO_BS \\"      >> /entrypoint
+RUN echo "         -w \$BDEVPERF_IO_TYPE -t \$BDEVPERF_TEST_DURATION"  >> /entrypoint
+RUN chmod 755 /entrypoint
+
+ENTRYPOINT [ "/entrypoint" ]
diff --git a/Makefile b/Makefile
@@ -18,7 +18,7 @@ setup: ## Configure huge-pages (requires sudo/root password)
 	@echo Actual Hugepages allocation: $$(cat $(HUGEPAGES_DIR))
 	@[ $$(cat $(HUGEPAGES_DIR)) -eq $(HUGEPAGES) ]
 
-build push pull logs: SVC ?= spdk nvmeof nvmeof-cli ceph
+build push pull logs: SVC ?= spdk bdevperf nvmeof nvmeof-cli ceph
 
 build: export NVMEOF_GIT_BRANCH != git name-rev --name-only HEAD
 build: export NVMEOF_GIT_COMMIT != git rev-parse HEAD

diff --git a/docker-compose.yaml b/docker-compose.yaml
@@ -7,6 +7,7 @@ services:
     build:
       context: spdk/
       dockerfile: ../Dockerfile.spdk
+      target: spdk
       args:
         SPDK_VERSION:
         SPDK_CEPH_VERSION:
@@ -31,6 +32,25 @@ services:
     image: spdk-rpm-export
     build:
       target: rpm-export
+  bdevperf:
+    extends:
+      service: spdk
+    image: bdevperf
+    build:
+      target: bdevperf
+    volumes:
+      # sudo bash -c 'echo 1024 > /sys/devices/system/node/node0/hugepages/hugepages-2048kB/nr_hugepages'
+      # https://spdk.io/doc/containers.html
+      # TODO: Pending of https://github.com/spdk/spdk/issues/2973
+      - /dev/hugepages:/dev/hugepages
+      - /dev/vfio/vfio:/dev/vfio/vfio
+    cap_add:
+      - SYS_ADMIN # huge-pages
+      - CAP_SYS_NICE # RTE
+      - SYS_PTRACE # gdb
+    ulimits:
+      nofile: $NVMEOF_NOFILE
+      memlock: -1
   ceph:
     image: quay.io/ceph/vstart-cluster:$CEPH_CLUSTER_VERSION
     container_name: ceph