From 4f9f5e13d760e45f4633843cebd59152bb175f8f Mon Sep 17 00:00:00 2001 From: Alexander Indenbaum Date: Wed, 16 Aug 2023 13:27:12 +0000 Subject: [PATCH] Add spdk bdevperf build, container and ci test - https://spdk.io/doc/bdevperf.html - https://spdk.io/doc/nvmf_multipath_howto.html Signed-off-by: Alexander Indenbaum --- .github/workflows/build-container.yml | 25 +++++++++++++++++++++++-- Dockerfile.spdk | 14 +++++++++++++- Makefile | 2 +- docker-compose.yaml | 20 ++++++++++++++++++++ 4 files changed, 57 insertions(+), 4 deletions(-) diff --git a/.github/workflows/build-container.yml b/.github/workflows/build-container.yml index 1bb201162..9c540619d 100644 --- a/.github/workflows/build-container.yml +++ b/.github/workflows/build-container.yml @@ -17,7 +17,7 @@ jobs: submodules: recursive - name: Build container images - run: make build SVC="spdk nvmeof nvmeof-cli ceph" + run: make build SVC="spdk bdevperf nvmeof nvmeof-cli ceph" - name: Save container images run: | @@ -25,6 +25,7 @@ jobs: docker save quay.io/ceph/nvmeof:$NVMEOF_VERSION > nvmeof.tar docker save quay.io/ceph/nvmeof-cli:$NVMEOF_VERSION > nvmeof-cli.tar docker save quay.io/ceph/vstart-cluster:$CEPH_VERSION > vstart-cluster.tar + docker save bdevperf > bdevperf.tar - name: Upload container images uses: actions/upload-artifact@v3 @@ -34,6 +35,7 @@ jobs: nvmeof.tar nvmeof-cli.tar vstart-cluster.tar + bdevperf.tar - name: Build stand-alone packages (RPMs and Python wheel) id: build-standalone-packages @@ -124,7 +126,7 @@ jobs: needs: build runs-on: ubuntu-latest env: - HUGEPAGES: 256 + HUGEPAGES: 512 steps: - name: Checkout code uses: actions/checkout@v3 @@ -142,6 +144,7 @@ jobs: docker load < nvmeof.tar docker load < nvmeof-cli.tar docker load < vstart-cluster.tar + docker load < bdevperf.tar - name: Start containers run: | @@ -175,6 +178,24 @@ jobs: eval $(make alias) nvmeof-cli get_subsystems + - name: Run bdevperf + run: | + # see https://spdk.io/doc/nvmf_multipath_howto.html + . .env + echo -n "ℹ️ Starting bdevperf container" + make up SVC=bdevperf OPTS="--detach" + sleep 10 + echo "ℹ️ bdevperf start up logs" + make logs SVC=bdevperf + rpc="/usr/libexec/spdk/scripts/rpc.py" + echo "ℹ️ bdevperf bdev_nvme_set_options" + make exec SVC=bdevperf OPTS=-T CMD="$rpc -v -s /tmp/bdevperf.sock bdev_nvme_set_options -r -1" + echo "ℹ️ bdevperf tcp connect ip: $NVMEOF_IP_ADDRESS port: $NVMEOF_IO_PORT nqn: $NQN" + make exec SVC=bdevperf OPTS=-T CMD="$rpc -v -s /tmp/bdevperf.sock bdev_nvme_attach_controller -b Nvme0 -t tcp -a $NVMEOF_IP_ADDRESS -s $NVMEOF_IO_PORT -f ipv4 -n $NQN -l -1 -o 10" + echo "ℹ️ bdevperf perform_tests" + bdevperf="/usr/libexec/spdk/scripts/bdevperf.py" + make exec SVC=bdevperf OPTS=-T CMD="$bdevperf -v -t 300 -s /tmp/bdevperf.sock perform_tests" + #- name: Test mounting nvmeof device locally # run: | # . .env diff --git a/Dockerfile.spdk b/Dockerfile.spdk index 925f3302f..3b73f65a8 100644 --- a/Dockerfile.spdk +++ b/Dockerfile.spdk @@ -50,12 +50,15 @@ RUN \ MAKEFLAGS=$SPDK_MAKEFLAGS \ rpmbuild/rpm.sh $SPDK_CONFIGURE_ARGS +# build bdevperf example, will not be a part of generated rpm +RUN make -C ./examples/bdev/bdevperf + #------------------------------------------------------------------------------ FROM registry.access.redhat.com/ubi9/ubi AS rpm-export COPY --from=build /root/rpmbuild/rpm /rpm #------------------------------------------------------------------------------ -FROM registry.access.redhat.com/ubi9/ubi +FROM registry.access.redhat.com/ubi9/ubi as spdk ARG SPDK_CEPH_VERSION \ SPDK_VERSION @@ -116,3 +119,12 @@ RUN \ ENTRYPOINT [ "/usr/local/bin/nvmf_tgt" ] CMD [ "-u", "-r", "/var/tmp/spdk.sock" ] + +#------------------------------------------------------------------------------ +FROM spdk AS bdevperf + +COPY --from=build /src/build/examples/bdevperf /usr/local/bin/bdevperf +COPY --from=build /src/examples/bdev/bdevperf/bdevperf.py /usr/libexec/spdk/scripts/bdevperf.py + +ENTRYPOINT [ "/usr/local/bin/bdevperf" ] +CMD [ "-z", "-r", "/tmp/bdevperf.sock", "-q", "128", "-o", "4096", "-w", "verify", "-t", "90" ] diff --git a/Makefile b/Makefile index 491ff218a..21a1e2b2e 100644 --- a/Makefile +++ b/Makefile @@ -18,7 +18,7 @@ setup: ## Configure huge-pages (requires sudo/root password) @echo Actual Hugepages allocation: $$(cat $(HUGEPAGES_DIR)) @[ $$(cat $(HUGEPAGES_DIR)) -eq $(HUGEPAGES) ] -build push pull logs: SVC ?= spdk nvmeof nvmeof-cli ceph +build push pull logs: SVC ?= spdk bdevperf nvmeof nvmeof-cli ceph build: export NVMEOF_GIT_BRANCH != git name-rev --name-only HEAD build: export NVMEOF_GIT_COMMIT != git rev-parse HEAD diff --git a/docker-compose.yaml b/docker-compose.yaml index 948269188..3728a9d97 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -7,6 +7,7 @@ services: build: context: spdk/ dockerfile: ../Dockerfile.spdk + target: spdk args: SPDK_VERSION: SPDK_CEPH_VERSION: @@ -31,6 +32,25 @@ services: image: spdk-rpm-export build: target: rpm-export + bdevperf: + extends: + service: spdk + image: bdevperf + build: + target: bdevperf + volumes: + # sudo bash -c 'echo 1024 > /sys/devices/system/node/node0/hugepages/hugepages-2048kB/nr_hugepages' + # https://spdk.io/doc/containers.html + # TODO: Pending of https://github.com/spdk/spdk/issues/2973 + - /dev/hugepages:/dev/hugepages + - /dev/vfio/vfio:/dev/vfio/vfio + cap_add: + - SYS_ADMIN # huge-pages + - CAP_SYS_NICE # RTE + - SYS_PTRACE # gdb + ulimits: + nofile: $NVMEOF_NOFILE + memlock: -1 ceph: image: quay.io/ceph/vstart-cluster:$CEPH_CLUSTER_VERSION container_name: ceph