spdk-build as artifact

Signed-off-by: Alexander Indenbaum <[email protected]>
ceph · Aug 17, 2023 · 2c4f34a · 2c4f34a
1 parent f4f7038
commit 2c4f34a
Show file tree

Hide file tree

Showing 4 changed files with 36 additions and 12 deletions.
diff --git a/.github/workflows/build-container.yml b/.github/workflows/build-container.yml
@@ -17,14 +17,15 @@ jobs:
           submodules: recursive
 
       - name: Build container images
-        run: make build SVC="spdk nvmeof nvmeof-cli ceph"
+        run: make build SVC="spdk bdevperf nvmeof nvmeof-cli ceph"
 
       - name: Save container images
         run: |
           . .env
           docker save quay.io/ceph/nvmeof:$NVMEOF_VERSION > nvmeof.tar
           docker save quay.io/ceph/nvmeof-cli:$NVMEOF_VERSION > nvmeof-cli.tar
           docker save quay.io/ceph/vstart-cluster:$CEPH_VERSION  > vstart-cluster.tar
+          docker save bdevperf > bdevperf.tar
 
       - name: Upload container images
         uses: actions/upload-artifact@v3
@@ -34,6 +35,7 @@ jobs:
             nvmeof.tar
             nvmeof-cli.tar
             vstart-cluster.tar
+            bdevperf.tar
 
       - name: Build stand-alone packages (RPMs and Python wheel)
         id: build-standalone-packages
@@ -124,7 +126,7 @@ jobs:
     needs: build
     runs-on: ubuntu-latest
     env:
-      HUGEPAGES: 256
+      HUGEPAGES: 512
     steps:
       - name: Checkout code
         uses: actions/checkout@v3
@@ -142,6 +144,7 @@ jobs:
           docker load < nvmeof.tar
           docker load < nvmeof-cli.tar
           docker load < vstart-cluster.tar
+          docker load < bdevperf.tar
 
       - name: Start containers
         run: |
@@ -175,14 +178,23 @@ jobs:
           eval $(make alias)
           nvmeof-cli get_subsystems
 
-      - name: run bdevperf
+      - name: Run bdevperf
         run: |
           # see https://spdk.io/doc/nvmf_multipath_howto.html
           . .env
-          bdevperf_container=$(DOCKER_BUILDKIT=1 COMPOSE_DOCKER_CLI_BUILD=1 docker-compose run --detach  --entrypoint=./build/examples/bdevperf spdk-build -m 0x4 -z -r /tmp/bdevperf.sock -q 128 -o 4096 -w verify -t 90)
-          docker exec  $bdevperf_container ./scripts/rpc.py -v -s /tmp/bdevperf.sock bdev_nvme_set_options -r -1
-          docker exec  $bdevperf_container ./scripts/rpc.py -v -s /tmp/bdevperf.sock bdev_nvme_attach_controller -b Nvme0 -t tcp -a $NVMEOF_IP_ADDRESS -s $NVMEOF_IO_PORT -f ipv4 -n $NQN -l -1 -o 10
-          docker exec  $bdevperf_container bash -c "PYTHONPATH=$PYTHONPATH:/root/src/spdk/python ./examples/bdev/bdevperf/bdevperf.py -t 1 -s /tmp/bdevperf.sock perform_tests"
+          echo -n "ℹ️  Starting bdevperf container"
+          make up  SVC=bdevperf OPTS="--detach"
+          sleep 10
+          echo "ℹ️  bdevperf start up logs"
+          make logs SVC=bdevperf
+          rpc="/usr/libexec/spdk/scripts/rpc.py"
+          echo "ℹ️  bdevperf bdev_nvme_set_options"
+          make exec SVC=bdevperf OPTS=-T CMD="$rpc -v -s /tmp/bdevperf.sock bdev_nvme_set_options -r -1"
+          echo "ℹ️  bdevperf tcp connect ip: $NVMEOF_IP_ADDRESS port: $NVMEOF_IO_PORT nqn: $NQN"
+          make exec SVC=bdevperf OPTS=-T CMD="$rpc -v -s /tmp/bdevperf.sock bdev_nvme_attach_controller -b Nvme0 -t tcp -a $NVMEOF_IP_ADDRESS -s $NVMEOF_IO_PORT -f ipv4 -n $NQN -l -1 -o 10"
+          echo "ℹ️  bdevperf perform_tests"
+          bdevperf="/usr/libexec/spdk/scripts/bdevperf.py"
+          make exec SVC=bdevperf OPTS=-T CMD="$bdevperf -v -t 300 -s /tmp/bdevperf.sock perform_tests"
 
       #- name: Test mounting nvmeof device locally
       #  run: |

diff --git a/Dockerfile.spdk b/Dockerfile.spdk
@@ -49,14 +49,16 @@ RUN \
     RPM_RELEASE=0 \
     MAKEFLAGS=$SPDK_MAKEFLAGS \
     rpmbuild/rpm.sh $SPDK_CONFIGURE_ARGS
+
+# build bdevperf example, will not be a part of generated rpm
 RUN make -C ./examples/bdev/bdevperf
 
 #------------------------------------------------------------------------------
 FROM registry.access.redhat.com/ubi9/ubi AS rpm-export
 COPY --from=build /root/rpmbuild/rpm /rpm
 
 #------------------------------------------------------------------------------
-FROM registry.access.redhat.com/ubi9/ubi
+FROM registry.access.redhat.com/ubi9/ubi as spdk
 
 ARG SPDK_CEPH_VERSION \
     SPDK_VERSION
@@ -117,3 +119,12 @@ RUN \
 
 ENTRYPOINT [ "/usr/local/bin/nvmf_tgt" ]
 CMD [ "-u", "-r", "/var/tmp/spdk.sock" ]
+
+#------------------------------------------------------------------------------
+FROM spdk AS bdevperf
+
+COPY --from=build /src/build/examples/bdevperf /usr/local/bin/bdevperf
+COPY --from=build /src/examples/bdev/bdevperf/bdevperf.py /usr/libexec/spdk/scripts/bdevperf.py
+
+ENTRYPOINT [ "/usr/local/bin/bdevperf" ]
+CMD [ "-z", "-r", "/tmp/bdevperf.sock", "-q", "128", "-o", "4096", "-w", "verify", "-t",  "90" ]
diff --git a/Makefile b/Makefile
@@ -18,7 +18,7 @@ setup: ## Configure huge-pages (requires sudo/root password)
 	@echo Actual Hugepages allocation: $$(cat $(HUGEPAGES_DIR))
 	@[ $$(cat $(HUGEPAGES_DIR)) -eq $(HUGEPAGES) ]
 
-build push pull logs: SVC ?= spdk nvmeof nvmeof-cli ceph
+build push pull logs: SVC ?= spdk bdevperf nvmeof nvmeof-cli ceph
 
 build: export NVMEOF_GIT_BRANCH != git name-rev --name-only HEAD
 build: export NVMEOF_GIT_COMMIT != git rev-parse HEAD

diff --git a/docker-compose.yaml b/docker-compose.yaml
@@ -7,6 +7,7 @@ services:
     build:
       context: spdk/
       dockerfile: ../Dockerfile.spdk
+      target: spdk
       args:
         SPDK_VERSION:
         SPDK_CEPH_VERSION:
@@ -31,12 +32,12 @@ services:
     image: spdk-rpm-export
     build:
       target: rpm-export
-  spdk-build:
+  bdevperf:
     extends:
       service: spdk
-    image: spdk-build
+    image: bdevperf
     build:
-      target: build
+      target: bdevperf
     volumes:
       # sudo bash -c 'echo 1024 > /sys/devices/system/node/node0/hugepages/hugepages-2048kB/nr_hugepages'
       # https://spdk.io/doc/containers.html