fio: add iostat and keep the output files

  - that we can do more investigation with these output files

Signed-off-by: Vicente Cheng <[email protected]>

Dockerfile

@@ -1,7 +1,7 @@
 FROM ubuntu:20.04
 
-RUN apt update && apt install -y fio bash jq
+RUN apt update && apt install -y fio bash jq vim atop sysstat util-linux
 
 ADD ./fio/ /fio/
-WORKDIR ["/fio/"]
+WORKDIR /fio
 ENTRYPOINT ["bash", "/fio/run.sh"]

deploy/fio-cmp.yaml

@@ -49,15 +49,21 @@ spec:
           value: Local-Path
         - name: FIRST_VOL_FILE
           value: "/volume1/test"
+        - name: FIRST_VOL_BACKEND_DEVICE
+          value: "" # the backend device of first volume
         - name: SECOND_VOL_NAME
           value: Longhorn
         - name: SECOND_VOL_FILE
           value: "/volume2/test"
+        - name: SECOND_VOL_BACKEND_DEVICE
+          value: "" # the backend device of second volume
         - name: SIZE
           value: "30G" # must be smaller or match the PVC size
         - name: CPU_IDLE_PROF
           value: "disabled" # must be "enabled" or "disabled"
         volumeMounts:
+        - name: output
+          mountPath: output/
         - name: vol1
           mountPath: /volume1/
         - name: vol2
@@ -68,8 +74,14 @@ spec:
         #  devicePath: /volume1/test
         #- name: vol2
         #  devicePath: /volume2/test
+        securityContext:
+          privileged: true
       restartPolicy: Never
       volumes:
+      - name: output
+        hostPath:
+          path: /root/output # replace with your host path for output
+          type: DirectoryOrCreate
       - name: vol1
         persistentVolumeClaim:
           claimName: kbench-pvc-1

deploy/fio.yaml

@@ -30,6 +30,8 @@ spec:
         env:
         #- name: QUICK_MODE  # for debugging
         #  value: "1"
+        - name: LONGHORN_BACKEND_DEVICE
+          value: ""  # the backend device of Longhorn volume
         - name: FILE_NAME
           value: "/volume/test"
         - name: SIZE
@@ -42,6 +44,8 @@ spec:
         #volumeDevices:
         #- name: vol
         #  devicePath: /volume/test
+        securityContext:
+          privileged: true
       restartPolicy: Never
       volumes:
       - name: vol

fio/cmp_parse.sh

@@ -15,13 +15,13 @@ if [ -z "SECOND_VOL_NAME" ]; then
 	exit 1
 fi
 
-FIRST_IOPS=${FIRST_VOL_NAME}-iops.json
-FIRST_BW=${FIRST_VOL_NAME}-bandwidth.json
-FIRST_LAT=${FIRST_VOL_NAME}-latency.json
+FIRST_IOPS=/output/${FIRST_VOL_NAME}-iops.json
+FIRST_BW=/output/${FIRST_VOL_NAME}-bandwidth.json
+FIRST_LAT=/output/${FIRST_VOL_NAME}-latency.json
 
-SECOND_IOPS=${SECOND_VOL_NAME}-iops.json
-SECOND_BW=${SECOND_VOL_NAME}-bandwidth.json
-SECOND_LAT=${SECOND_VOL_NAME}-latency.json
+SECOND_IOPS=/output/${SECOND_VOL_NAME}-iops.json
+SECOND_BW=/output/${SECOND_VOL_NAME}-bandwidth.json
+SECOND_LAT=/output/${SECOND_VOL_NAME}-latency.json
 
 parse_iops $FIRST_IOPS
 FIRST_RAND_READ_IOPS=$RAND_READ_IOPS
@@ -71,7 +71,7 @@ SECOND_CPU_IDLE_PCT_LAT=$CPU_IDLE_PCT_LAT
 
 calc_cmp_lat
 
-RESULT=${FIRST_VOL_NAME}_vs_${SECOND_VOL_NAME}.summary
+RESULT=/output/${FIRST_VOL_NAME}_vs_${SECOND_VOL_NAME}.summary
 
 QUICK_MODE_TEXT="Quick Mode: disabled"
 if [ -n "$QUICK_MODE" ]; then

fio/cmp_run.sh

@@ -25,10 +25,14 @@ if [ -z "SECOND_VOL_FILE" ]; then
 	exit 1
 fi
 
+# clean up the previous result
+rm -rf /output/*
+
 #disable parsing in run.sh
 export SKIP_PARSE=1
 
-$CURRENT_DIR/run.sh $FIRST_VOL_FILE $FIRST_VOL_NAME
-$CURRENT_DIR/run.sh $SECOND_VOL_FILE $SECOND_VOL_NAME
+# already cleanup, skip it in run.sh
+SKIP_CLEANUP=1 $CURRENT_DIR/run.sh $FIRST_VOL_FILE $FIRST_VOL_NAME $FIRST_VOL_BACKEND_DEVICE
+SKIP_CLEANUP=1 $CURRENT_DIR/run.sh $SECOND_VOL_FILE $SECOND_VOL_NAME $SECOND_VOL_BACKEND_DEVICE
 
 $CURRENT_DIR/cmp_parse.sh

fio/parse.sh

@@ -12,9 +12,9 @@ then
 fi
 
 PREFIX=${1}
-OUTPUT_IOPS=${PREFIX}-iops.json
-OUTPUT_BW=${PREFIX}-bandwidth.json
-OUTPUT_LAT=${PREFIX}-latency.json
+OUTPUT_IOPS=/output/${PREFIX}-iops.json
+OUTPUT_BW=/output/${PREFIX}-bandwidth.json
+OUTPUT_LAT=/output/${PREFIX}-latency.json
 
 if [ ! -f "$OUTPUT_IOPS" ]; then
         echo "$OUTPUT_IOPS doesn't exist"
@@ -34,7 +34,7 @@ else
         parse_lat $OUTPUT_LAT
 fi
 
-RESULT=${1}.summary
+RESULT=/output/${1}.summary
 
 QUICK_MODE_TEXT="Quick Mode: disabled"
 if [ -n "$QUICK_MODE" ]; then

fio/run.sh

@@ -1,5 +1,6 @@
 #!/bin/bash
 
+#sleep 1000000
 set -e
 
 CURRENT_DIR="$(dirname "$(readlink -f "$0")")"
@@ -29,7 +30,15 @@ then
 fi
 echo TEST_OUTPUT_PREFIX: $OUTPUT
 
-TEST_SIZE=$3
+BACKEND_DEVICE=$3
+if [ -z "$BACKEND_DEVICE" ]; then
+        BACKEND_DEVICE=$LONGHORN_BACKEND_DEVICE
+fi
+if [ -z "$BACKEND_DEVICE" ]; then
+        BACKEND_DEVICE=""
+fi
+
+TEST_SIZE=$4
 if [ -z "$TEST_SIZE" ]; then
        TEST_SIZE=$SIZE
 fi
@@ -48,19 +57,45 @@ if [ -n "$QUICK_MODE" ]; then
         LAT_FIO="latency-quick.fio"
 fi
 
-OUTPUT_IOPS=${OUTPUT}-iops.json
-OUTPUT_BW=${OUTPUT}-bandwidth.json
-OUTPUT_LAT=${OUTPUT}-latency.json
+OUTPUT_IOPS=/output/${OUTPUT}-iops.json
+OUTPUT_IOPS_IOSTAT=/output/${OUTPUT}-iops-iostat.log
+OUTPUT_BW=/output/${OUTPUT}-bandwidth.json
+OUTPUT_BW_IOSTAT=/output/${OUTPUT}-bandwidth-iostat.log
+OUTPUT_LAT=/output/${OUTPUT}-latency.json
+OUTPUT_LAT_IOSTAT=/output/${OUTPUT}-latency-iostat.log
+
+
+# clean up the previous result
+if [ -z "$SKIP_CLEANUP" ]; then
+        rm -rf /output/*
+fi
 
 echo Benchmarking $IOPS_FIO into $OUTPUT_IOPS
+iostat -x -k -t -y 1 $BACKEND_DEVICE > $OUTPUT_IOPS_IOSTAT &
+IOSTAT_PID=$!
+
 fio $CURRENT_DIR/$IOPS_FIO $IDLE_PROF --filename=$TEST_FILE --size=$TEST_SIZE \
 	--output-format=json --output=$OUTPUT_IOPS
+echo "Benchmarking $IOPS_FIO into $OUTPUT_IOPS done, kill the iostat process (pid: $IOSTAT_PID)..."
+kill -9 $IOSTAT_PID
+
 echo Benchmarking $BW_FIO into $OUTPUT_BW
+iostat -x -k -t -y 1 $BACKEND_DEVICE > $OUTPUT_BW_IOSTAT &
+IOSTAT_PID=$!
+
 fio $CURRENT_DIR/$BW_FIO $IDLE_PROF --filename=$TEST_FILE --size=$TEST_SIZE \
 	--output-format=json --output=$OUTPUT_BW
+echo "Benchmarking $BW_FIO into $OUTPUT_BW done, kill the iostat process (pid: $IOSTAT_PID)..."
+kill -9 $IOSTAT_PID
+
 echo Benchmarking $LAT_FIO into $OUTPUT_LAT
+iostat -x -k -t -y 1 $BACKEND_DEVICE > $OUTPUT_LAT_IOSTAT &
+IOSTAT_PID=$!
+
 fio $CURRENT_DIR/$LAT_FIO $IDLE_PROF --filename=$TEST_FILE --size=$TEST_SIZE \
 	--output-format=json --output=$OUTPUT_LAT
+echo "Benchmarking $LAT_FIO into $OUTPUT_LAT done, kill the iostat process (pid: $IOSTAT_PID)..."
+kill -9 $IOSTAT_PID
 
 if [ -z "$SKIP_PARSE" ]; then
         $CURRENT_DIR/parse.sh $OUTPUT