From 8a7ed766ca7e3f0e2e9affa1ec15c0903d4014fa Mon Sep 17 00:00:00 2001 From: nsarras Date: Tue, 17 Oct 2023 15:48:27 -0700 Subject: [PATCH 1/5] Adding gen2 s3 benchmark changes --- .gitlab-ci.yml | 2 ++ README.md | 6 ++++++ dss_client/scripts/build.sh | 14 ++++++++++++++ dss_s3benchmark/dependencies.txt | 8 ++++++++ 4 files changed, 30 insertions(+) create mode 100644 dss_s3benchmark/dependencies.txt diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 3a66722..fba30f7 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -27,6 +27,8 @@ build dss_client: script: - git config --global http.sslVerify false - ./dss_client/scripts/getrdddeps.sh + # Install Package Dependencies for S3 benchmark + - yum -y install $(cat dss_s3benchmark/dependencies.txt) - build-wrapper-linux-x86-64 --out-dir bw-output/ ./dss_client/scripts/build.sh artifacts: name: dss client build diff --git a/README.md b/README.md index 44f5a04..ab51b7f 100644 --- a/README.md +++ b/README.md @@ -42,3 +42,9 @@ the read access performance of the DSS Object storage through ML model training. A customized benchmark tool based on the original [Wasabi s3-benchmark](https://github.com/wasabi-tech/s3-benchmark) tool. [DSS s3-benchmark README](./dss_s3benchmark/README.md) + +#### DSS s3-benchmark dependencies + +Make sure bzip2-devel package is not installed +Make sure following RPMs are installed CUnit(-devel), libzstd, golang, devtoolset-11 packages. Devtoolset-11 requires centos-release-scl package, aws-sdk-cpp-1.9 package (can also install artifact version of aws sdk) +Dependencies are listed in `dss_s3benchmark/dependencies.txt` diff --git a/dss_client/scripts/build.sh b/dss_client/scripts/build.sh index 96353d4..3158ea1 100755 --- a/dss_client/scripts/build.sh +++ b/dss_client/scripts/build.sh @@ -36,6 +36,8 @@ set -e SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) DSS_CLIENT_DIR=$(realpath "$SCRIPT_DIR/..") DSS_ECOSYSTEM_DIR=$(realpath "$DSS_CLIENT_DIR/..") +DSS_S3BENCHMARK_DIR="$DSS_ECOSYSTEM_DIR/dss_s3benchmark" +S3BENCHPATH="$DSS_S3BENCHMARK_DIR/s3-benchmark" TOP_DIR="$DSS_ECOSYSTEM_DIR/.." BUILD_DIR="$DSS_CLIENT_DIR/build" STAGING_DIR="$BUILD_DIR/staging" @@ -95,6 +97,15 @@ pushd "$BUILD_DIR" make -j popd +# Build s3 benchmark +pushd "$DSS_S3BENCHMARK_DIR" + bash scripts/build.sh + if [ ! -f "$S3BENCHPATH"] + then + die "S3 benchmark did not build properly" + fi +popd + # Create dss_client staging dir mkdir -p "$STAGING_DIR" @@ -105,6 +116,9 @@ cp "$BUILD_DIR"/test_dss "$STAGING_DIR" # Copy Client benchmark directory to staging directory cp -r "$DSS_CLIENT_DIR/benchmark" "$STAGING_DIR" +# Copy s3 benchmark directory to staging directory +cp "$S3BENCHPATH" "$STAGING_DIR" + # Copy Client data integrity test to staging directory mkdir "$STAGING_DIR/test" cp "$DSS_CLIENT_DIR/test/test_data_integrity.py" "$STAGING_DIR/test" diff --git a/dss_s3benchmark/dependencies.txt b/dss_s3benchmark/dependencies.txt new file mode 100644 index 0000000..71dfa8c --- /dev/null +++ b/dss_s3benchmark/dependencies.txt @@ -0,0 +1,8 @@ +centos-release-scl +devtoolset-11 +bzip2 +bzip2-libs +CUnit +CUnit-devel +libzstd +golang \ No newline at end of file From 7c19943ed46b660ba7bc3c3da111e86a2e4528c3 Mon Sep 17 00:00:00 2001 From: nsarras Date: Fri, 27 Oct 2023 11:34:01 -0700 Subject: [PATCH 2/5] Resolving dss client build script error --- dss_client/scripts/build.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/dss_client/scripts/build.sh b/dss_client/scripts/build.sh index 3158ea1..a56b367 100755 --- a/dss_client/scripts/build.sh +++ b/dss_client/scripts/build.sh @@ -99,8 +99,8 @@ popd # Build s3 benchmark pushd "$DSS_S3BENCHMARK_DIR" - bash scripts/build.sh - if [ ! -f "$S3BENCHPATH"] + scripts/build.sh + if [ ! -f "$S3BENCHPATH" ] then die "S3 benchmark did not build properly" fi From a3fff8f701764137cb9c6489b4897509f10fde6b Mon Sep 17 00:00:00 2001 From: Matthew Durand Date: Thu, 2 Nov 2023 14:32:44 -0700 Subject: [PATCH 3/5] Install s3-bench deps before client build --- buildspec/dss-client.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/buildspec/dss-client.yml b/buildspec/dss-client.yml index 01a0e9c..e576606 100644 --- a/buildspec/dss-client.yml +++ b/buildspec/dss-client.yml @@ -13,6 +13,8 @@ phases: commands: # Download and extract RDD libs from prior dss-sdk target artifact for this branch - ./dss_client/scripts/getrdddeps.sh + # Install Package Dependencies for S3 benchmark + - yum -y install $(cat dss_s3benchmark/dependencies.txt) build: commands: # Build dss-client with Sonar build-wrapper for C/C++ static analysis From 7cb37daf4f30a46f5e030c465e49ca74d506065e Mon Sep 17 00:00:00 2001 From: nsarras Date: Mon, 6 Nov 2023 17:22:32 -0800 Subject: [PATCH 4/5] Updating README with s3benchmark build instructions --- README.md | 6 ----- dss_s3benchmark/README.md | 50 ++++++++++++++++++++++++++++++--------- 2 files changed, 39 insertions(+), 17 deletions(-) diff --git a/README.md b/README.md index ab51b7f..44f5a04 100644 --- a/README.md +++ b/README.md @@ -42,9 +42,3 @@ the read access performance of the DSS Object storage through ML model training. A customized benchmark tool based on the original [Wasabi s3-benchmark](https://github.com/wasabi-tech/s3-benchmark) tool. [DSS s3-benchmark README](./dss_s3benchmark/README.md) - -#### DSS s3-benchmark dependencies - -Make sure bzip2-devel package is not installed -Make sure following RPMs are installed CUnit(-devel), libzstd, golang, devtoolset-11 packages. Devtoolset-11 requires centos-release-scl package, aws-sdk-cpp-1.9 package (can also install artifact version of aws sdk) -Dependencies are listed in `dss_s3benchmark/dependencies.txt` diff --git a/dss_s3benchmark/README.md b/dss_s3benchmark/README.md index 84aaef6..0c431f8 100644 --- a/dss_s3benchmark/README.md +++ b/dss_s3benchmark/README.md @@ -11,7 +11,35 @@ To leverage this tool, the following prerequisites apply: * Access to the appropriate AWS EC2 (or equivalent) compute resource (optimal performance is realized using m4.10xlarge EC2 Ubuntu with 10 GB ENA) * Dependency: Download github.com/openMPDK/dss-sdk repo and compile +Make sure bzip2-devel package is not installed +Make sure following RPMs are installed CUnit(-devel), libzstd, golang, devtoolset-11 packages. Devtoolset-11 requires centos-release-scl package, aws-sdk-cpp-1.9 package (can also install artifact version of aws sdk) +Dependencies are listed in `dss_s3benchmark/dependencies.txt` + + # Building the Program + +## To leverage build script +clone top level DSS parent repository and run the dss_s3benchmark build script + +``` +git clone --recursive https://github.com/OpenMPDK/DSS.git +cd DSS/dss-sdk/scripts/ +bash build_all.sh kdd-samsung-remote +cd ../../dss-ecosystem/dss_client/scripts +bash build.sh +cd ../../dss_s3benchmark/ +bash scripts/build.sh +``` + +After building and attempting to run the binary, you may receive an error if libdss.so is not properly +installed on the same machine because s3 benchmark depends on the dss client library which is dependent +on the aws-sdk v1.9 package. + +To resolve this a runtime issue of missing libdss.so, +you can create a ```/etc/ld.so.conf.d/s3-benchmark.conf``` +and add the path of the libdss.so file to that and run ldconfig + +## To build from scratch Obtain a local copy of the repository using the following git command with any directory that is convenient: ``` @@ -89,46 +117,46 @@ Loop 1: GET time 60.1 secs, objects = 5483, speed = 91.3MB/sec, 91.3 operations/ Loop 1: DELETE time 1.9 secs, 2923.4 deletes/sec. Benchmark completed. -./s3-benchmark -a minio -b som4 -s minio123 -u http://10.1.51.21:9000 -t 100 -z 10M -n 100 -o 1 +./s3-benchmark -a minio -b som4 -s minio123 -u -t 100 -z 10M -n 100 -o 1 Wasabi benchmark program v2.0 -Parameters: url=http://10.1.51.21:9000, bucket=som4, region=us-east-1, duration=60, threads=100, num_ios=100, op_type=1, loops=1, size=10M +Parameters: url=, bucket=som4, region=us-east-1, duration=60, threads=100, num_ios=100, op_type=1, loops=1, size=10M 2021/01/05 18:22:52 WARNING: createBucket som4 error, ignoring BucketAlreadyOwnedByYou: Your previous request to create the named bucket succeeded and you already own it. status code: 409, request id: 1657835058733E40, host id: Loop 1: PUT time 38.9 secs, objects = 10000, speed = 2.5GB/sec, 257.1 operations/sec. Slowdowns = 0 -./s3-benchmark -a minio -b som4 -s minio123 -u http://10.1.51.21:9000 -t 100 -z 10M -n 100 -o 2 +./s3-benchmark -a minio -b som4 -s minio123 -u -t 100 -z 10M -n 100 -o 2 Wasabi benchmark program v2.0 -Parameters: url=http://10.1.51.21:9000, bucket=som4, region=us-east-1, duration=60, threads=100, num_ios=100, op_type=2, loops=1, size=10M +Parameters: url=, bucket=som4, region=us-east-1, duration=60, threads=100, num_ios=100, op_type=2, loops=1, size=10M 2021/01/05 18:23:39 WARNING: createBucket som4 error, ignoring BucketAlreadyOwnedByYou: Your previous request to create the named bucket succeeded and you already own it. status code: 409, request id: 1657835B38D61A94, host id: Loop 1: GET time 14.9 secs, objects = 10000, speed = 6.6GB/sec, 672.1 operations/sec. Slowdowns = 0 -./s3-benchmark -a minio -b som4 -s minio123 -u http://10.1.51.21:9000 -t 100 -z 10M -n 100 -o 3 +./s3-benchmark -a minio -b som4 -s minio123 -u -t 100 -z 10M -n 100 -o 3 Wasabi benchmark program v2.0 -Parameters: url=http://10.1.51.21:9000, bucket=som4, region=us-east-1, duration=60, threads=100, num_ios=100, op_type=3, loops=1, size=10M +Parameters: url=, bucket=som4, region=us-east-1, duration=60, threads=100, num_ios=100, op_type=3, loops=1, size=10M 2021/01/05 18:24:04 WARNING: createBucket som4 error, ignoring BucketAlreadyOwnedByYou: Your previous request to create the named bucket succeeded and you already own it. status code: 409, request id: 16578360FD53602A, host id: Loop 1: DELETE time 4.3 secs, 2342.4 deletes/sec. Slowdowns = 0 //Prefix based run -root@msl-ssg-si04 s3-bench]./s3-benchmark -a minio -b som10 -s minio123 -u http://10.1.51.21:9000 -t 100 -z 1M -n 100 -o 1 -p samsung-s3-bench +root@msl-ssg-si04 s3-bench]./s3-benchmark -a minio -b som10 -s minio123 -u -t 100 -z 1M -n 100 -o 1 -p samsung-s3-bench Wasabi benchmark program v2.0 -Parameters: url=http://10.1.51.21:9000, bucket=som10, region=us-east-1, duration=60, threads=100, num_ios=100, op_type=1, loops=1, size=1M +Parameters: url=, bucket=som10, region=us-east-1, duration=60, threads=100, num_ios=100, op_type=1, loops=1, size=1M 2021/02/17 17:08:12 WARNING: createBucket som10 error, ignoring BucketAlreadyOwnedByYou: Your previous request to create the named bucket succeeded and you already own it. status code: 409, request id: 1664B231AFA7F9A4, host id: Loop 1: PUT time 4.6 secs, objects = 10000, speed = 2.1GB/sec, 2157.8 operations/sec. Slowdowns = 0 //Time based read -root@msl-ssg-si04 s3-bench]./s3-benchmark -a minio -b som10 -s minio123 -u http://10.1.51.21:9000 -t 100 -z 1M -d 30 -c 100 -o 2 -p samsung-s3-bench +root@msl-ssg-si04 s3-bench]./s3-benchmark -a minio -b som10 -s minio123 -u -t 100 -z 1M -d 30 -c 100 -o 2 -p samsung-s3-bench Wasabi benchmark program v2.0 -Parameters: url=http://10.1.51.21:9000, bucket=som10, region=us-east-1, duration=30, threads=100, num_ios=0, op_type=2, loops=1, size=1M +Parameters: url=, bucket=som10, region=us-east-1, duration=30, threads=100, num_ios=0, op_type=2, loops=1, size=1M 2021/02/17 17:06:55 WARNING: createBucket som10 error, ignoring BucketAlreadyOwnedByYou: Your previous request to create the named bucket succeeded and you already own it. status code: 409, request id: 1664B21FB5A67930, host id: Loop 1: GET time 30.0 secs, objects = 201106, speed = 6.5GB/sec, 6700.2 operations/sec. Slowdowns = 0 //Gen2 RDD way -./s3-benchmark -a minio -b rddtest207 -s minio123 -u http://208.0.0.33:9000 -t 100 -z 1M -c 1000 -d 20 -o 2 -p dss-gen2-test-207 -rdd 1 -rdd_ips 207.0.0.30,207.0.0.31,207.0.0.32,207.0.0.33 -rdd_port 1234 -instance_id 3 +./s3-benchmark -a minio -b rddtest207 -s minio123 -u -t 100 -z 1M -c 1000 -d 20 -o 2 -p dss-gen2-test-207 -rdd 1 -rdd_ips ,,, -rdd_port 1234 -instance_id 3 ``` From ac437dc92625e8cf1daccc21d432a46f1b42e34d Mon Sep 17 00:00:00 2001 From: "matt.d" Date: Tue, 7 Nov 2023 16:40:52 -0800 Subject: [PATCH 5/5] remove yum steps from build --- .gitlab-ci.yml | 2 -- buildspec/dss-client.yml | 2 -- 2 files changed, 4 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index fba30f7..3a66722 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -27,8 +27,6 @@ build dss_client: script: - git config --global http.sslVerify false - ./dss_client/scripts/getrdddeps.sh - # Install Package Dependencies for S3 benchmark - - yum -y install $(cat dss_s3benchmark/dependencies.txt) - build-wrapper-linux-x86-64 --out-dir bw-output/ ./dss_client/scripts/build.sh artifacts: name: dss client build diff --git a/buildspec/dss-client.yml b/buildspec/dss-client.yml index e576606..01a0e9c 100644 --- a/buildspec/dss-client.yml +++ b/buildspec/dss-client.yml @@ -13,8 +13,6 @@ phases: commands: # Download and extract RDD libs from prior dss-sdk target artifact for this branch - ./dss_client/scripts/getrdddeps.sh - # Install Package Dependencies for S3 benchmark - - yum -y install $(cat dss_s3benchmark/dependencies.txt) build: commands: # Build dss-client with Sonar build-wrapper for C/C++ static analysis