From 46910a1a56070b3eb7e9f3ad42efd6c19043cf6f Mon Sep 17 00:00:00 2001 From: Matthew Durand Date: Tue, 30 Jul 2024 11:17:53 -0700 Subject: [PATCH] Add support for Rocky8, Rocky9, Ubuntu build (#61) * Add dependency install scripts and include docs - Support CentOS, Rocky8, Rocky9 - Install all deps automatically in docker env - Update README * Use python3 explicitly * Don't install dependencies in build.sh These deps are now installed via scripts/dependencies/install.sh * Add Ubuntu dependency script * Only build RPMs if OS is centos or rocky * Dont build DSSD libucore * Add missing libboost-filesystem-dev package * Add additional includes needed for gcc 10+ - need to explicitly import string - need boost/cstdint.hpp for uint64_t * Add Ubuntu 22.04 as supported OS * Update dependencies and enhance docker build - use env var to determine docker optimizations - set env var in dockerfile - fix missing && before rm dependencies dir - set vault yum repos for centos (EOL) - remove unneeded redhat-lsb dep from cent * Install CentOS deps after sed yum vault * Streamline rhel deps - merge centos and rocky - use symlink from centos to rocky * Download sonar-scanner from env var * Determine sonar scanner dir name from zip - Sonar is not consistent so I can't derive the dir name any other way * Correct var name in comments * Fix host compile issues with GCC 10+ * Improve README - minor formatting improvements - add submodule instructions - add build script instructions * Minor formatting fix for consistency * Add docker build instructions to README * Add workdir to dockerfiles - update docs to streamline use of workdir for the sake of clarity --- README.md | 98 ++++++++++--- buildspec/sonar-scanner.yml | 13 +- host/build.sh | 12 -- host/src/include_private/nkv_framework.h | 15 +- host/src/nkv_framework.cpp | 18 ++- host/src/remote/auto_discovery.cpp | 2 +- scripts/dependencies/install.sh | 28 ++++ scripts/dependencies/os/centos.sh | 1 + scripts/dependencies/os/common.sh | 44 ++++++ scripts/dependencies/os/rocky.sh | 131 ++++++++++++++++++ scripts/dependencies/os/ubuntu.sh | 72 ++++++++++ scripts/dependencies/python/requirements.txt | 3 + scripts/docker/centos7.DOCKERFILE | 9 ++ scripts/docker/rocky8.DOCKERFILE | 7 + scripts/docker/rocky9.DOCKERFILE | 7 + scripts/docker/ubuntu22.DOCKERFILE | 7 + target/build.sh | 12 +- .../core/framework/src/dfly_lock_service.cpp | 2 + target/core/framework/src/dfly_request.cpp | 2 + target/oss/dssd-stats.patch | 6 +- target/scripts/genconfig.py | 2 +- 21 files changed, 437 insertions(+), 54 deletions(-) create mode 100755 scripts/dependencies/install.sh create mode 120000 scripts/dependencies/os/centos.sh create mode 100755 scripts/dependencies/os/common.sh create mode 100644 scripts/dependencies/os/rocky.sh create mode 100644 scripts/dependencies/os/ubuntu.sh create mode 100644 scripts/dependencies/python/requirements.txt create mode 100644 scripts/docker/centos7.DOCKERFILE create mode 100644 scripts/docker/rocky8.DOCKERFILE create mode 100644 scripts/docker/rocky9.DOCKERFILE create mode 100644 scripts/docker/ubuntu22.DOCKERFILE diff --git a/README.md b/README.md index b181289b..4ba97b15 100644 --- a/README.md +++ b/README.md @@ -2,41 +2,105 @@ dss-sdk is a sub-component of the larger project, [DSS](https://github.com/OpenMPDK/DSS). -## Major Components +## Major components + - The target component is located under the [target](target) directory. - The host component is located under the [host](host) directory. -## Build Scripts -The build scripts and README.md for them are located in the [scripts](scripts) directory. - ## Dependencies -### Operating System -CentOS 7.8 +### Supported operating systems + +dss-sdk target and host can be built using one of the following: + +- CentOS 7.8 +- Rockylinux 8 +- Rockylinux 9 +- Ubuntu 22.04 + +### Install build dependencies + +```bash +sudo ./scripts/dependencies/install.sh +``` + +### Build dss-sdk + +Prior to building dss-sdk, ensure that you have checked-out submodules: + +```bash +git clone --recurse-submodules https://github.com/OpenMPDK/dss-sdk +``` + +Alternatively: + +```bash +git clone https://github.com/OpenMPDK/dss-sdk +git submodule update --init --recursive +``` + +#### Build dss-sdk target + +```bash +./scripts/build_target.sh +``` + +#### Build dss-sdk host + +note: dss-sdk target must be built prior to building host. -### Packages ```bash -sudo yum install epel-release -sudo yum groupinstall "Development Tools" -sudo yum install bc boost-devel check cmake cmake3 CUnit-devel dejagnu dpkg elfutils-libelf-devel expect glibc-devel jemalloc-devel Judy-devel libaio-devel libcurl-devel libuuid-devel meson ncurses-devel numactl-devel openssl-devel pulseaudio-libs-devel python3 python3-devel python3-pip rdma-core-devel redhat-lsb ruby-devel snappy-devel tbb-devel wget zlib-devel +./scripts/build_host.sh kdd-samsung-remote ``` -### Python Package +#### Build dss-sdk all (target + host) + ```bash -python3 -m pip install pybind11 +./scripts/build_all.sh kdd-samsung-remote ``` -### Ruby packages +### Build dss-sdk with docker + +Required: [Install Docker Engine](https://docs.docker.com/engine/install/) in your development environment. + +dss-sdk can be built with any of the below docker base images: + +- centos:centos7.8.2003 +- rockylinux:8-minimal +- rockylinux:9-minimal +- ubuntu:22.04 + +#### Build with base image + +Example build using Ubuntu 22 base image: + ```bash -gem install dotenv:2.7.6 cabin:0.9.0 arr-pm:0.0.11 ffi:1.12.2 rchardet:1.8.0 git:1.7.0 rexml:3.2.4 backports:3.21.0 clamp:1.0.1 mustache:0.99.8 stud:0.0.23 insist:1.0.0 pleaserun:0.0.32 fpm:1.13.1 +docker run -w /dss-sdk -i -t --rm -v "$(pwd)":/dss-sdk ubuntu:22.04 /bin/bash +./scripts/dependencies/install.sh +./scripts/build_all.sh kdd-samsung-remote ``` -Note: GCC version 5.1 is required to build dss-sdk. The build script for GCC can be found in the DSS repo, in the scripts directory (https://github.com/OpenMPDK/DSS/blob/master/scripts/build_gcc.sh). +#### Create a dss-sdk build image from dockerfile + +Alternatively, you can build [one of the dockerfiles in scripts/docker](scripts/docker) to create an image with the build dependencies: + +```bash +docker build -t dss-sdk:ubuntu22-master -f scripts/docker/ubuntu22.DOCKERFILE . +``` + +#### Build dss-sdk from dockerfile image + +To build with the `dss-sdk:ubuntu22-master` image you have built, [as described above](#Create-a-dss-sdk-build-image-from-dockerfile): + +```bash +docker run -i -t --rm -v "$(pwd)":/dss-sdk dss-sdk:ubunu22-master ./scripts/build_all.sh kdd-samsung-remote +``` ## Contributing + We welcome any contributions to dss-sdk. Please fork the repository and submit a pull request with your proposed changes. Before submitting a pull request, please make sure that your changes pass the existing unit tests and that new unit tests have been added to cover any new functionality. -# DSS-SDK Pytest Framework +## DSS-SDK Pytest Framework Unit tests for testing DataMover utility and module code. Leverages pytest-cov to generate a code coverage report @@ -52,6 +116,7 @@ pytest-gitcov Also refer to requirements.txt file if you would like to install these packages with pip ## Run Pytest + You must be in the dss-sdk directory Structure: @@ -59,7 +124,6 @@ Structure: Here are some examples run from the dss-ecosystem directory - Run all tests by specifying the test folder `python3 -m pytest tests -v -rA --cov=. --cov-report term --color=yes --disable-warnings` diff --git a/buildspec/sonar-scanner.yml b/buildspec/sonar-scanner.yml index 16dc47eb..4defc2c7 100644 --- a/buildspec/sonar-scanner.yml +++ b/buildspec/sonar-scanner.yml @@ -18,16 +18,19 @@ phases: - aws s3 cp --recursive "$DSSS3URI/cache/dss-sdk/$GITHUB_RUN_NUMBER/unit/df_out/reports/" df_out/reports/ --only-show-errors # replace the old CODEBUILD_SRC_DIR with the current one in bw-output - sed -i -r "s|/codebuild/output/src[^/]+/src/github.com/OpenMPDK/dss-sdk|$CODEBUILD_SRC_DIR|g" bw-output/compile_commands.json - # Download the latest sonar-scanner + # Download sonar-scanner from SONAR_SCANNER_URL - Github-defined org var - rm -rf /sonar-scanner* - - wget --no-verbose --content-disposition -E -c "https://search.maven.org/remote_content?g=org.sonarsource.scanner.cli&a=sonar-scanner-cli&v=LATEST&c=linux&e=zip" - - unzip -q sonar-scanner-cli-*.zip -d / - - rm -f sonar-scanner-cli*.zip + - export SONAR_SCANNER_FILENAME="${SONAR_SCANNER_URL##*/}" + - wget --no-verbose --content-disposition -E -c "$SONAR_SCANNER_URL" + # Get sonar-scanner root dir filename from printed contents of zip file + - export SONAR_SCANNER_DIR=$(unzip -l "$SONAR_SCANNER_FILENAME" | awk '{print $4}' | grep '/' | cut -d '/' -f 1 | sort | uniq -c | sort -nr | head -n 1 | awk '{print $2}') + - unzip -q "$SONAR_SCANNER_FILENAME" -d / + - rm -f "$SONAR_SCANNER_FILENAME" build: commands: # Run sonar-scanner and ingest coverage report(s) - | - /sonar-scanner-*-linux/bin/sonar-scanner \ + /$SONAR_SCANNER_DIR/bin/sonar-scanner \ -Dsonar.branch.name="$([[ "$GITHUB_REF_NAME" != *"/merge" ]] && echo "$GITHUB_REF_NAME")" \ -Dsonar.host.url=https://sonarcloud.io \ -Dsonar.pullrequest.github.summary_comment=true \ diff --git a/host/build.sh b/host/build.sh index 1083c539..f1ec92eb 100755 --- a/host/build.sh +++ b/host/build.sh @@ -117,18 +117,6 @@ else exit 1 fi -# Install required package dependencies -for PACKAGE in boost-devel libcurl-devel numactl-devel tbb-devel -do - if [ "$(yum list installed | cut -f1 -d' ' | grep --extended ^${PACKAGE} -c)" -eq 1 ] - then - echo "${PACKAGE} already installed" - else - echo "sudo yum install ${PACKAGE}" - yum install ${PACKAGE} - fi -done - # Build dss-sdk host rm -rf "$OUTDIR" mkdir "$OUTDIR" diff --git a/host/src/include_private/nkv_framework.h b/host/src/include_private/nkv_framework.h index 5045bac9..7126175c 100644 --- a/host/src/include_private/nkv_framework.h +++ b/host/src/include_private/nkv_framework.h @@ -244,7 +244,8 @@ std::atomic pending_io_size; std::atomic pending_io_value; std::condition_variable cv_path; - nkv_lruCache *cnt_cache; + //nkv_lruCache *cnt_cache; + std::vector *> cnt_cache; pthread_rwlock_t lru_rw_lock; std::mutex lru_lock; std::atomic nkv_num_dc_keys; @@ -281,11 +282,17 @@ pthread_rwlock_init(&data_rw_lock_list[iter], NULL); } - listing_keys = new std::unordered_map > [nkv_listing_cache_num_shards]; + listing_keys = new std::unordered_map > (nkv_listing_cache_num_shards); if (nkv_in_memory_exec) { - data_cache = new std::unordered_map [nkv_listing_cache_num_shards]; + data_cache = new std::unordered_map (nkv_listing_cache_num_shards); + } + cnt_cache.resize(nkv_read_cache_shard_size); + for (auto i=0; i *cache_obj + = new nkv_lruCache(nkv_read_cache_size); + cnt_cache.push_back(cache_obj); } - cnt_cache = new nkv_lruCache [nkv_read_cache_shard_size](nkv_read_cache_size); nkv_num_dc_keys = 0; // ustats diff --git a/host/src/nkv_framework.cpp b/host/src/nkv_framework.cpp index 0a5f349b..39ca694b 100644 --- a/host/src/nkv_framework.cpp +++ b/host/src/nkv_framework.cpp @@ -821,7 +821,7 @@ nkv_result NKVTargetPath::do_store_io_to_path(const nkv_key* n_key, const nkv_st memcpy(c_buffer, n_value->value, n_value->length); nkv_value_wrapper nkvvalue (c_buffer, n_value->length, n_value->length); std::lock_guard lck (lru_lock); - cnt_cache[shard_id].put(key_str, std::move(nkvvalue)); + cnt_cache[shard_id]->put(key_str, std::move(nkvvalue)); } } } @@ -969,7 +969,7 @@ nkv_result NKVTargetPath::do_retrieve_io_from_path(const nkv_key* n_key, const n std::size_t key_prefix = std::hash{}(key_str); shard_id = key_prefix % nkv_read_cache_shard_size; std::lock_guard lck (lru_lock); - const nkv_value_wrapper& nkvvalue = cnt_cache[shard_id].get(key_str, cache_hit); + const nkv_value_wrapper& nkvvalue = cnt_cache[shard_id]->get(key_str, cache_hit); //nkv_value_wrapper nkvvalue = cnt_cache[shard_id].get(key_str, cache_hit); //nkv_value_wrapper* nkvvalue = cnt_cache[shard_id].get(key_str, cache_hit); if (cache_hit) { @@ -1058,7 +1058,7 @@ nkv_result NKVTargetPath::do_retrieve_io_from_path(const nkv_key* n_key, const n //nkv_value_wrapper* nkvvalue = new nkv_value_wrapper(NULL, 0, 0); smg_info(logger, "Cache put non-existence, key = %s, dev_path = %s, ip = %s", key_str.c_str(), dev_path.c_str(), path_ip.c_str()); std::lock_guard lck (lru_lock); - cnt_cache[shard_id].put(key_str, std::move(nkvvalue)); + cnt_cache[shard_id]->put(key_str, std::move(nkvvalue)); //cnt_cache[shard_id].put(key_str, nkvvalue); } } @@ -1115,7 +1115,7 @@ nkv_result NKVTargetPath::do_retrieve_io_from_path(const nkv_key* n_key, const n nkv_value_wrapper nkvvalue (c_buffer, n_value->actual_length, n_value->actual_length); smg_info(logger, "Cache put after get, key = %s, dev_path = %s, ip = %s", key_str.c_str(), dev_path.c_str(), path_ip.c_str()); std::lock_guard lck (lru_lock); - cnt_cache[shard_id].put(key_str, std::move(nkvvalue)); + cnt_cache[shard_id]->put(key_str, std::move(nkvvalue)); //cnt_cache[shard_id].put(key_str, nkvvalue); } else { smg_warn(logger, "data key = %s, length = %u, dev_path = %s, ip = %s", key_str.c_str(), n_value->actual_length, dev_path.c_str(), path_ip.c_str()); @@ -1458,7 +1458,7 @@ nkv_result NKVTargetPath::do_delete_io_from_path (const nkv_key* n_key, nkv_post int32_t shard_id = key_prefix % nkv_read_cache_shard_size; bool cache_hit = false; std::lock_guard lck (lru_lock); - cnt_cache[shard_id].del(key_str, cache_hit); + cnt_cache[shard_id]->del(key_str, cache_hit); } } } @@ -2218,10 +2218,14 @@ NKVTargetPath::~NKVTargetPath() { if (nkv_in_memory_exec) { delete[] data_cache; } - delete[] cnt_cache; + //delete[] cnt_cache; + for (int i=0; i& lines) { - ifstream fh (file_path); + std::ifstream fh (file_path); int32_t index = 1; try{ if (fh.is_open()) { diff --git a/scripts/dependencies/install.sh b/scripts/dependencies/install.sh new file mode 100755 index 00000000..928e577f --- /dev/null +++ b/scripts/dependencies/install.sh @@ -0,0 +1,28 @@ +#! /usr/bin/env bash +# shellcheck source=/dev/null +set -e + +# Path variables +SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) + +if [[ -e /etc/os-release ]]; then + source /etc/os-release +else + ID=unknown +fi + +# Default paths in case they are not exported automatically +export PATH=$PATH:/usr/local/bin:/usr/local/sbin + +for id in $ID $ID_LIKE; do + if [[ -e $SCRIPT_DIR/os/$id.sh ]]; then + echo "os: $id" + source "$SCRIPT_DIR/os/$id.sh" + source "$SCRIPT_DIR/os/common.sh" + exit 0 + fi +done + +printf "Non-supported distribution detected: %s\n" "$ID" >&2 +echo "Aborting!" +exit 1 diff --git a/scripts/dependencies/os/centos.sh b/scripts/dependencies/os/centos.sh new file mode 120000 index 00000000..59ef09c0 --- /dev/null +++ b/scripts/dependencies/os/centos.sh @@ -0,0 +1 @@ +rocky.sh \ No newline at end of file diff --git a/scripts/dependencies/os/common.sh b/scripts/dependencies/os/common.sh new file mode 100755 index 00000000..52de7d93 --- /dev/null +++ b/scripts/dependencies/os/common.sh @@ -0,0 +1,44 @@ +#! /usr/bin/env bash +set -e + +# Path variables +SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) +REQUIREMENTS=$(realpath "$SCRIPT_DIR/../python/requirements.txt") + +# Upgrade python3 pip to latest +python3 -m pip install pip --upgrade + +# Install python modules from requirements.txt +PIP_ARGS=() +PIP_ARGS+=("-r") +PIP_ARGS+=("$REQUIREMENTS") + +# Optimizations for Docker build +if [[ $DOCKER ]] +then + PIP_ARGS+=("--no-cache-dir") +fi + +# Install python modules from requirements.txt via pip +INSTALL_STRING="python3 -m pip install ${PIP_ARGS[*]}" +echo "executing command: $INSTALL_STRING" +eval "$INSTALL_STRING" + +# Set git config if not already set +for CONFIG in name email +do + if git config --list | grep "user.$CONFIG" + then + echo "git user.$CONFIG is configured." + else + echo "WARNING: git user.$CONFIG is not configured. Setting a temporary user.$CONFIG." + echo "You should set a proper git "user.$CONFIG" with command: git config --global user.$CONFIG <>" + git config --global user.$CONFIG "builder@msl.lab" + fi +done + +# Set git safe.directory globally if docker +if [[ $DOCKER ]] +then + git config --global --add safe.directory '*' +fi diff --git a/scripts/dependencies/os/rocky.sh b/scripts/dependencies/os/rocky.sh new file mode 100644 index 00000000..5f0470fb --- /dev/null +++ b/scripts/dependencies/os/rocky.sh @@ -0,0 +1,131 @@ +#! /usr/bin/env bash +set -e + +# Build Dependencies +install_build_deps() { + + # Install epel-release - must be installed first + BUILD_DEPS=() + # Install dnf if OS is rocky + if [[ $ID == 'rocky' ]]; then BUILD_DEPS+=('dnf'); fi + BUILD_DEPS+=('epel-release') + package_manager 'install' + + # Install / Enable CRB (PowerTools) on Rockylinux + if [[ $ID == 'rocky' ]]; then + BUILD_DEPS=("'dnf-command(config-manager)'") + package_manager 'install' + crb enable + fi + + # Install and enable mariadb-devel module on Rocky8 only (needed for Judy-devel) + if [[ $ROCKY_SUPPORT_PRODUCT == "Rocky-Linux-8" ]] + then + BUILD_DEPS=('mariadb-devel') + package_manager 'install' + package_manager 'module enable' + fi + + # Build dependencies + BUILD_DEPS=() + BUILD_DEPS+=('bc') + BUILD_DEPS+=('bison') + BUILD_DEPS+=('boost-devel') + BUILD_DEPS+=('check') + BUILD_DEPS+=('cmake') + BUILD_DEPS+=('cmake3') + BUILD_DEPS+=('cppunit-devel') + BUILD_DEPS+=('CUnit-devel') + BUILD_DEPS+=('dejagnu') + BUILD_DEPS+=('dpkg') + BUILD_DEPS+=('elfutils-libelf-devel') + BUILD_DEPS+=('flex') + BUILD_DEPS+=('expect') + BUILD_DEPS+=('gcc-c++') + BUILD_DEPS+=('git') + BUILD_DEPS+=('glibc-devel') + BUILD_DEPS+=('jemalloc-devel') + BUILD_DEPS+=('Judy-devel') + BUILD_DEPS+=('kernel-devel') + BUILD_DEPS+=('libaio-devel') + BUILD_DEPS+=('libcurl-devel') + BUILD_DEPS+=('libuuid-devel') + BUILD_DEPS+=('make') + BUILD_DEPS+=('meson') + BUILD_DEPS+=('ncurses-devel') + BUILD_DEPS+=('numactl-devel') + BUILD_DEPS+=('openssl-devel') + BUILD_DEPS+=('pulseaudio-libs-devel') + BUILD_DEPS+=('python3-devel') + BUILD_DEPS+=('python3-pip') + BUILD_DEPS+=('rdma-core-devel') + BUILD_DEPS+=('rpm-build') + BUILD_DEPS+=('snappy-devel') + BUILD_DEPS+=('tbb-devel') + BUILD_DEPS+=('wget') + BUILD_DEPS+=('zlib-devel') + + package_manager 'install' +} + +# Install from package mananger +package_manager() { + OPERATION=$1 + # Detect package manager tool + INSTALLER_BIN="" + + if [[ -f '/usr/bin/dnf' ]] + then + echo "using dnf" + INSTALLER_BIN='dnf' + elif [[ -f '/usr/bin/yum' ]] + then + echo "using yum" + INSTALLER_BIN='yum' + elif [[ -f '/usr/bin/microdnf' ]] + then + echo "using microdnf" + INSTALLER_BIN='microdnf' + else + # Can't find an appropriate installer + echo "can't find a valid installer" + exit 1 + fi + + # Optimizations for Docker build + if [[ $DOCKER ]] + then + if [[ $INSTALLER_BIN != 'yum' ]] + then + BUILD_DEPS+=('--nodocs') + fi + BUILD_DEPS+=('--noplugins') + BUILD_DEPS+=('--setopt=install_weak_deps=0') + fi + + INSTALL_STRING="$INSTALLER_BIN $OPERATION -y ${BUILD_DEPS[*]}" + echo "executing command: $INSTALL_STRING" + eval "$INSTALL_STRING" +} + + +install_build_deps + +# Farther cleanup if Docker environment +if [[ -f /.dockerenv ]] +then + CLEANUP_STRING="$INSTALLER_BIN clean all" + echo "executing command: $CLEANUP_STRING" + eval "$CLEANUP_STRING" + rm -rf /var/lib/dnf/history* \ + /var/lib/yum/history* \ + /var/lib/dnf/repos/* \ + /var/lib/yum/repos/* \ + /var/lib/rpm/__db* \ + /usr/share/man \ + /usr/share/doc \ + /usr/share/licenses \ + /tmp/* \ + /var/log/dnf* \ + /var/log/yum* +fi diff --git a/scripts/dependencies/os/ubuntu.sh b/scripts/dependencies/os/ubuntu.sh new file mode 100644 index 00000000..6d4dccc8 --- /dev/null +++ b/scripts/dependencies/os/ubuntu.sh @@ -0,0 +1,72 @@ +#! /usr/bin/env bash +set -e + +export DEBIAN_FRONTEND=noninteractive +export TZ=Etc/UTC + +# Build Dependencies +install_build_deps() { + BUILD_DEPS=() + package_manager 'update' + + # Build dependencies + BUILD_DEPS+=('bc') + BUILD_DEPS+=('bison') + BUILD_DEPS+=('check') + BUILD_DEPS+=('cmake') + BUILD_DEPS+=('dejagnu') + BUILD_DEPS+=('dpkg') + BUILD_DEPS+=('expect') + BUILD_DEPS+=('flex') + BUILD_DEPS+=('g++') + BUILD_DEPS+=('git') + BUILD_DEPS+=('libaio-dev') + BUILD_DEPS+=('libboost-dev') + BUILD_DEPS+=('libboost-filesystem-dev') + BUILD_DEPS+=('libc6-dev') + BUILD_DEPS+=('libcppunit-dev') + BUILD_DEPS+=('libcunit1-dev') + BUILD_DEPS+=('libcurl4-openssl-dev') + BUILD_DEPS+=('libelf-dev') + BUILD_DEPS+=('libjemalloc-dev') + BUILD_DEPS+=('libjudy-dev') + BUILD_DEPS+=('libnuma-dev') + BUILD_DEPS+=('libpulse-dev') + BUILD_DEPS+=('libpython3-dev') + BUILD_DEPS+=('librdmacm-dev') + BUILD_DEPS+=('libsnappy-dev') + BUILD_DEPS+=('libssl-dev') + BUILD_DEPS+=('libtbb-dev') + BUILD_DEPS+=('linux-headers-generic') + BUILD_DEPS+=('meson') + BUILD_DEPS+=('ncurses-dev') + BUILD_DEPS+=('python3') + BUILD_DEPS+=('python3-pip') + BUILD_DEPS+=('uuid-dev') + BUILD_DEPS+=('wget') + BUILD_DEPS+=('zlib1g-dev') + package_manager 'install' +} + +# Install from package mananger +package_manager() { + OPERATION=$1 + # Detect package manager tool + INSTALLER_BIN="" + + if [[ -f '/usr/bin/apt-get' ]] + then + echo "using apt-get" + INSTALLER_BIN='apt-get' + else + # Can't find an appropriate installer + echo "can't find a valid installer" + exit 1 + fi + + INSTALL_STRING="$INSTALLER_BIN $OPERATION -y ${BUILD_DEPS[*]}" + echo "executing command: $INSTALL_STRING" + eval "$INSTALL_STRING" +} + +install_build_deps diff --git a/scripts/dependencies/python/requirements.txt b/scripts/dependencies/python/requirements.txt new file mode 100644 index 00000000..a621db86 --- /dev/null +++ b/scripts/dependencies/python/requirements.txt @@ -0,0 +1,3 @@ +# Python dependencies +gcovr==5.0 +pybind11==2.11.1 diff --git a/scripts/docker/centos7.DOCKERFILE b/scripts/docker/centos7.DOCKERFILE new file mode 100644 index 00000000..b1815c86 --- /dev/null +++ b/scripts/docker/centos7.DOCKERFILE @@ -0,0 +1,9 @@ +FROM centos:centos7.8.2003 +ADD scripts/dependencies /dependencies +RUN set -eux \ + && sed -i s/mirror.centos.org/vault.centos.org/g /etc/yum.repos.d/*.repo \ + && sed -i s/^#.*baseurl=http/baseurl=http/g /etc/yum.repos.d/*.repo \ + && sed -i s/^mirrorlist=http/#mirrorlist=http/g /etc/yum.repos.d/*.repo \ + && DOCKER=true /dependencies/install.sh \ + && rm -rf /dependencies +WORKDIR /dss-sdk diff --git a/scripts/docker/rocky8.DOCKERFILE b/scripts/docker/rocky8.DOCKERFILE new file mode 100644 index 00000000..ed49a163 --- /dev/null +++ b/scripts/docker/rocky8.DOCKERFILE @@ -0,0 +1,7 @@ +FROM rockylinux:8-minimal + +ADD scripts/dependencies /dependencies +RUN set -eux \ + && DOCKER=true /dependencies/install.sh \ + && rm -rf /dependencies +WORKDIR /dss-sdk diff --git a/scripts/docker/rocky9.DOCKERFILE b/scripts/docker/rocky9.DOCKERFILE new file mode 100644 index 00000000..5958e087 --- /dev/null +++ b/scripts/docker/rocky9.DOCKERFILE @@ -0,0 +1,7 @@ +FROM rockylinux:9-minimal + +ADD scripts/dependencies /dependencies +RUN set -eux \ + && DOCKER=true /dependencies/install.sh \ + && rm -rf /dependencies +WORKDIR /dss-sdk diff --git a/scripts/docker/ubuntu22.DOCKERFILE b/scripts/docker/ubuntu22.DOCKERFILE new file mode 100644 index 00000000..387a3ca6 --- /dev/null +++ b/scripts/docker/ubuntu22.DOCKERFILE @@ -0,0 +1,7 @@ +FROM ubuntu:22.04 + +ADD scripts/dependencies /dependencies +RUN set -eux \ + && DOCKER=true /dependencies/install.sh \ + && rm -rf /dependencies +WORKDIR /dss-sdk diff --git a/target/build.sh b/target/build.sh index 0c9a0290..d8cd50ea 100755 --- a/target/build.sh +++ b/target/build.sh @@ -1,4 +1,5 @@ #!/usr/bin/env bash +# shellcheck source=/dev/null # shellcheck disable=SC1090 # # set -o xtrace @@ -377,10 +378,13 @@ pushd "$BUILD_DIR" cp -rf "$BUILD_DIR/nkv-target" "$RPM_BUILD_DIR/BUILD/nkv-target/usr/dss/" - generateSpecFile "$RPM_SPEC_FILE" "$packageName" "$gitVersion" - generateRPM "$RPM_SPEC_FILE" "$RPM_BUILD_DIR" || die "ERR: Failed to build RPM" - - cp "$RPM_BUILD_DIR"/RPMS/x86_64/*.rpm "$BUILD_DIR"/ + # Only build RPM for centos or rocky + source /etc/os-release + if [[ ( $ID == 'centos' ) || ( $ID == 'rocky' )]]; then + generateSpecFile "$RPM_SPEC_FILE" "$packageName" "$gitVersion" + generateRPM "$RPM_SPEC_FILE" "$RPM_BUILD_DIR" || die "ERR: Failed to build RPM" + cp "$RPM_BUILD_DIR"/RPMS/x86_64/*.rpm "$BUILD_DIR"/ + fi popd diff --git a/target/core/framework/src/dfly_lock_service.cpp b/target/core/framework/src/dfly_lock_service.cpp index b63c29e6..c8c20930 100644 --- a/target/core/framework/src/dfly_lock_service.cpp +++ b/target/core/framework/src/dfly_lock_service.cpp @@ -32,6 +32,8 @@ */ #include +#include "boost/cstdint.hpp" +#include #ifndef DSS_BUILD_CUNIT_TEST # include "dragonfly.h" diff --git a/target/core/framework/src/dfly_request.cpp b/target/core/framework/src/dfly_request.cpp index 7ad48a47..869aa8fb 100644 --- a/target/core/framework/src/dfly_request.cpp +++ b/target/core/framework/src/dfly_request.cpp @@ -35,6 +35,8 @@ #include "dragonfly.h" #include "nvmf_internal.h" #include +#include "boost/cstdint.hpp" +#include void dss_req_set_from_nvme_cmd(struct dfly_request *req) { diff --git a/target/oss/dssd-stats.patch b/target/oss/dssd-stats.patch index 38996736..26c00431 100644 --- a/target/oss/dssd-stats.patch +++ b/target/oss/dssd-stats.patch @@ -55,7 +55,7 @@ index a3e097f..ab04137 100644 if (iter_c != 0 && iter_i > 0) (void) poll(NULL, 0, iter_i * 1000); diff --git a/lib/Makefile b/lib/Makefile -index c69c3bd..f2b8a82 100644 +index c69c3bd..7b9b494 100644 --- a/lib/Makefile +++ b/lib/Makefile @@ -15,14 +15,38 @@ @@ -79,8 +79,8 @@ index c69c3bd..f2b8a82 100644 + libvmem \ + libbson \ + libustat \ -+ libutrace \ -+ libucore ++ libutrace ++ #libucore +# libvmem_malloc + +OBJDIR := ./objs diff --git a/target/scripts/genconfig.py b/target/scripts/genconfig.py index c1fca0af..74b66ae5 100755 --- a/target/scripts/genconfig.py +++ b/target/scripts/genconfig.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 import os import re