Merge pull request #90 from epuertat/build-container

build: containerize deployment

Reviewed-by: Alexander Indenbaum <[email protected]>
Reviewed-by: Orit Wasserman <[email protected]>

.dockerignore

@@ -0,0 +1,9 @@
+**
+# Exclude everything except:
+!control/*.py
+!proto/*.proto
+!pyproject.toml
+!pdm.lock
+!pdm.toml
+!README.md
+!LICENSE

.env

@@ -0,0 +1,59 @@
+# Globals
+VERSION="0.0.1"
+CEPH_VERSION="17.2.6"
+SPDK_VERSION="23.01"
+MAINTAINER="Ceph Developers <[email protected]>"
+
+# Performance
+NVMEOF_NOFILE=20480 # Max number of open files (depends on number of hosts connected)
+HUGEPAGES=1024 # 2 GB
+HUGEPAGES_DIR="/sys/kernel/mm/hugepages/hugepages-2048kB/nr_hugepages"
+
+# NVMe-oF
+NVMEOF_VERSION="${VERSION}"
+NVMEOF_CONFIG="./ceph-nvmeof.conf"
+NVMEOF_SPDK_VERSION="${SPDK_VERSION}"
+NVMEOF_NAME="ceph-nvmeof"
+NVMEOF_SUMMARY="Ceph NVMe over Fabrics Gateway"
+NVMEOF_DESCRIPTION="Service to provide block storage on top of Ceph for platforms (e.g.: VMWare) without native Ceph support (RBD), replacing existing approaches (iSCSI) with a newer and more versatile standard (NVMe-oF)."
+NVMEOF_URL="https://github.com/ceph/ceph-nvmeof"
+NVMEOF_TAGS="ceph,nvme-of,nvme-of gateway,rbd,block storage"
+NVMEOF_WANTS="ceph,rbd"
+NVMEOF_IP_ADDRESS="192.168.13.3"
+NVMEOF_IO_PORT=4420
+NVMEOF_GW_PORT=5500
+NVMEOF_DISC_PORT=8009
+NVMEOF_EXPOSE_SERVICES="${NVMEOF_IO_PORT}/tcp:nvme,${NVMEOF_GW_PORT}/tcp:grpc,${NVMEOF_DISC_PORT}/tcp:nvme-disc"
+NVMEOF_GIT_REPO="https://github.com/ceph/ceph-nvmeof.git"
+
+# NVMe-oF CLI
+MVMEOF_CLI_VERSION="${VERSION}"
+NVMEOF_CLI_NAME="ceph-nvmeof-cli"
+NVMEOF_CLI_SUMMARY="Ceph NVMe over Fabrics CLI"
+NVMEOF_CLI_DESCRIPTION="Command line interface for Ceph NVMe over Fabrics Gateway"
+
+# SPDK
+SPDK_CEPH_VERSION="${CEPH_VERSION}"
+SPDK_NAME="SPDK"
+SPDK_SUMMARY="Build Ultra High-Performance Storage Applications with the Storage Performance Development Kit"
+SPDK_DESCRIPTION="The Storage Performance Development Kit (SPDK) provides a set of tools and libraries for writing high performance, scalable, user-mode storage applications"
+SPDK_URL="https://spdk.io"
+
+SPDK_PKGDEP_ARGS="--rbd"
+SPDK_CONFIGURE_ARGS="--with-rbd --disable-tests --disable-unit-tests --disable-examples"
+SPDK_MAKEFLAGS=
+SPDK_CENTOS_BASE="https://mirror.stream.centos.org/9-stream/BaseOS/x86_64/os/Packages/"
+SPDK_CENTOS_REPO_VER="9.0-21.el9"
+
+# Ceph Cluster
+CEPH_CLUSTER_VERSION="${CEPH_VERSION}"
+CEPH_VSTART_ARGS="--without-dashboard --memstore"
+
+# Demo settings
+RBD_POOL="rbd"
+RBD_IMAGE_NAME="demo_image"
+RBD_IMAGE_SIZE="10M"
+BDEV_NAME="demo_bdev"
+NQN="nqn.2016-06.io.spdk:cnode1"
+SERIAL="SPDK00000000000001"
+NVMEOF_FIRST_GATEWAY_NAME="gw-1"

.github/workflows/build-container.yml

@@ -0,0 +1,91 @@
+name: "CI"
+on:
+  push:
+    branches:
+      - devel
+  pull_request:
+    branches:
+      - devel
+  schedule:
+    - cron: '0 0 * * *'
+  workflow_dispatch:
+env:
+  HUGEPAGES: 256
+
+jobs:
+  build-and-test:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v3
+        with:
+          submodules: recursive
+
+      - name: Build container images
+        run: make build
+
+      - name: Setup huge-pages
+        run: make setup HUGEPAGES=$HUGEPAGES
+
+      - name: Start containers
+        run: |
+          make up OPTS=--detach || (make logs OPTS=''; exit 1)
+
+      - name: Wait for the Gateway to be listening
+        timeout-minutes: 1
+        run: |
+          . .env
+          HOST_PORT=$(make -s port OPTS="--index=1" CMD="nvmeof $NVMEOF_GW_PORT" | tr ":" " ")
+          until nc -z $HOST_PORT; do
+            echo -n .
+            sleep 1
+          done
+          echo
+
+      - name: List containers
+        run: make ps
+
+      - name: List processes
+        run: make top
+
+      - name: Test
+        run: |
+          . .env
+          make demo SERVER_ADDRESS=$NVMEOF_IP_ADDRESS OPTS=-T # Disable TTY
+
+      - name: Get subsystems
+        run: |
+          # https://github.com/actions/toolkit/issues/766
+          shopt -s expand_aliases
+          eval $(make alias)
+          nvmeof-cli get_subsystems
+
+      #- name: Test mounting nvmeof device locally
+      #  run: |
+      #    . .env
+      #    sudo modprobe nvme-fabrics
+      #    sudo nvme list
+      #    sudo nvme discover -t tcp -a $NVMEOF_IP_ADDRESS -s $NVMEOF_IO_PORT
+      #    sudo nvme connect -t tcp --traddr $NVMEOF_IP_ADDRESS -s $NVMEOF_IO_PORT -n nqn.2016-06.io.spdk:cnode1
+      #    sudo nvme list
+      #    NVMEOF_DEVICE=$(sudo nvme list -o json | jq '.Devices[] | select(.ModelNumber=="SPDK bdev Controller").DevicePath')
+      #    sudo mkfs $NVMEOF_DEVICE
+      #    MOUNT_POINT=$(mktemp -d)
+      #    sudo mount $NVMEOF_DEVICE $MOUNT_POINT
+      #    cd $MOUNT_POINT
+      #    touch test
+
+      # For debugging purposes (provides an SSH connection to the runner)
+      #- name: Setup tmate session
+      #  uses: mxschmitt/action-tmate@v3
+      #  with:
+      #    limit-access-to-actor: true
+
+      - name: Display logs
+        run: make logs OPTS=''
+
+      - name: Shut containers down
+        run: make down
+
+      - name: Clean up environment
+        run: make clean

.gitignore

@@ -1,8 +1,8 @@
-
+*.swp
+*_pb2*.py*
 __pycache__
-spdk
-control/generated/gateway_pb2_grpc.py
-control/generated/gateway_pb2.py
+__pypackages__
+.pdm-python
 server.crt
 server.key
 client.crt

Dockerfile

@@ -0,0 +1,125 @@
+# syntax = docker/dockerfile:1.4
+
+ARG NVMEOF_SPDK_VERSION \
+    NVMEOF_TARGET  # either 'gateway' or 'cli'
+
+#------------------------------------------------------------------------------
+# Base image for NVMEOF_TARGET=cli (nvmeof-cli)
+FROM registry.access.redhat.com/ubi9/ubi AS base-cli
+ENV GRPC_DNS_RESOLVER=native
+ENTRYPOINT ["python3", "-m", "control.cli"]
+CMD []
+
+#------------------------------------------------------------------------------
+# Base image for NVMEOF_TARGET=gateway (nvmeof-gateway)
+FROM quay.io/ceph/spdk:${NVMEOF_SPDK_VERSION:-NULL} AS base-gateway
+RUN \
+    --mount=type=cache,target=/var/cache/dnf \
+    --mount=type=cache,target=/var/lib/dnf \
+    dnf install -y python3-rados
+ENTRYPOINT ["python3", "-m", "control"]
+CMD ["-c", "/src/ceph-nvmeof.conf"]
+
+#------------------------------------------------------------------------------
+# Intermediate layer for Python set-up
+FROM base-$NVMEOF_TARGET AS python-intermediate
+
+RUN \
+    --mount=type=cache,target=/var/cache/dnf \
+    --mount=type=cache,target=/var/lib/dnf \
+    dnf update -y
+
+ENV PYTHONUNBUFFERED=1 \
+    PYTHONIOENCODING=UTF-8 \
+    LC_ALL=C.UTF-8 \
+    LANG=C.UTF-8 \
+    PIP_NO_CACHE_DIR=off \
+    PYTHON_MAJOR=3 \
+    PYTHON_MINOR=9 \
+    PDM_ONLY_BINARY=:all:
+
+ARG APPDIR=/src
+
+ARG NVMEOF_NAME \
+    NVMEOF_SUMMARY \
+    NVMEOF_DESCRIPTION \
+    NVMEOF_URL \
+    NVMEOF_VERSION \
+    NVMEOF_MAINTAINER \
+    NVMEOF_TAGS \
+    NVMEOF_WANTS \
+    NVMEOF_EXPOSE_SERVICES \
+    BUILD_DATE \
+    NVMEOF_GIT_REPO \
+    NVMEOF_GIT_BRANCH \
+    NVMEOF_GIT_COMMIT
+
+# Generic labels
+LABEL name="$NVMEOF_NAME" \
+      version="$NVMEOF_VERSION" \
+      summary="$NVMEOF_SUMMARY" \
+      description="$NVMEOF_DESCRIPTION" \
+      maintainer="$NVMEOF_MAINTAINER" \
+      release="" \
+      url="$NVMEOF_URL" \
+      build-date="$BUILD_DATE" \
+      vcs-ref="$NVMEOF_GIT_COMMIT"
+
+# k8s-specific labels
+LABEL io.k8s.display-name="$NVMEOF_SUMMARY" \
+      io.k8s.description="$NVMEOF_DESCRIPTION"
+
+# k8s-specific labels
+LABEL io.openshift.tags="$NVMEOF_TAGS" \
+      io.openshift.wants="$NVMEOF_WANTS" \
+      io.openshift.expose-services="$NVMEOF_EXPOSE_SERVICES"
+
+# Ceph-specific labels
+LABEL io.ceph.component="$NVMEOF_NAME" \
+      io.ceph.summary="$NVMEOF_SUMMARY" \
+      io.ceph.description="$NVMEOF_DESCRIPTION" \
+      io.ceph.url="$NVMEOF_URL" \
+      io.ceph.version="$NVMEOF_VERSION" \
+      io.ceph.maintainer="$NVMEOF_MAINTAINER" \
+      io.ceph.git.repo="$NVMEOF_GIT_REPO" \
+      io.ceph.git.branch="$NVMEOF_GIT_BRANCH" \
+      io.ceph.git.commit="$NVMEOF_GIT_COMMIT"
+
+ENV PYTHONPATH=$APPDIR/proto:$APPDIR/__pypackages__/$PYTHON_MAJOR.$PYTHON_MINOR/lib
+
+WORKDIR $APPDIR
+
+#------------------------------------------------------------------------------
+FROM python-intermediate AS builder-base
+ARG PDM_VERSION=2.7.4 \
+    PDM_INSTALL_CMD=sync \
+    PDM_INSTALL_FLAGS="-v --no-isolation --no-self --no-editable"
+ENV PDM_INSTALL_FLAGS=$PDM_INSTALL_FLAGS
+
+# https://pdm.fming.dev/latest/usage/advanced/#use-pdm-in-a-multi-stage-dockerfile
+RUN \
+    --mount=type=cache,target=/var/cache/dnf \
+    --mount=type=cache,target=/var/lib/dnf \
+    dnf install -y python3-pip
+RUN \
+    --mount=type=cache,target=/root/.cache/pip \
+    pip install -U pip setuptools
+
+RUN \
+    --mount=type=cache,target=/root/.cache/pip \
+    pip install pdm==$PDM_VERSION
+
+#------------------------------------------------------------------------------
+FROM builder-base AS builder
+
+COPY pyproject.toml pdm.lock pdm.toml ./
+RUN \
+    --mount=type=cache,target=/root/.cache/pdm \
+    pdm "$PDM_INSTALL_CMD" $PDM_INSTALL_FLAGS
+
+COPY . .
+RUN pdm run protoc
+
+#------------------------------------------------------------------------------
+FROM python-intermediate
+COPY --from=builder $APPDIR .

Dockerfile.ceph

@@ -0,0 +1,96 @@
+# syntax = docker/dockerfile:1.4
+FROM quay.io/centos/centos:stream9-minimal AS build
+
+ARG CEPH_CLUSTER_VERSION
+
+COPY <<EOF /etc/yum.repos.d/ceph.repo
+[Ceph]
+name=Ceph packages for \$basearch
+baseurl=https://download.ceph.com/rpm-$CEPH_CLUSTER_VERSION/el\$releasever/\$basearch
+enabled=1
+priority=2
+gpgcheck=1
+gpgkey=https://download.ceph.com/keys/release.asc
+
+[Ceph-noarch]
+name=Ceph noarch packages
+baseurl=https://download.ceph.com/rpm-$CEPH_CLUSTER_VERSION/el\$releasever/noarch
+enabled=1
+priority=2
+gpgcheck=1
+gpgkey=https://download.ceph.com/keys/release.asc
+EOF
+
+ARG CEPH_PACKAGES="\
+    ceph-common \
+    ceph-mon \
+    ceph-osd \
+    ceph-mds \
+    ceph-mgr \
+    ceph-radosgw \
+    ceph-exporter \
+    hostname \
+    jq \
+    net-tools \
+    "
+
+RUN rpm -vih https://dl.fedoraproject.org/pub/epel/epel-release-latest-9.noarch.rpm
+RUN rpm --import 'https://download.ceph.com/keys/release.asc'
+RUN cd /etc/yum.repos.d/ && curl -O https://copr.fedorainfracloud.org/coprs/ceph/el9/repo/epel-9/ceph-el9-epel-9.repo
+
+## WORKAROUND: remove when https://tracker.ceph.com/issues/61882 is fixed
+RUN rpm -vih "https://buildlogs.centos.org/centos/9-stream/storage/x86_64/ceph-quincy/Packages/t/thrift-0.14.0-7.el9s.x86_64.rpm"
+
+RUN \
+    --mount=type=cache,target=/var/cache/microdnf \
+    microdnf install -y \
+        --enablerepo crb \
+        --nobest \
+        --nodocs \
+        --setopt=install_weak_deps=0 \
+        --setopt=keepcache=1 \
+        --setopt=cachedir=/var/cache/microdnf \
+        $CEPH_PACKAGES
+
+#------------------------------------------------------------------------------
+FROM build
+
+LABEL maintainer \
+      ceph=True \
+      RELEASE \
+      GIT_REPO \
+      GIT_BRANCH \
+      GIT_COMMIT
+
+ENV MON=1 \
+    MGR=1 \
+    OSD=3 \
+    MDS=0 \
+    FS=0 \
+    RGW=0 \
+    NFS=0 \
+    CEPH_PORT=10000 \
+    CEPH_VSTART_ARGS="--without-dashboard"
+
+ENV CEPH_BIN=/usr/bin \
+    CEPH_LIB=/usr/lib64/ceph \
+    CEPH_CONF_PATH=/etc/ceph \
+    EC_PATH=/usr/lib64/ceph/erasure-code \
+    OBJCLASS_PATH=/usr/lib64/rados-classes \
+    MGR_PYTHON_PATH=/usr/share/ceph/mgr \
+    PYBIND=/usr/share/ceph/mgr
+
+VOLUME $CEPH_CONF_PATH
+RUN chown ceph:ceph $CEPH_CONF_PATH
+
+RUN ln -sf $EC_PATH/* $CEPH_LIB && \
+    ln -sf $OBJCLASS_PATH/* $CEPH_LIB && \
+    ln -sf $CEPH_LIB/compressor/* $CEPH_LIB
+
+USER ceph
+WORKDIR /ceph
+ADD --chown=ceph:ceph --chmod=755 https://raw.githubusercontent.com/ceph/ceph/v$CEPH_CLUSTER_VERSION/src/vstart.sh .
+
+ENTRYPOINT \
+    ./vstart.sh --new $CEPH_VSTART_ARGS && \
+    sleep infinity