diff --git a/Dockerfile b/Dockerfile index cf629f1..584073b 100644 --- a/Dockerfile +++ b/Dockerfile @@ -5,42 +5,42 @@ FROM registry.suse.com/bci/bci-base:15.5 as s3gw-base ADD https://download.opensuse.org/repositories/filesystems:/ceph:/s3gw/15.5/repodata/repomd.xml /tmp/repodata-s3gw.xml # Add OBS repository for additional dependencies necessary on Leap 15.5 RUN zypper ar \ - https://download.opensuse.org/repositories/filesystems:/ceph:/s3gw/15.5/ \ - s3gw-deps \ - && zypper --gpg-auto-import-keys ref + https://download.opensuse.org/repositories/filesystems:/ceph:/s3gw/15.5/ \ + s3gw-deps \ + && zypper --gpg-auto-import-keys ref # Try `zypper install` up to three times to workaround mirror timeouts RUN for i in {1..3} ; do zypper -n install \ - libsqlite3-0=3.43.1 \ - libblkid1 \ - libexpat1 \ - libtcmalloc4 \ - libfmt9 \ - liboath0 \ - libicu-suse65_1 \ - libthrift-0_16_0 \ - libboost_atomic1_80_0 \ - libboost_chrono1_80_0 \ - libboost_context1_80_0 \ - libboost_coroutine1_80_0 \ - libboost_date_time1_80_0 \ - libboost_filesystem1_80_0 \ - libboost_iostreams1_80_0 \ - libboost_program_options1_80_0 \ - libboost_random1_80_0 \ - libboost_regex1_80_0 \ - libboost_serialization1_80_0 \ - libboost_system1_80_0 \ - libboost_thread1_80_0 \ - && break ; done \ - && zypper clean --all \ - && mkdir -p \ - /radosgw/bin \ - /radosgw/lib \ - /data - -ENV PATH=/radosgw/bin:$PATH -ENV LD_LIBRARY_PATH=/radosgw/lib:$LD_LIBRARY_PATH + libsqlite3-0=3.43.1 \ + libblkid1 \ + libexpat1 \ + libtcmalloc4 \ + libfmt9 \ + liboath0 \ + libicu-suse65_1 \ + libthrift-0_16_0 \ + libboost_atomic1_80_0 \ + libboost_chrono1_80_0 \ + libboost_context1_80_0 \ + libboost_coroutine1_80_0 \ + libboost_date_time1_80_0 \ + libboost_filesystem1_80_0 \ + libboost_iostreams1_80_0 \ + libboost_program_options1_80_0 \ + libboost_random1_80_0 \ + libboost_regex1_80_0 \ + libboost_serialization1_80_0 \ + libboost_system1_80_0 \ + libboost_thread1_80_0 \ + && break ; done \ + && zypper clean --all \ + && mkdir -p \ + /s3gw/bin \ + /s3gw/lib \ + /data + +ENV PATH=/s3gw/bin:$PATH +ENV LD_LIBRARY_PATH=/s3gw/lib:$LD_LIBRARY_PATH FROM s3gw-base as buildenv @@ -54,106 +54,106 @@ ADD https://download.opensuse.org/update/leap/15.5/oss/repodata/repomd.xml /tmp/ ADD https://download.opensuse.org/update/leap/15.5/backports/repodata/repomd.xml /tmp/repodata-backports-update.xml ADD https://download.opensuse.org/update/leap/15.5/sle/repodata/repomd.xml /tmp/repodata-sle-update.xml RUN zypper ar \ - http://download.opensuse.org/distribution/leap/15.5/repo/oss/ repo-oss \ - && zypper ar http://download.opensuse.org/update/leap/15.5/oss/ repo-update \ - && zypper ar http://download.opensuse.org/update/leap/15.5/backports/ repo-backports-update \ - && zypper ar http://download.opensuse.org/update/leap/15.5/sle/ repo-sle-update \ - && zypper --gpg-auto-import-keys ref + http://download.opensuse.org/distribution/leap/15.5/repo/oss/ repo-oss \ + && zypper ar http://download.opensuse.org/update/leap/15.5/oss/ repo-update \ + && zypper ar http://download.opensuse.org/update/leap/15.5/backports/ repo-backports-update \ + && zypper ar http://download.opensuse.org/update/leap/15.5/sle/ repo-sle-update \ + && zypper --gpg-auto-import-keys ref # Try `zypper install` up to three times to workaround mirror timeouts RUN for i in {1..3} ; do zypper -n install --no-recommends \ - 'cmake>3.5' \ - 'fmt-devel>=6.2.1' \ - 'gperftools-devel>=2.4' \ - 'libblkid-devel>=2.17' \ - 'liblz4-devel>=1.7' \ - 'libthrift-devel>=0.13.0' \ - 'pkgconfig(libudev)' \ - 'pkgconfig(systemd)' \ - 'pkgconfig(udev)' \ - babeltrace-devel \ - binutils \ - ccache \ - cmake \ - cpp12 \ - cryptsetup-devel \ - cunit-devel \ - fdupes \ - fuse-devel \ - gcc-c++ \ - gcc12 \ - gcc12-c++ \ - git \ - gperf \ - gtest \ - gmock \ - jq \ - keyutils-devel \ - libaio-devel \ - libasan6 \ - libboost_atomic1_80_0-devel \ - libboost_context1_80_0-devel \ - libboost_coroutine1_80_0-devel \ - libboost_filesystem1_80_0-devel \ - libboost_iostreams1_80_0-devel \ - libboost_program_options1_80_0-devel \ - libboost_python-py3-1_80_0-devel \ - libboost_random1_80_0-devel \ - libboost_regex1_80_0-devel \ - libboost_system1_80_0-devel \ - libboost_thread1_80_0-devel \ - libbz2-devel \ - libcap-devel \ - libcap-ng-devel \ - libcurl-devel \ - libexpat-devel \ - libicu-devel \ - libnl3-devel \ - liboath-devel \ - libopenssl-devel \ - libpmem-devel \ - libpmemobj-devel \ - librabbitmq-devel \ - librdkafka-devel \ - libstdc++6-devel-gcc12 \ - libtool \ - libtsan0 \ - libxml2-devel \ - lttng-ust-devel \ - lua53-devel \ - lua53-luarocks \ - make \ - memory-constraints \ - mozilla-nss-devel \ - nasm \ - ncurses-devel \ - net-tools \ - ninja \ - ninja \ - openldap2-devel \ - patch \ - perl \ - pkgconfig \ - procps \ - python3 \ - python3-Cython \ - python3-PrettyTable \ - python3-PyYAML \ - python3-Sphinx \ - python3-devel \ - python3-setuptools \ - rdma-core-devel \ - re2-devel \ - rpm-build \ - snappy-devel \ - sqlite-devel \ - systemd-rpm-macros \ - systemd-rpm-macros \ - valgrind-devel \ - xfsprogs-devel \ - xmlstarlet \ - && break ; done \ - && zypper clean --all + 'cmake>3.5' \ + 'fmt-devel>=6.2.1' \ + 'gperftools-devel>=2.4' \ + 'libblkid-devel>=2.17' \ + 'liblz4-devel>=1.7' \ + 'libthrift-devel>=0.13.0' \ + 'pkgconfig(libudev)' \ + 'pkgconfig(systemd)' \ + 'pkgconfig(udev)' \ + babeltrace-devel \ + binutils \ + ccache \ + cmake \ + cpp12 \ + cryptsetup-devel \ + cunit-devel \ + fdupes \ + fuse-devel \ + gcc-c++ \ + gcc12 \ + gcc12-c++ \ + git \ + gperf \ + gtest \ + gmock \ + jq \ + keyutils-devel \ + libaio-devel \ + libasan6 \ + libboost_atomic1_80_0-devel \ + libboost_context1_80_0-devel \ + libboost_coroutine1_80_0-devel \ + libboost_filesystem1_80_0-devel \ + libboost_iostreams1_80_0-devel \ + libboost_program_options1_80_0-devel \ + libboost_python-py3-1_80_0-devel \ + libboost_random1_80_0-devel \ + libboost_regex1_80_0-devel \ + libboost_system1_80_0-devel \ + libboost_thread1_80_0-devel \ + libbz2-devel \ + libcap-devel \ + libcap-ng-devel \ + libcurl-devel \ + libexpat-devel \ + libicu-devel \ + libnl3-devel \ + liboath-devel \ + libopenssl-devel \ + libpmem-devel \ + libpmemobj-devel \ + librabbitmq-devel \ + librdkafka-devel \ + libstdc++6-devel-gcc12 \ + libtool \ + libtsan0 \ + libxml2-devel \ + lttng-ust-devel \ + lua53-devel \ + lua53-luarocks \ + make \ + memory-constraints \ + mozilla-nss-devel \ + nasm \ + ncurses-devel \ + net-tools \ + ninja \ + ninja \ + openldap2-devel \ + patch \ + perl \ + pkgconfig \ + procps \ + python3 \ + python3-Cython \ + python3-PrettyTable \ + python3-PyYAML \ + python3-Sphinx \ + python3-devel \ + python3-setuptools \ + rdma-core-devel \ + re2-devel \ + rpm-build \ + snappy-devel \ + sqlite-devel \ + systemd-rpm-macros \ + systemd-rpm-macros \ + valgrind-devel \ + xfsprogs-devel \ + xmlstarlet \ + && break ; done \ + && zypper clean --all COPY $SRC_CEPH_DIR /srv/ceph @@ -166,30 +166,32 @@ FROM s3gw-base as s3gw-unittests # Try `zypper install` up to three times to workaround mirror timeouts RUN for i in {1..3} ; do zypper -n install --no-recommends \ - gtest \ - gmock \ - && break ; done \ - && zypper clean --all + gtest \ + gmock \ + && break ; done \ + && zypper clean --all -COPY --from=buildenv /srv/ceph/build/bin/unittest_rgw_* /radosgw/bin/ +COPY --from=buildenv /srv/ceph/build/bin/unittest_rgw_* /s3gw/bin/ COPY --from=buildenv [ \ - "/srv/ceph/build/lib/librados.so", \ - "/srv/ceph/build/lib/librados.so.2", \ - "/srv/ceph/build/lib/librados.so.2.0.0", \ - "/srv/ceph/build/lib/libceph-common.so", \ - "/srv/ceph/build/lib/libceph-common.so.2", \ - "/radosgw/lib/" ] + "/srv/ceph/build/lib/librados.so", \ + "/srv/ceph/build/lib/librados.so.2", \ + "/srv/ceph/build/lib/librados.so.2.0.0", \ + "/srv/ceph/build/lib/libceph-common.so", \ + "/srv/ceph/build/lib/libceph-common.so.2", \ + "/s3gw/lib/" ] ENTRYPOINT [ "bin/bash", "-x", "-c" ] -CMD [ "find /radosgw/bin -name \"unittest_rgw_*\" -print0 | xargs -0 -n1 bash -ec"] +CMD [ "find /s3gw/bin -name \"unittest_rgw_*\" -print0 | xargs -0 -n1 bash -ec"] FROM s3gw-base as s3gw ARG QUAY_EXPIRATION=Never ARG S3GW_VERSION=Development -ARG ID=s3gw +ARG S3GW_ID=s3gw -ENV ID=${ID} +ENV S3GW_ID=${S3GW_ID} +ENV S3GW_DNS_NAME="" +ENV S3GW_DEBUG="none" LABEL Name=s3gw LABEL Version=${S3GW_VERSION} @@ -197,22 +199,23 @@ LABEL quay.expires-after=${QUAY_EXPIRATION} VOLUME ["/data"] -COPY --from=buildenv /srv/ceph/build/bin/radosgw /radosgw/bin +COPY --from=buildenv /srv/ceph/build/bin/radosgw /s3gw/bin COPY --from=buildenv [ \ - "/srv/ceph/build/lib/librados.so", \ - "/srv/ceph/build/lib/librados.so.2", \ - "/srv/ceph/build/lib/librados.so.2.0.0", \ - "/srv/ceph/build/lib/libceph-common.so", \ - "/srv/ceph/build/lib/libceph-common.so.2", \ - "/radosgw/lib/" ] - + "/srv/ceph/build/lib/librados.so", \ + "/srv/ceph/build/lib/librados.so.2", \ + "/srv/ceph/build/lib/librados.so.2.0.0", \ + "/srv/ceph/build/lib/libceph-common.so", \ + "/srv/ceph/build/lib/libceph-common.so.2", \ + "/s3gw/lib/" ] + +COPY tools/entrypoint.sh /s3gw/bin/ + +# ports for S3 endpoints +# http: 7480 +# https: 7481 EXPOSE 7480 EXPOSE 7481 +# port for status endpoints +EXPOSE 9090 -ENTRYPOINT [ "radosgw", "-d", \ - "--no-mon-config", \ - "--id", "${ID}", \ - "--rgw-data", "/data/", \ - "--run-dir", "/run/", \ - "--rgw-sfs-data-path", "/data" ] -CMD [ "--rgw-backend-store", "sfs", "--debug-rgw", "1" ] +ENTRYPOINT [ "entrypoint.sh" ] diff --git a/docs/advanced-usage.md b/docs/advanced-usage.md new file mode 100644 index 0000000..ae486ac --- /dev/null +++ b/docs/advanced-usage.md @@ -0,0 +1,121 @@ +# Advanced usage + +The processes and steps described in this page should not be applicable to the +vast majority of s3gw deployments. However, depending on circumstance, one might +need to perform some of these for testing or debugging. + +The following sections rely on `podman` for examples. Using `docker` instead +should work in the same way, although option names may need to be adjusted on +occasion. + +## Running the container + +One might need to run the container standalone, in which case it's important to +understand how to properly configure it. Running the container with `--help` +will provide some insight. + +`podman run -it quay.io/s3gw/s3gw:latest --help` + +Which should result in something like the following, minus eventual changes not +reflected in this example: + + usage: /s3gw/bin/entrypoint.sh [options... -- args] + + options: + --help Shows this message. + --id ID Specifies a custom instance ID (default: s3gw). + --cert FILE KEY Specifies SSL certificate. Expects a .crt as first + argument, and a .key as second argument. + --dns-name VALUE For vhost style buckets, VALUE should be the DNS domain + to be used. + --debug LEVEL Runs with debug. Levels available: "high", "medium", + "low", "none" (default: none). + --no-telemetry Disable telemetry. + --telemetry-url URL Specifies telemetry URL. + --with-status Enables status frontend at port 9090. + + args: + Any option supported by RADOS Gateway. For advanced use only. + + env variables: + S3GW_ID Specifies a custom instance ID. + S3GW_DNS_NAME Specifies a DNS domain to be used for vhost style buckets. + S3GW_DEBUG Specifies the debug level to be used. + S3GW_CERT_FILE Specifies the SSL certificate file. + S3GW_CERT_KEY Specifies the SSL certificate key file. + +Keep in mind that to access the service, one needs to expose the required ports. +For s3gw these are port 7480 for non-SSL access, and port 7481 for SSL access. +This can be achieved with: + +`podman run -it -p 7480:7480 quay.io/s3gw/s3gw:latest` + +In the following sections, we will be describing some of the available options. + +### Using vhost style buckets + +S3 supports two styles of bucket access: path and vhost. The former considers +the buckets will be part of the address path, like in `my.domain.tld/my-bucket/`, +whereas the latter assumes the bucket will be part of the vhost, +`my-bucket.my.domain.tld`. + +By default, s3gw runs with path style buckets. One can enable vhost style +buckets by providing the DNS domain name to be used for the service. For +example, + + podman run -it -p 7480:7480 \ + quay.io/s3gw/s3gw:latest \ + --dns-name my.domain.tld + +### Specifying certificates + +s3gw supports running with SSL, but will require a certificate to be provided. +The location of this certificate must be local to the container namespace, and +thus should be bind mounted into the container. + + podman run -it -p 7481:7481 \ + -v ./my-cert.crt:/data/my-cert.crt \ + -v ./my-cert.key:/data/my-cert.key \ + quay.io/s3gw/s3gw:latest \ + --cert /data/my-cert.crt /data/my-cert.key + +The s3gw service should now be available on the host's port 7481 with SSL +enabled. + +### Debugging output + +s3gw supports four different log levels: `none`, `low`, `medium`, and `high`. By +default, s3gw runs with `none`. + +The vast majority of deployments will only require `none` or `low`, for as long +as they are behaving nicely. In case of misbehavior, it is reasonable to look +towards the logs for indication of what the problem might be. In this case, the +typical deployment will rely on `medium`. Only in very exceptional situations +will a `high` be useful. Keep in mind that at a `high` debug level, the system +will be put under a lot of strain from the amount of outputted messages, and the +value of additional verbosity may not be significant. + +Running with debug only requires specifying the `--debug LEVEL` argument to the +container, such as + + podman run -it -p 7480:7480 \ + quay.io/s3gw/s3gw:latest \ + --debug medium + +### Environment variables + +The available environment variables reflect the arguments accepted by the +container. These can be provided to the container using the `--env VAR=VALUE` +argument to `podman run`. For example, + + podman run -it -p 7480:7480 \ + --env S3GW_DEBUG=medium \ + quay.io/s3gw/s3gw:latest + +### Additional arguments + +The container accepts additional arguments, that will be passed directly to the +underlying RADOS Gateway binary. These should be used only if one knows what +they are doing. To obtain a small subset of the option supported, running with +`-- --help` will provide some information. Please note that not all the options +provided in this help message will be applicable. diff --git a/docs/config-s3gw.md b/docs/config-s3gw.md index afc8215..3062e5d 100644 --- a/docs/config-s3gw.md +++ b/docs/config-s3gw.md @@ -207,15 +207,18 @@ set up a node-local volume for testing if you do not have Longhorn. ## Log settings The log verbosity can also be configured for the s3gw pod. Set the `logLevel` -property to a number, with `"0"` being the least verbose and higher numbers -being more verbose: +property to one of `none`, `low`, `medium`, or `high`: ```yaml -logLevel: "1" +logLevel: "none" ``` -The highest `logLevel` that we recommend is 10, however, any integer required -is valid. +Generally speaking, `none` or `low` should be fine for most deployments, unless +there is reason to believe it's misbehaving. In that case, the highest +`logLevel` we recommend is `medium`. Higher log levels should be used only in +case the system is misbehaving significantly and there's a suspicion more +verbose debugging might be necessary. Please note that at `high` the system may +be impacted. ## Developer options diff --git a/mkdocs.yml b/mkdocs.yml index 90c999e..35c8f36 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -23,6 +23,7 @@ nav: - Installing k3s for s3gw: "s3gw-with-k3s.md" - Installing s3gw with Helm charts: "helm-charts.md" - Configuring the s3gw: "config-s3gw.md" + - Advanced usage: "advanced-usage.md" - Troubleshooting the s3gw: "troubleshoot-s3gw.md" - "Development Manual": - Developing the S3gw: "developing.md" diff --git a/tools/entrypoint.sh b/tools/entrypoint.sh new file mode 100755 index 0000000..c2f547c --- /dev/null +++ b/tools/entrypoint.sh @@ -0,0 +1,221 @@ +#!/bin/bash +# +# run-s3gw.sh - wrapper to run a radosgw binary +# Copyright 2022 SUSE, LLC. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +default_id="s3gw" +default_debug_level="none" + +rgw_pid= + +usage() { + cat </dev/stderr + exit 1 + ;; + esac + + shift 1 + +done + +# this can only happen if the user specifies "--id ''". +[[ -z "${s3gw_id}" ]] && + echo "ERROR: s3gw ID can't be empty!" >/dev/stderr && + exit 1 + +rgw_debug_value=0 + +if [[ -n "${s3gw_debug_level}" ]]; then + case ${s3gw_debug_level} in + none) ;; + low) + rgw_debug_value=1 + ;; + medium) + rgw_debug_value=10 + ;; + high) + rgw_debug_value=20 + ;; + *) + echo "ERROR: unknown debug value '${s3gw_debug_level}'" >/dev/stderr + exit 1 + ;; + esac +fi + +if [[ -n "${cert_file}" ]]; then + if [[ -z "${cert_key}" ]]; then + echo "ERROR: missing SSL certificate key file" + exit 1 + fi +elif [[ -n "${cert_key}" ]]; then + echo "ERROR: missing SSL certificate file" + exit 1 +fi + +frontend_args="beast port=7480" +if [[ -n "${cert_file}" ]]; then + [[ ! -f "${cert_file}" ]] && + echo "ERROR: certificate file not found at '${cert_file}'" && + exit 1 + + [[ ! -f "${cert_key}" ]] && + echo "ERROR: certificate key file not found at '${cert_key}'" && + exit 1 + + frontend_args="beast port=7480 ssl_port=7481 + ssl_certificate=${cert_file} ssl_private_key=${cert_key}" +fi + +if [[ $with_status -eq 1 ]]; then + frontend_args+=", status bind=0.0.0.0 port=9090" +fi + +args=( + "--id" + "${s3gw_id}" + "--debug-rgw" + "${rgw_debug_value}" +) + +[[ -n "${s3gw_dns_name}" ]] && + args=("${args[@]}" "--rgw-dns-name" "${s3gw_dns_name}") + +args=("${args[@]}" "${extra_args[@]}") + +if [[ $with_telemetry -eq 1 ]]; then + if [[ -n "${telemetry_url}" ]]; then + args=( + "${args[@]}" + "--rgw-s3gw-telemetry-upgrade-responder-url" + "${telemetry_url}" + ) + fi +else + args=( + "${args[@]}" + "--rgw-s3gw-enable-telemetry" + "false" + ) +fi + +trap "stop_rgw" EXIT + +radosgw -d \ + --no-mon-config \ + --rgw-data /data/ \ + --run-dir /run/ \ + --rgw-sfs-data-path /data \ + --rgw-backend-store sfs \ + --rgw-frontends "${frontend_args}" \ + "${args[@]}" & + +rgw_pid=$! +wait ${rgw_pid}