From 09f299aedacced606e1da20f350db5549759fbea Mon Sep 17 00:00:00 2001 From: Eric Hough Date: Fri, 8 Mar 2019 13:57:05 -0800 Subject: [PATCH 01/14] NFS_LOG_LEVEL, not LOG_LEVEL --- doc/feature/logging.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/feature/logging.md b/doc/feature/logging.md index b65f395..1a33fca 100644 --- a/doc/feature/logging.md +++ b/doc/feature/logging.md @@ -16,7 +16,7 @@ services: image: erichough/nfs-server ... environment: - - LOG_LEVEL: DEBUG + - NFS_LOG_LEVEL: DEBUG ``` ### Normal log output From a09b0b223ccc2ba3964dd6ee86b0c4d758f8aae4 Mon Sep 17 00:00:00 2001 From: Eric Hough Date: Mon, 11 Mar 2019 13:59:39 -0700 Subject: [PATCH 02/14] initial add of state --- entrypoint.sh | 82 +++++++++++++++++++++++++++++++++++++-------------- 1 file changed, 60 insertions(+), 22 deletions(-) diff --git a/entrypoint.sh b/entrypoint.sh index 95ec8f5..59f54c8 100755 --- a/entrypoint.sh +++ b/entrypoint.sh @@ -58,6 +58,26 @@ readonly MOUNT_PATH_RPC_PIPEFS='/var/lib/nfs/rpc_pipefs' readonly REGEX_EXPORTS_LINES_TO_SKIP='^\s*#|^\s*$' +readonly LOG_LEVEL_INFO='INFO' +readonly LOG_LEVEL_DEBUG='DEBUG' + +readonly STATE_LOG_LEVEL='log_level' +readonly STATE_IS_LOGGING_DEBUG='is_logging_debug' +readonly STATE_IS_LOGGING_INFO='is_logging_info' + +# "state" is our only global variable, which is an associative array of normalized data +declare -A state + + +###################################################################################### +### string utils +###################################################################################### + +toupper() { + + echo "$1" | awk '{ print toupper($0) }' +} + ###################################################################################### ### logging @@ -84,7 +104,7 @@ log_header() { echo " ================================================================== - $(echo "$1" | awk '{print toupper($0)}') + $(toupper "$1") ==================================================================" } @@ -152,7 +172,7 @@ stop_mount() { if mount | grep -Eq ^"$type on $path\\s+"; then local args=() - if is_debug_requested; then + if is_logging_debug; then args+=('-v') fi args+=("$path") @@ -176,7 +196,7 @@ stop_nfsd() { stop_exportfs() { local args=('-ua') - if is_debug_requested; then + if is_logging_debug; then args+=('-v') fi @@ -293,15 +313,6 @@ has_linux_capability() { return 1 } -is_debug_requested() { - - if echo "${!ENV_VAR_NFS_LOG_LEVEL}" | grep -Eqi '^DEBUG$'; then - return 0 - fi - - return 1 -} - get_requested_count_nfsd_threads() { if [[ -n "${!ENV_VAR_NFS_SERVER_THREAD_COUNT}" ]]; then @@ -313,6 +324,12 @@ get_requested_count_nfsd_threads() { fi } +is_logging_debug() { + + [[ -n ${state[$STATE_IS_LOGGING_DEBUG]} ]] && return 0 || return 1 +} + + ###################################################################################### ### runtime configuration assertions ###################################################################################### @@ -400,6 +417,23 @@ assert_log_level() { ### initialization ###################################################################################### +init_state_logging() { + + # if the user didn't request a specific log level, the default is INFO + local -r normalized_log_level=$(toupper "${!ENV_VAR_NFS_LOG_LEVEL:-$LOG_LEVEL_INFO}") + + if ! echo "$normalized_log_level" | grep -Eq 'DEBUG|INFO'; then + bail "the only acceptable values for $ENV_VAR_NFS_LOG_LEVEL are: DEBUG, INFO" + fi + + state[$STATE_LOG_LEVEL]=$normalized_log_level; + state[$STATE_IS_LOGGING_INFO]=1 + + if [[ $normalized_log_level = "$LOG_LEVEL_DEBUG" ]]; then + state[$STATE_IS_LOGGING_DEBUG]=1 + fi +} + init_trap() { trap stop SIGTERM SIGINT @@ -409,13 +443,17 @@ init_exports() { # first, see if it's bind-mounted if mount | grep -Eq "^[^ ]+ on $PATH_FILE_ETC_EXPORTS type "; then - log "$PATH_FILE_ETC_EXPORTS is bind-mounted" + if is_logging_debug; then + log "$PATH_FILE_ETC_EXPORTS is bind-mounted" + fi return fi # maybe it's baked-in to the image if [[ -f $PATH_FILE_ETC_EXPORTS && -r $PATH_FILE_ETC_EXPORTS && -s $PATH_FILE_ETC_EXPORTS ]]; then - log "$PATH_FILE_ETC_EXPORTS is baked into the image" + if is_logging_debug; then + log "$PATH_FILE_ETC_EXPORTS is baked into the image" + fi return fi @@ -482,7 +520,6 @@ init_assertions() { assert_port "$ENV_VAR_NFS_PORT_STATD_OUT" assert_nfs_version assert_nfsd_threads - assert_log_level # check kernel modules assert_kernel_mod nfs @@ -515,7 +552,7 @@ boot_helper_mount() { local -r type=$(basename "$path") local args=('-t' "$type" "$path") - if is_debug_requested; then + if is_logging_debug; then args+=('-vvv') fi @@ -585,7 +622,7 @@ boot_main_mounts() { boot_main_exportfs() { local args=('-ar') - if is_debug_requested; then + if is_logging_debug; then args+=('-v') fi @@ -598,7 +635,7 @@ boot_main_mountd() { read -r -a version_flags <<< "$(boot_helper_get_version_flags)" local -r port=$(get_requested_port_mountd) local args=('--port' "$port" "${version_flags[@]}") - if is_debug_requested; then + if is_logging_debug; then args+=('--debug' 'all') fi @@ -623,7 +660,7 @@ boot_main_idmapd() { local args=('-S') local func=boot_helper_start_daemon - if is_debug_requested; then + if is_logging_debug; then args+=('-vvv' '-f') func=boot_helper_start_non_daemon fi @@ -652,7 +689,7 @@ boot_main_nfsd() { local -r port=$(get_requested_port_nfsd) local args=('--tcp' '--udp' '--port' "$port" "${version_flags[@]}" "$threads") - if is_debug_requested; then + if is_logging_debug; then args+=('--debug') fi @@ -670,7 +707,7 @@ boot_main_svcgssd() { fi local args=('-f') - if is_debug_requested; then + if is_logging_debug; then args+=('-vvv') fi @@ -716,7 +753,7 @@ summarize_exports() { # if debug is enabled, read /var/lib/nfs/etab as it contains the "real" export data. but it also contains more # information that most people will usually need to see local file_to_read="$PATH_FILE_ETC_EXPORTS" - if is_debug_requested; then + if is_logging_debug; then file_to_read='/var/lib/nfs/etab' fi @@ -759,6 +796,7 @@ init() { log_header 'setting up ...' + init_state_logging init_exports init_assertions init_trap From bc3f702e55ac90d002969e709c59e05940a991b3 Mon Sep 17 00:00:00 2001 From: Eric Hough Date: Mon, 11 Mar 2019 14:19:19 -0700 Subject: [PATCH 03/14] adding nfsd thread count to state --- entrypoint.sh | 48 +++++++++++++++++++++++++++++++++--------------- 1 file changed, 33 insertions(+), 15 deletions(-) diff --git a/entrypoint.sh b/entrypoint.sh index 59f54c8..43b028e 100755 --- a/entrypoint.sh +++ b/entrypoint.sh @@ -64,6 +64,7 @@ readonly LOG_LEVEL_DEBUG='DEBUG' readonly STATE_LOG_LEVEL='log_level' readonly STATE_IS_LOGGING_DEBUG='is_logging_debug' readonly STATE_IS_LOGGING_INFO='is_logging_info' +readonly STATE_NFSD_THREAD_COUNT='nfsd_thread_count' # "state" is our only global variable, which is an associative array of normalized data declare -A state @@ -315,13 +316,7 @@ has_linux_capability() { get_requested_count_nfsd_threads() { - if [[ -n "${!ENV_VAR_NFS_SERVER_THREAD_COUNT}" ]]; then - echo "${!ENV_VAR_NFS_SERVER_THREAD_COUNT}" - else - local -r cpu_count="$(grep -Ec ^processor /proc/cpuinfo)" - on_failure bail 'unable to detect CPU count. set NFS_SERVER_THREAD_COUNT environment variable' - echo "$cpu_count"; - fi + echo "${state[$STATE_NFSD_THREAD_COUNT]}" } is_logging_debug() { @@ -384,13 +379,6 @@ assert_nfs_version() { fi } -assert_nfsd_threads() { - - if [[ "$(get_requested_count_nfsd_threads)" -lt 1 ]]; then - bail "please set $ENV_VAR_NFS_SERVER_THREAD_COUNT to a positive integer" - fi -} - assert_at_least_one_export() { # ensure /etc/exports has at least one line @@ -434,6 +422,36 @@ init_state_logging() { fi } +init_state_nfsd_thread_count() { + + local count + + if [[ -n "${!ENV_VAR_NFS_SERVER_THREAD_COUNT}" ]]; then + + count="${!ENV_VAR_NFS_SERVER_THREAD_COUNT}" + + if [[ $count -lt 1 ]]; then + bail "please set $ENV_VAR_NFS_SERVER_THREAD_COUNT to a positive integer" + fi + + if is_logging_debug; then + log "will use requested rpc.nfsd thread count of $count" + fi + + else + + count="$(grep -Ec ^processor /proc/cpuinfo)" + on_failure bail "unable to detect CPU count. set $ENV_VAR_NFS_SERVER_THREAD_COUNT environment variable" + + if is_logging_debug; then + log "will use $count rpc.nfsd server thread(s) (1 thread per CPU)" + fi + + fi + + state[$STATE_NFSD_THREAD_COUNT]=$count +} + init_trap() { trap stop SIGTERM SIGINT @@ -519,7 +537,6 @@ init_assertions() { assert_port "$ENV_VAR_NFS_PORT_STATD_IN" assert_port "$ENV_VAR_NFS_PORT_STATD_OUT" assert_nfs_version - assert_nfsd_threads # check kernel modules assert_kernel_mod nfs @@ -798,6 +815,7 @@ init() { init_state_logging init_exports + init_state_nfsd_thread_count init_assertions init_trap From fc06571084681c780641c3fc33ef741c5a2cfbad Mon Sep 17 00:00:00 2001 From: Eric Hough Date: Mon, 11 Mar 2019 14:28:25 -0700 Subject: [PATCH 04/14] adding nfsd port to state --- entrypoint.sh | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/entrypoint.sh b/entrypoint.sh index 43b028e..1b63f84 100755 --- a/entrypoint.sh +++ b/entrypoint.sh @@ -65,6 +65,7 @@ readonly STATE_LOG_LEVEL='log_level' readonly STATE_IS_LOGGING_DEBUG='is_logging_debug' readonly STATE_IS_LOGGING_INFO='is_logging_info' readonly STATE_NFSD_THREAD_COUNT='nfsd_thread_count' +readonly STATE_NFSD_PORT='nfsd_port' # "state" is our only global variable, which is an associative array of normalized data declare -A state @@ -252,7 +253,7 @@ get_requested_port_mountd() { get_requested_port_nfsd() { - echo "${!ENV_VAR_NFS_PORT:-$DEFAULT_NFS_PORT}" + echo "${state[$STATE_NFSD_PORT]}" } get_requested_port_statd_in() { @@ -452,6 +453,12 @@ init_state_nfsd_thread_count() { state[$STATE_NFSD_THREAD_COUNT]=$count } +init_state_nfsd_port() { + + assert_port "$ENV_VAR_NFS_PORT" + state[$STATE_NFSD_PORT]=${!ENV_VAR_NFS_PORT:-$DEFAULT_NFS_PORT} +} + init_trap() { trap stop SIGTERM SIGINT @@ -532,7 +539,6 @@ init_exports() { init_assertions() { # validate any user-supplied environment variables - assert_port "$ENV_VAR_NFS_PORT" assert_port "$ENV_VAR_NFS_PORT_MOUNTD" assert_port "$ENV_VAR_NFS_PORT_STATD_IN" assert_port "$ENV_VAR_NFS_PORT_STATD_OUT" @@ -816,6 +822,7 @@ init() { init_state_logging init_exports init_state_nfsd_thread_count + init_state_nfsd_port init_assertions init_trap From fc93e6e4a318401b8f061270aa353da0d19efcea Mon Sep 17 00:00:00 2001 From: Eric Hough Date: Mon, 11 Mar 2019 14:42:30 -0700 Subject: [PATCH 05/14] adding ports to state --- entrypoint.sh | 49 +++++++++++++++++++------------------------------ 1 file changed, 19 insertions(+), 30 deletions(-) diff --git a/entrypoint.sh b/entrypoint.sh index 1b63f84..718e7cb 100755 --- a/entrypoint.sh +++ b/entrypoint.sh @@ -66,6 +66,9 @@ readonly STATE_IS_LOGGING_DEBUG='is_logging_debug' readonly STATE_IS_LOGGING_INFO='is_logging_info' readonly STATE_NFSD_THREAD_COUNT='nfsd_thread_count' readonly STATE_NFSD_PORT='nfsd_port' +readonly STATE_MOUNTD_PORT='mountd_port' +readonly STATE_STATD_PORT_IN='statd_port_in' +readonly STATE_STATD_PORT_OUT='statd_port_out' # "state" is our only global variable, which is an associative array of normalized data declare -A state @@ -248,7 +251,7 @@ get_requested_nfs_version() { get_requested_port_mountd() { - echo "${!ENV_VAR_NFS_PORT_MOUNTD:-$DEFAULT_NFS_PORT_MOUNTD}" + echo "${state[$STATE_MOUNTD_PORT]}" } get_requested_port_nfsd() { @@ -258,39 +261,27 @@ get_requested_port_nfsd() { get_requested_port_statd_in() { - echo "${!ENV_VAR_NFS_PORT_STATD_IN:-$DEFAULT_NFS_PORT_STATD_IN}" + echo "${state[$STATE_STATD_PORT_IN]}" } get_requested_port_statd_out() { - echo "${!ENV_VAR_NFS_PORT_STATD_OUT:-$DEFAULT_NFS_PORT_STATD_OUT}" + echo "${state[$STATE_STATD_PORT_OUT]}" } is_kerberos_requested() { - if [[ -n "${!ENV_VAR_NFS_ENABLE_KERBEROS}" ]]; then - return 0 - fi - - return 1 + [[ -n "${!ENV_VAR_NFS_ENABLE_KERBEROS}" ]] && return 0 || return 1 } is_nfs3_enabled() { - if [[ -z "${!ENV_VAR_NFS_DISABLE_VERSION_3}" ]]; then - return 0 - fi - - return 1 + [[ -z "${!ENV_VAR_NFS_DISABLE_VERSION_3}" ]] && return 0 || return 1 } is_idmapd_requested() { - if [[ -f "$PATH_FILE_ETC_IDMAPD_CONF" ]]; then - return 0 - fi - - return 1 + [[ -f "$PATH_FILE_ETC_IDMAPD_CONF" ]] && return 0 || return 1 } is_kernel_module_loaded() { @@ -394,13 +385,6 @@ assert_cap_sysadmin() { fi } -assert_log_level() { - - if ! echo "${!ENV_VAR_NFS_LOG_LEVEL}" | grep -Eqi "^$|^DEBUG$"; then - bail "the only acceptable value for $ENV_VAR_NFS_LOG_LEVEL is DEBUG" - fi -} - ###################################################################################### ### initialization @@ -417,6 +401,7 @@ init_state_logging() { state[$STATE_LOG_LEVEL]=$normalized_log_level; state[$STATE_IS_LOGGING_INFO]=1 + state[$STATE_IS_LOGGING_DEBUG]=0 if [[ $normalized_log_level = "$LOG_LEVEL_DEBUG" ]]; then state[$STATE_IS_LOGGING_DEBUG]=1 @@ -453,10 +438,17 @@ init_state_nfsd_thread_count() { state[$STATE_NFSD_THREAD_COUNT]=$count } -init_state_nfsd_port() { +init_state_ports() { assert_port "$ENV_VAR_NFS_PORT" + assert_port "$ENV_VAR_NFS_PORT_MOUNTD" + assert_port "$ENV_VAR_NFS_PORT_STATD_IN" + assert_port "$ENV_VAR_NFS_PORT_STATD_OUT" + state[$STATE_NFSD_PORT]=${!ENV_VAR_NFS_PORT:-$DEFAULT_NFS_PORT} + state[$STATE_MOUNTD_PORT]=${!ENV_VAR_NFS_PORT_MOUNTD:-$DEFAULT_NFS_PORT_MOUNTD} + state[$STATE_STATD_PORT_IN]=${!ENV_VAR_NFS_PORT_STATD_IN:-$DEFAULT_NFS_PORT_STATD_IN} + state[$STATE_STATD_PORT_OUT]=${!ENV_VAR_NFS_PORT_STATD_OUT:-$DEFAULT_NFS_PORT_STATD_OUT} } init_trap() { @@ -539,9 +531,6 @@ init_exports() { init_assertions() { # validate any user-supplied environment variables - assert_port "$ENV_VAR_NFS_PORT_MOUNTD" - assert_port "$ENV_VAR_NFS_PORT_STATD_IN" - assert_port "$ENV_VAR_NFS_PORT_STATD_OUT" assert_nfs_version # check kernel modules @@ -822,7 +811,7 @@ init() { init_state_logging init_exports init_state_nfsd_thread_count - init_state_nfsd_port + init_state_ports init_assertions init_trap From 09874c98af5eeaeba824859be3cf5b111113a799 Mon Sep 17 00:00:00 2001 From: Eric Hough Date: Mon, 11 Mar 2019 17:32:41 -0700 Subject: [PATCH 06/14] move NFS_VERSION into state and further declutter non-debug output --- entrypoint.sh | 160 ++++++++++++++++++++++++-------------------------- 1 file changed, 77 insertions(+), 83 deletions(-) diff --git a/entrypoint.sh b/entrypoint.sh index 718e7cb..2e451b8 100755 --- a/entrypoint.sh +++ b/entrypoint.sh @@ -69,6 +69,7 @@ readonly STATE_NFSD_PORT='nfsd_port' readonly STATE_MOUNTD_PORT='mountd_port' readonly STATE_STATD_PORT_IN='statd_port_in' readonly STATE_STATD_PORT_OUT='statd_port_out' +readonly STATE_NFS_VERSION='nfs_version' # "state" is our only global variable, which is an associative array of normalized data declare -A state @@ -246,7 +247,7 @@ stop() { get_requested_nfs_version() { - echo "${!ENV_VAR_NFS_VERSION:-$DEFAULT_NFS_VERSION}" + echo "${state[$STATE_NFS_VERSION]}" } get_requested_port_mountd() { @@ -289,7 +290,10 @@ is_kernel_module_loaded() { local -r module=$1 if lsmod | grep -Eq "^$module\\s+" || [[ -d "/sys/module/$module" ]]; then - log "kernel module $module is loaded" + + if is_logging_debug; then + log "kernel module $module is loaded" + fi return 0 fi @@ -359,32 +363,6 @@ assert_port() { fi } -assert_nfs_version() { - - local -r requested_version="$(get_requested_nfs_version)" - - echo "$requested_version" | grep -Eq '^3$|^4(\.[1-2])?$' - on_failure bail "please set $ENV_VAR_NFS_VERSION to one of: 4.2, 4.1, 4, 3" - - if ! is_nfs3_enabled && [[ "$requested_version" = '3' ]]; then - bail 'you cannot simultaneously enable and disable NFS version 3' - fi -} - -assert_at_least_one_export() { - - # ensure /etc/exports has at least one line - grep -Evq "$REGEX_EXPORTS_LINES_TO_SKIP" $PATH_FILE_ETC_EXPORTS - on_failure bail "$PATH_FILE_ETC_EXPORTS has no exports" -} - -assert_cap_sysadmin() { - - if ! has_linux_capability 'cap_sys_admin'; then - bail 'missing CAP_SYS_ADMIN. be sure to run this image with --cap-add SYS_ADMIN or --privileged' - fi -} - ###################################################################################### ### initialization @@ -401,10 +379,10 @@ init_state_logging() { state[$STATE_LOG_LEVEL]=$normalized_log_level; state[$STATE_IS_LOGGING_INFO]=1 - state[$STATE_IS_LOGGING_DEBUG]=0 if [[ $normalized_log_level = "$LOG_LEVEL_DEBUG" ]]; then state[$STATE_IS_LOGGING_DEBUG]=1 + log "log level set to $LOG_LEVEL_DEBUG" fi } @@ -451,6 +429,20 @@ init_state_ports() { state[$STATE_STATD_PORT_OUT]=${!ENV_VAR_NFS_PORT_STATD_OUT:-$DEFAULT_NFS_PORT_STATD_OUT} } +init_state_nfs_version() { + + local -r requested_version="${!ENV_VAR_NFS_VERSION:-$DEFAULT_NFS_VERSION}" + + echo "$requested_version" | grep -Eq '^3$|^4(\.[1-2])?$' + on_failure bail "please set $ENV_VAR_NFS_VERSION to one of: 4.2, 4.1, 4, 3" + + if ! is_nfs3_enabled && [[ "$requested_version" = '3' ]]; then + bail 'you cannot simultaneously enable and disable NFS version 3' + fi + + state[$STATE_NFS_VERSION]=$requested_version +} + init_trap() { trap stop SIGTERM SIGINT @@ -460,89 +452,90 @@ init_exports() { # first, see if it's bind-mounted if mount | grep -Eq "^[^ ]+ on $PATH_FILE_ETC_EXPORTS type "; then + if is_logging_debug; then log "$PATH_FILE_ETC_EXPORTS is bind-mounted" fi - return - fi # maybe it's baked-in to the image - if [[ -f $PATH_FILE_ETC_EXPORTS && -r $PATH_FILE_ETC_EXPORTS && -s $PATH_FILE_ETC_EXPORTS ]]; then + elif [[ -f $PATH_FILE_ETC_EXPORTS && -r $PATH_FILE_ETC_EXPORTS && -s $PATH_FILE_ETC_EXPORTS ]]; then + if is_logging_debug; then log "$PATH_FILE_ETC_EXPORTS is baked into the image" fi - return - fi - local count_valid_exports=0 - local exports='' - local candidate_export_vars - local candidate_export_var + # fallback to environment variables + else - # collect all candidate environment variable names - candidate_export_vars=$(compgen -A variable | grep -E 'NFS_EXPORT_[0-9]+' | sort) - on_failure bail 'failed to detect NFS_EXPORT_* variables' + local count_valid_exports=0 + local exports='' + local candidate_export_vars + local candidate_export_var - if [[ -z "$candidate_export_vars" ]]; then - bail "please provide $PATH_FILE_ETC_EXPORTS to the container or set at least one NFS_EXPORT_* environment variable" - fi + # collect all candidate environment variable names + candidate_export_vars=$(compgen -A variable | grep -E 'NFS_EXPORT_[0-9]+' | sort) + on_failure bail 'failed to detect NFS_EXPORT_* variables' - log "building $PATH_FILE_ETC_EXPORTS from environment variables" + if [[ -z "$candidate_export_vars" ]]; then + bail "please provide $PATH_FILE_ETC_EXPORTS to the container or set at least one NFS_EXPORT_* environment variable" + fi - for candidate_export_var in $candidate_export_vars; do + log "building $PATH_FILE_ETC_EXPORTS from environment variables" - local line="${!candidate_export_var}" + for candidate_export_var in $candidate_export_vars; do - # skip comments and empty lines - if [[ "$line" =~ $REGEX_EXPORTS_LINES_TO_SKIP ]]; then - log_warning "skipping $candidate_export_var environment variable since it contains only whitespace or a comment" - continue; - fi + local line="${!candidate_export_var}" - local line_as_array - read -r -a line_as_array <<< "$line" - local dir="${line_as_array[0]}" + # skip comments and empty lines + if [[ "$line" =~ $REGEX_EXPORTS_LINES_TO_SKIP ]]; then + log_warning "skipping $candidate_export_var environment variable since it contains only whitespace or a comment" + continue; + fi - if [[ ! -d "$dir" ]]; then - log_warning "skipping $candidate_export_var environment variable since $dir is not a container directory" - continue - fi + local line_as_array + read -r -a line_as_array <<< "$line" + local dir="${line_as_array[0]}" - if [[ $count_valid_exports -gt 0 ]]; then - exports=$exports$'\n' - fi + if [[ ! -d "$dir" ]]; then + log_warning "skipping $candidate_export_var environment variable since $dir is not a container directory" + continue + fi + + if [[ $count_valid_exports -gt 0 ]]; then + exports=$exports$'\n' + fi + + exports=$exports$line - exports=$exports$line + (( count_valid_exports++ )) - (( count_valid_exports++ )) + done - done + log "collected $count_valid_exports valid export(s) from NFS_EXPORT_* environment variables" - log "collected $count_valid_exports valid export(s) from NFS_EXPORT_* environment variables" + if [[ $count_valid_exports -eq 0 ]]; then + bail 'no valid exports' + fi - if [[ $count_valid_exports -eq 0 ]]; then - bail 'no valid exports' + echo "$exports" > $PATH_FILE_ETC_EXPORTS + on_failure bail "unable to write to $PATH_FILE_ETC_EXPORTS" fi - echo "$exports" > $PATH_FILE_ETC_EXPORTS - on_failure bail "unable to write to $PATH_FILE_ETC_EXPORTS" + # make sure we have at least one export + grep -Evq "$REGEX_EXPORTS_LINES_TO_SKIP" $PATH_FILE_ETC_EXPORTS + on_failure bail "$PATH_FILE_ETC_EXPORTS has no exports" } -init_assertions() { +init_runtime_assertions() { - # validate any user-supplied environment variables - assert_nfs_version + if ! has_linux_capability 'cap_sys_admin'; then + bail 'missing CAP_SYS_ADMIN. be sure to run this image with --cap-add SYS_ADMIN or --privileged' + fi # check kernel modules assert_kernel_mod nfs assert_kernel_mod nfsd - # make sure we have at least one export - assert_at_least_one_export - - # ensure we have CAP_SYS_ADMIN - assert_cap_sysadmin - # perform Kerberos assertions if is_kerberos_requested; then @@ -566,9 +559,9 @@ boot_helper_mount() { if is_logging_debug; then args+=('-vvv') + log "mounting $type filesystem onto $path" fi - log "mounting $type filesystem onto $path" mount "${args[@]}" on_failure stop "unable to mount $type filesystem onto $path" } @@ -638,7 +631,7 @@ boot_main_exportfs() { args+=('-v') fi - boot_helper_start_daemon 'exporting filesystem(s)' $PATH_BIN_EXPORTFS "${args[@]}" + boot_helper_start_daemon 'starting exportfs' $PATH_BIN_EXPORTFS "${args[@]}" } boot_main_mountd() { @@ -809,10 +802,11 @@ init() { log_header 'setting up ...' init_state_logging - init_exports init_state_nfsd_thread_count init_state_ports - init_assertions + init_state_nfs_version + init_exports + init_runtime_assertions init_trap log 'setup complete' From eb31efd1cc9708f0da3677acd8b3781040c047e4 Mon Sep 17 00:00:00 2001 From: Eric Hough Date: Mon, 11 Mar 2019 17:44:53 -0700 Subject: [PATCH 07/14] update logging docs --- doc/feature/logging.md | 25 ++++++++++++------------- 1 file changed, 12 insertions(+), 13 deletions(-) diff --git a/doc/feature/logging.md b/doc/feature/logging.md index 1a33fca..5e84faf 100644 --- a/doc/feature/logging.md +++ b/doc/feature/logging.md @@ -1,8 +1,8 @@ # Logging -By default, the image will output a reasonable level of logging information so you can see verify that the server is operating as expected. +By default, the image will output a reasonable level of logging information so you can verify that the server is operating as expected. -You can bump up the log level via the `NFS_LOG_LEVEL` environment variable. Currently, the only acceptable value is `DEBUG`. +You can adjust the logging level via the `NFS_LOG_LEVEL` environment variable. Currently, the only acceptable values are `INFO` (default) and `DEBUG`. In your `docker-run` command: ``` @@ -29,18 +29,13 @@ Normal, non-debug logging will look something like this: ================================================================== ----> building /etc/exports from environment variables ----> collected 4 valid export(s) from NFS_EXPORT_* environment variables -----> kernel module nfs is loaded -----> kernel module nfsd is loaded -----> kernel module rpcsec_gss_krb5 is loaded ----> setup complete ================================================================== STARTING SERVICES ... ================================================================== -----> mounting rpc_pipefs filesystem onto /var/lib/nfs/rpc_pipefs -----> mounting nfsd filesystem onto /proc/fs/nfsd ----> starting rpcbind -----> exporting filesystem(s) +----> starting exportfs ----> starting rpc.mountd on port 32767 ----> starting statd on port 32765 (outgoing from port 32766) ----> starting idmapd @@ -54,7 +49,7 @@ Normal, non-debug logging will look something like this: ----> list of enabled NFS protocol versions: 3 ----> list of container exports: ----> /nfs/htpc-media *(ro,no_subtree_check,insecure,async) -----> /nfs/homes/staff *(rw,no_subtree_check,insecure,sec=krb5p) +----> /nfs/homes/staff *(rw,no_subtree_check,insecure,no_root_squash,sec=krb5p) ----> /nfs/homes/ehough *(rw,no_subtree_check,insecure,no_root_squash,sec=krb5p) ----> /nfs/backup/duplicacy *(rw,no_subtree_check,insecure,sec=krb5p,all_squash,anonuid=0,anongid=0) ----> list of container ports that should be exposed: @@ -71,13 +66,16 @@ Normal, non-debug logging will look something like this: ### Debug output -Debug output will look something like this: +Debug output will be much more detailed, and it may be very helpful when diagnosing NFS problems. ``` ================================================================== SETTING UP ... ================================================================== -----> /etc/exports is baked into the image +----> log level set to DEBUG +----> will use requested rpc.nfsd thread count of 16 +----> building /etc/exports from environment variables +----> collected 4 valid export(s) from NFS_EXPORT_* environment variables ----> kernel module nfs is loaded ----> kernel module nfsd is loaded ----> kernel module rpcsec_gss_krb5 is loaded @@ -91,7 +89,7 @@ mount: mount('rpc_pipefs','/var/lib/nfs/rpc_pipefs','rpc_pipefs',0x00008000,'(nu ----> mounting nfsd filesystem onto /proc/fs/nfsd mount: mount('nfsd','/proc/fs/nfsd','nfsd',0x00008000,'(null)'):0 ----> starting rpcbind -----> exporting filesystem(s) +----> starting exportfs exporting *:/nfs/backup/duplicacy exporting *:/nfs/homes/ehough exporting *:/nfs/homes/staff @@ -132,7 +130,7 @@ entering poll ----> list of container exports: ----> /nfs/backup/duplicacy *(rw,sync,wdelay,hide,nocrossmnt,insecure,root_squash,all_squash,no_subtree_check,secure_locks,acl,no_pnfs,anonuid=0,anongid=0,sec=krb5p,rw,insecure,root_squash,all_squash) ----> /nfs/homes/ehough *(rw,sync,wdelay,hide,nocrossmnt,insecure,no_root_squash,no_all_squash,no_subtree_check,secure_locks,acl,no_pnfs,anonuid=65534,anongid=65534,sec=krb5p,rw,insecure,no_root_squash,no_all_squash) -----> /nfs/homes/staff *(rw,sync,wdelay,hide,nocrossmnt,insecure,root_squash,no_all_squash,no_subtree_check,secure_locks,acl,no_pnfs,anonuid=65534,anongid=65534,sec=krb5p,rw,insecure,root_squash,no_all_squash) +----> /nfs/homes/staff *(rw,sync,wdelay,hide,nocrossmnt,insecure,no_root_squash,no_all_squash,no_subtree_check,secure_locks,acl,no_pnfs,anonuid=65534,anongid=65534,sec=krb5p,rw,insecure,no_root_squash,no_all_squash) ----> /nfs/htpc-media *(ro,async,wdelay,hide,nocrossmnt,insecure,root_squash,no_all_squash,no_subtree_check,secure_locks,acl,no_pnfs,anonuid=65534,anongid=65534,sec=sys,ro,insecure,root_squash,no_all_squash) ----> list of container ports that should be exposed: ----> 111 (TCP and UDP) @@ -154,4 +152,5 @@ sending null reply writing message: \x \x6082024606092a864886f71201020201006e82023530820231a003020105a10302010ea20703050020000000a38201306182012c30820128a003020105a10d1b0b484f5547482e4d41544953a2153013a003020103a10c300a1b036e66731b036e6173a381fa3081f7a003020113a103020106a281ea0481e79cf640041a02f97332a25760a707a3f039f61301d07b2dfbb8bf9448cbfd168d0958c7717b535f7799c87390469c94045a6cd3f54c91ddeda9274b1ea492a43e6b17dec3ad17aaa94b9e61cdd4f4c8e7a35d8e84d56c7657e63536358e1316e2e8362922b47b465dd57aa29cb743128432decee09c3a06e6b4d5f6cebcd0978ee37bf0155a01a6ed623dc7b3068163fedaadec1e1509788db701c5308c703aa0e3196188e40c22afc361d2d9762750627c091516f05059a1f965df187dc981f4ac59bf3f424f23e676109a8c93af93b66f704f78703e1ef642fddf5b01d7deb40db26642e9f4f0a481e73081e4a003020111a281dc0481d9299c7ea474abf5d08a5f4f977254552e712f783f89bf40eb2cbd0082614593e377ec8cfe1c1ffb1bc0fad366382258f63857928240933914478fbceadc3b3bfe2e1f9a477c601d6b20c19898813878f45cea78ae601a342f000faf89c2e0e4c37fdb5db7937ac327ac0470c1f97dd421a112a6739467132d598db38ff99f9a88a8ac44e72f5cd088bd4d6159e15be75a7447d556134bda4fa0a96e64e3350d3a198e0635e4e7fb4900962aed3912fe0f316a1fa27121133232816b1177c707c0c37b396add3b347be38db756f05815ca3de5b3874782d80bf9 1552080460 0 0 \x01000000 \x60819906092a864886f71201020202006f8189308186a003020105a10302010fa27a3078a003020111a271046fdfb95cbe1237d785691a0ca14b4f7443142dda2b2a1b2845499bdb69b538719fbfc99b71d72ae61d7bd9966c106b2381fd08690082de26da5b8f521081035b5d7b8bf6c6eda85fd73c1c76ff03bec7693695e0b3d9e72069ec3772f93c4dbc5e8ce698a0854b494714bd5801204af3 finished handling null request entering poll +... ``` \ No newline at end of file From 2fbbe145ab6406cffeef5a5977e1179f2329b912 Mon Sep 17 00:00:00 2001 From: Eric Hough Date: Tue, 12 Mar 2019 11:20:27 -0700 Subject: [PATCH 08/14] refactor kill_process_if_running to term_process --- entrypoint.sh | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/entrypoint.sh b/entrypoint.sh index 2e451b8..6b9c98f 100755 --- a/entrypoint.sh +++ b/entrypoint.sh @@ -151,15 +151,15 @@ on_failure() { ### process control ###################################################################################### -kill_process_if_running() { +term_process() { local -r base=$(basename "$1") local -r pid=$(pidof "$base") if [[ -n $pid ]]; then - log "killing $base" - kill -TERM "$pid" - on_failure warn "unable to kill $base" + log "terminating $base" + kill "$pid" + on_failure warn "unable to terminate $base" else log "$base was not running" fi @@ -180,10 +180,10 @@ stop_mount() { local args=() if is_logging_debug; then args+=('-v') + log "un-mounting $type filesystem from $path" fi args+=("$path") - log "un-mounting $type filesystem from $path" umount "${args[@]}" on_failure warn "unable to un-mount $type filesystem from $path" @@ -194,9 +194,9 @@ stop_mount() { stop_nfsd() { - log 'stopping nfsd' + log 'terminating nfsd' $PATH_BIN_NFSD 0 - on_failure warn 'unable to stop nfsd. if it had started already, check Docker host for lingering [nfsd] processes' + on_failure warn 'unable to terminate nfsd. if it had started already, check Docker host for lingering [nfsd] processes' } stop_exportfs() { @@ -216,22 +216,22 @@ stop() { log_header 'terminating ...' if is_kerberos_requested; then - kill_process_if_running "$PATH_BIN_RPC_SVCGSSD" + term_process "$PATH_BIN_RPC_SVCGSSD" fi stop_nfsd if is_idmapd_requested; then - kill_process_if_running "$PATH_BIN_IDMAPD" + term_process "$PATH_BIN_IDMAPD" fi if is_nfs3_enabled; then - kill_process_if_running "$PATH_BIN_STATD" + term_process "$PATH_BIN_STATD" fi - kill_process_if_running "$PATH_BIN_MOUNTD" + term_process "$PATH_BIN_MOUNTD" stop_exportfs - kill_process_if_running "$PATH_BIN_RPCBIND" + term_process "$PATH_BIN_RPCBIND" stop_mount "$MOUNT_PATH_NFSD" stop_mount "$MOUNT_PATH_RPC_PIPEFS" @@ -701,7 +701,7 @@ boot_main_nfsd() { boot_helper_start_daemon "starting rpc.nfsd on port $port with $threads server thread(s)" $PATH_BIN_NFSD "${args[@]}" if ! is_nfs3_enabled; then - kill_process_if_running "$PATH_BIN_RPCBIND" + term_process "$PATH_BIN_RPCBIND" fi } From 050fc477d516aba8c27038a866098b2a14d6b6e0 Mon Sep 17 00:00:00 2001 From: Eric Hough Date: Tue, 12 Mar 2019 11:34:58 -0700 Subject: [PATCH 09/14] removing a few redundant functions --- entrypoint.sh | 60 +++++++++++++-------------------------------------- 1 file changed, 15 insertions(+), 45 deletions(-) diff --git a/entrypoint.sh b/entrypoint.sh index 6b9c98f..3d32fa7 100755 --- a/entrypoint.sh +++ b/entrypoint.sh @@ -245,31 +245,6 @@ stop() { ### runtime environment detection ###################################################################################### -get_requested_nfs_version() { - - echo "${state[$STATE_NFS_VERSION]}" -} - -get_requested_port_mountd() { - - echo "${state[$STATE_MOUNTD_PORT]}" -} - -get_requested_port_nfsd() { - - echo "${state[$STATE_NFSD_PORT]}" -} - -get_requested_port_statd_in() { - - echo "${state[$STATE_STATD_PORT_IN]}" -} - -get_requested_port_statd_out() { - - echo "${state[$STATE_STATD_PORT_OUT]}" -} - is_kerberos_requested() { [[ -n "${!ENV_VAR_NFS_ENABLE_KERBEROS}" ]] && return 0 || return 1 @@ -285,6 +260,11 @@ is_idmapd_requested() { [[ -f "$PATH_FILE_ETC_IDMAPD_CONF" ]] && return 0 || return 1 } +is_logging_debug() { + + [[ -n ${state[$STATE_IS_LOGGING_DEBUG]} ]] && return 0 || return 1 +} + is_kernel_module_loaded() { local -r module=$1 @@ -310,16 +290,6 @@ has_linux_capability() { return 1 } -get_requested_count_nfsd_threads() { - - echo "${state[$STATE_NFSD_THREAD_COUNT]}" -} - -is_logging_debug() { - - [[ -n ${state[$STATE_IS_LOGGING_DEBUG]} ]] && return 0 || return 1 -} - ###################################################################################### ### runtime configuration assertions @@ -568,7 +538,7 @@ boot_helper_mount() { boot_helper_get_version_flags() { - local -r requested_version="$(get_requested_nfs_version)" + local -r requested_version="${state[$STATE_NFS_VERSION]}" local flags=('--nfs-version' "$requested_version" '--no-nfs-version' 2) if ! is_nfs3_enabled; then @@ -638,7 +608,7 @@ boot_main_mountd() { local version_flags read -r -a version_flags <<< "$(boot_helper_get_version_flags)" - local -r port=$(get_requested_port_mountd) + local -r port="${state[$STATE_MOUNTD_PORT]}" local args=('--port' "$port" "${version_flags[@]}") if is_logging_debug; then args+=('--debug' 'all') @@ -679,8 +649,8 @@ boot_main_statd() { return fi - local -r port_in=$(get_requested_port_statd_in) - local -r port_out=$(get_requested_port_statd_out) + local -r port_in="${state[$STATE_STATD_PORT_IN]}" + local -r port_out="${state[$STATE_STATD_PORT_OUT]}" local -r args=('--no-notify' '--port' "$port_in" '--outgoing-port' "$port_out") boot_helper_start_daemon "starting statd on port $port_in (outgoing from port $port_out)" $PATH_BIN_STATD "${args[@]}" @@ -690,8 +660,8 @@ boot_main_nfsd() { local version_flags read -r -a version_flags <<< "$(boot_helper_get_version_flags)" - local -r threads=$(get_requested_count_nfsd_threads) - local -r port=$(get_requested_port_nfsd) + local -r threads="${state[$STATE_NFSD_THREAD_COUNT]}" + local -r port="${state[$STATE_NFSD_PORT]}" local args=('--tcp' '--udp' '--port' "$port" "${version_flags[@]}" "$threads") if is_logging_debug; then @@ -726,7 +696,7 @@ boot_main_svcgssd() { summarize_nfs_versions() { - local -r reqd_version="$(get_requested_nfs_version)" + local -r reqd_version="${state[$STATE_NFS_VERSION]}" local versions='' case "$reqd_version" in @@ -777,9 +747,9 @@ summarize_exports() { summarize_ports() { - local -r port_nfsd="$(get_requested_port_nfsd)" - local -r port_mountd="$(get_requested_port_mountd)" - local -r port_statd_in="$(get_requested_port_statd_in)" + local -r port_nfsd="${state[$STATE_NFSD_PORT]}" + local -r port_mountd="${state[$STATE_MOUNTD_PORT]}" + local -r port_statd_in="${state[$STATE_STATD_PORT_IN]}" if ! is_nfs3_enabled; then log "list of container ports that should be exposed: $port_nfsd (TCP)" From 057c33cf53b9a70f0d212010d6d57244cafda74e Mon Sep 17 00:00:00 2001 From: Eric Hough Date: Tue, 12 Mar 2019 11:37:12 -0700 Subject: [PATCH 10/14] refactor has_linux_capability to is_granted_linux_capability --- entrypoint.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/entrypoint.sh b/entrypoint.sh index 3d32fa7..38074b5 100755 --- a/entrypoint.sh +++ b/entrypoint.sh @@ -281,7 +281,7 @@ is_kernel_module_loaded() { return 1 } -has_linux_capability() { +is_granted_linux_capability() { if capsh --print | grep -Eq "^Current: = .*,?${1}(,|$)"; then return 0 @@ -310,7 +310,7 @@ assert_kernel_mod() { return fi - if [[ ! -d /lib/modules ]] || ! has_linux_capability 'sys_module'; then + if [[ ! -d /lib/modules ]] || ! is_granted_linux_capability 'sys_module'; then bail "$module module is not loaded in the Docker host's kernel (try: modprobe $module)" fi @@ -498,7 +498,7 @@ init_exports() { init_runtime_assertions() { - if ! has_linux_capability 'cap_sys_admin'; then + if ! is_granted_linux_capability 'cap_sys_admin'; then bail 'missing CAP_SYS_ADMIN. be sure to run this image with --cap-add SYS_ADMIN or --privileged' fi From e35ea4b276eaea03aa51c4ab8b05a8cf7de1a7dc Mon Sep 17 00:00:00 2001 From: Eric Hough Date: Tue, 12 Mar 2019 11:38:43 -0700 Subject: [PATCH 11/14] typo; it's modprobe not modproble --- entrypoint.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/entrypoint.sh b/entrypoint.sh index 38074b5..838f2c2 100755 --- a/entrypoint.sh +++ b/entrypoint.sh @@ -316,7 +316,7 @@ assert_kernel_mod() { log "attempting to load kernel module $module" modprobe -v "$module" - on_failure bail "unable to dynamically load kernel module $module. try modproble $module on the Docker host" + on_failure bail "unable to dynamically load kernel module $module. try modprobe $module on the Docker host" if ! is_kernel_module_loaded "$module"; then bail "modprobe claims that it loaded kernel module $module, but it still appears to be missing" From d1ca8f150446b91eae2bfd2a5c290414a3dff509 Mon Sep 17 00:00:00 2001 From: Eric Hough Date: Tue, 12 Mar 2019 12:43:36 -0700 Subject: [PATCH 12/14] add docs to primary processes. make rpc.statd print to console during debug --- entrypoint.sh | 75 ++++++++++++++++++++++++++++++++++++++++++++++----- 1 file changed, 68 insertions(+), 7 deletions(-) diff --git a/entrypoint.sh b/entrypoint.sh index 838f2c2..29aac3b 100755 --- a/entrypoint.sh +++ b/entrypoint.sh @@ -606,6 +606,11 @@ boot_main_exportfs() { boot_main_mountd() { + # https://linux.die.net/man/8/rpc.mountd + # + # --debug turn on debugging. Valid kinds are: all, auth, call, general and parse. + # --port specifies the port number used for RPC listener sockets + local version_flags read -r -a version_flags <<< "$(boot_helper_get_version_flags)" local -r port="${state[$STATE_MOUNTD_PORT]}" @@ -620,10 +625,19 @@ boot_main_mountd() { boot_main_rpcbind() { - # rpcbind isn't required for NFSv4, but if it's not running then nfsd takes over 5 minutes to start up. - # it's a bug in either nfs-utils or the kernel, and the code of both is over my head. - # so as a workaround we start rpcbind now and (in v4-only scenarios) kill it after nfsd starts up - local -r args=('-ds') + # https://linux.die.net/man/8/rpcbind + # + # -d run in debug mode. in this mode, rpcbind will not fork when it starts, will print additional information during + # operation, and will abort on certain errors if -a is also specified. with this option, the name-to-address + # translation consistency checks are shown in detail + # -s cause rpcbind to change to the user daemon as soon as possible. this causes rpcbind to use non-privileged ports + # for outgoing connections, preventing non-privileged clients from using rpcbind to connect to services from a + # privileged port + + local args=('-s') + if is_logging_debug; then + arg+=('-d') + fi boot_helper_start_daemon 'starting rpcbind' $PATH_BIN_RPCBIND "${args[@]}" } @@ -633,6 +647,12 @@ boot_main_idmapd() { return fi + # https://linux.die.net/man/8/rpc.idmapd + # + # -S Server-only: perform no idmapping for any NFS client, even if one is detected + # -v increases the verbosity level (can be specified multiple times + # -f runs rpc.idmapd in the foreground and prints all output to the terminal + local args=('-S') local func=boot_helper_start_daemon if is_logging_debug; then @@ -649,15 +669,44 @@ boot_main_statd() { return fi + # https://linux.die.net/man/8/rpc.statd + # + # --no-syslog causes rpc.statd to write log messages on stderr instead of to the system log, if the -F option was + # also specified + # --foreground keeps rpc.statd attached to its controlling terminal so that NSM operation can be monitored + # directly or run under a debugger. if this option is not specified, rpc.statd backgrounds itself + # soon after it starts + # --no-notify prevents rpc.statd from running the sm-notify command when it starts up, preserving the existing + # NSM state number and monitor list + # --outgoing-port specifies the source port number the sm-notify command should use when sending reboot notifications + # --port specifies the port number used for RPC listener sockets + local -r port_in="${state[$STATE_STATD_PORT_IN]}" local -r port_out="${state[$STATE_STATD_PORT_OUT]}" - local -r args=('--no-notify' '--port' "$port_in" '--outgoing-port' "$port_out") + local args=('--no-notify' '--port' "$port_in" '--outgoing-port' "$port_out") + local func=boot_helper_start_daemon - boot_helper_start_daemon "starting statd on port $port_in (outgoing from port $port_out)" $PATH_BIN_STATD "${args[@]}" + if is_logging_debug; then + args+=('--no-syslog' '--foreground') + func=boot_helper_start_non_daemon + fi + + $func "starting statd on port $port_in (outgoing from port $port_out)" $PATH_BIN_STATD "${args[@]}" } boot_main_nfsd() { + # https://linux.die.net/man/8/rpc.nfsd + # + # --debug enable logging of debugging messages + # --port specify a diferent port to listen on for NFS requests. by default, rpc.nfsd will listen on port 2049 + # --tcp explicitly enable TCP connections from clients + # --udp explicitly enable UCP connections from clients + # nproc specify the number of NFS server threads. by default, just one thread is started. however, for optimum + # performance several threads should be used. the actual figure depends on the number of and the work load + # created by the NFS clients, but a useful starting point is 8 threads. effects of modifying that number can + # be checked using the nfsstat(8) program + local version_flags read -r -a version_flags <<< "$(boot_helper_get_version_flags)" local -r threads="${state[$STATE_NFSD_THREAD_COUNT]}" @@ -670,6 +719,9 @@ boot_main_nfsd() { boot_helper_start_daemon "starting rpc.nfsd on port $port with $threads server thread(s)" $PATH_BIN_NFSD "${args[@]}" + # rpcbind isn't required for NFSv4, but if it's not running then nfsd takes over 5 minutes to start up. + # it's a bug in either nfs-utils or the kernel, and the code of both is over my head. + # so as a workaround we start rpcbind always and (in v4-only scenarios) kill it after nfsd starts up if ! is_nfs3_enabled; then term_process "$PATH_BIN_RPCBIND" fi @@ -681,9 +733,18 @@ boot_main_svcgssd() { return fi + # https://linux.die.net/man/8/rpc.svcgssd + # + # -f runs rpc.svcgssd in the foreground and sends output to stderr (as opposed to syslogd) + # -v increases the verbosity of the output (can be specified multiple times) + # -r if the rpcsec_gss library supports setting debug level, increases the verbosity of the output (can be specified + # multiple times) + # -i if the nfsidmap library supports setting debug level, increases the verbosity of the output (can be specified + # multiple times) + local args=('-f') if is_logging_debug; then - args+=('-vvv') + args+=('-vvv' '-rrr' '-iii') fi boot_helper_start_non_daemon 'starting rpc.svcgssd' $PATH_BIN_RPC_SVCGSSD "${args[@]}" From 6a306ed3b7e1f377ca8d74cd3324933131414965 Mon Sep 17 00:00:00 2001 From: Eric Hough Date: Tue, 12 Mar 2019 13:01:42 -0700 Subject: [PATCH 13/14] prep changelog for 2.2.1 --- CHANGELOG.md | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index b467c2c..6594f20 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,11 +4,18 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/) and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.html). +## [2.2.1] - unreleased + +### Fixed +* `rpc.statd` debug output was invisible + ## [2.2.0] - 2019-03-08 -## Added + +### Added * Enhanced debugging via environment variable: `NFS_LOG_LEVEL=DEBUG`. This also produces less cluttered log output during regular, non-debug operation. -## Fixed + +### Fixed * `idmapd` would not start when `NFS_VERSION=3` * allow Kerberos without `idmapd`. Most users will probably want to run them together, but it isn't required. @@ -17,8 +24,10 @@ it isn't required. * `idmapd` debug output was invisible ## [2.1.0] - 2019-01-31 + ### Added * Ability to automatically load kernel modules. ([#18](https://github.com/ehough/docker-nfs-server/issues/18)). Credit to [@andyneff](https://github.com/andyneff). + ### Fixed * Minor bugs in `entrypoint.sh` From a3ef44a4f62d37309da04d765763d1afbbe9bc4e Mon Sep 17 00:00:00 2001 From: Eric Hough Date: Fri, 15 Mar 2019 12:13:26 -0700 Subject: [PATCH 14/14] prep for 2.2.1 release --- CHANGELOG.md | 5 +++- doc/advanced/nfs-versions.md | 2 +- doc/advanced/performance-tuning.md | 6 +++-- doc/feature/logging.md | 40 +++++++++++++++++------------- entrypoint.sh | 10 ++++---- 5 files changed, 37 insertions(+), 26 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 6594f20..dd9e17e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,11 +4,14 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/) and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.html). -## [2.2.1] - unreleased +## [2.2.1] - 2019-03-15 ### Fixed * `rpc.statd` debug output was invisible +### Changed +* Further de-cluttered non-debug logging output + ## [2.2.0] - 2019-03-08 ### Added diff --git a/doc/advanced/nfs-versions.md b/doc/advanced/nfs-versions.md index dda752f..7e2b8d6 100644 --- a/doc/advanced/nfs-versions.md +++ b/doc/advanced/nfs-versions.md @@ -4,5 +4,5 @@ By default, this image provides NFS versions 3 and 4 simultaneously. Using the f | Environment variable | Description | Default | |-------------------------|--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|-----------| -| `NFS_VERSION` | Set to `3`, `4`, `4.1`, or `4.2` to fine tune the NFS protocol version. Enabling any version will also enable any lesser versions. e.g. `4.2` will enable versions 4.2, 4.1, 4, **and** 3. | `4.2` | +| `NFS_VERSION` | Set to `3`, `4`, `4.1`, or `4.2` to fine tune the NFS protocol version. Enabling any version will also enable any lesser versions. e.g. `4.1` will enable versions 4.1, 4, **and** 3. | `4.2` | | `NFS_DISABLE_VERSION_3` | Set to a non-empty value (e.g. `NFS_DISABLE_VERSION_3=1`) to disable NFS version 3 and run a version-4-only server. This setting is not compatible with `NFS_VERSION=3` | *not set* | \ No newline at end of file diff --git a/doc/advanced/performance-tuning.md b/doc/advanced/performance-tuning.md index 78971ea..5da791b 100644 --- a/doc/advanced/performance-tuning.md +++ b/doc/advanced/performance-tuning.md @@ -1,7 +1,9 @@ # Performance tuning -The following tips might improve your NFS server's performance. +The NFS server itself requires very little tuning; out-of-the-box it's blazingly fast even under high loads. You'll find that most performance gains will come from setting both the appropriate [mount options](https://linux.die.net/man/5/nfs) in your clients as well as the right [export options](https://linux.die.net/man/5/exports) on your shared filesystems. + +That said, the following tips might improve your NFS server's performance. * Set the **`NFS_SERVER_THREAD_COUNT`** environment variable to control how many server threads `rpc.nfsd` will use. A good minimum is one thread per CPU core, but 4 or 8 threads per core is probably better. The default is one thread per CPU core. -* Running the container with `--network host` *might* improve network performance by 10% - 20% [[1](https://jtway.co/docker-network-performance-b95bce32b4b9),[2](https://www.percona.com/blog/2016/08/03/testing-docker-multi-host-network-performance/)], though this hasn't been tested. \ No newline at end of file +* Running the container with `--network host` *might* improve network performance by 10% - 20% on a heavily-loaded server [[1](https://jtway.co/docker-network-performance-b95bce32b4b9),[2](https://www.percona.com/blog/2016/08/03/testing-docker-multi-host-network-performance/)], though this hasn't been tested. \ No newline at end of file diff --git a/doc/feature/logging.md b/doc/feature/logging.md index 5e84faf..44fedde 100644 --- a/doc/feature/logging.md +++ b/doc/feature/logging.md @@ -28,7 +28,7 @@ Normal, non-debug logging will look something like this: SETTING UP ... ================================================================== ----> building /etc/exports from environment variables -----> collected 4 valid export(s) from NFS_EXPORT_* environment variables +----> collected 3 valid export(s) from NFS_EXPORT_* environment variables ----> setup complete ================================================================== @@ -37,8 +37,8 @@ Normal, non-debug logging will look something like this: ----> starting rpcbind ----> starting exportfs ----> starting rpc.mountd on port 32767 -----> starting statd on port 32765 (outgoing from port 32766) -----> starting idmapd +----> starting rpc.statd on port 32765 (outgoing from port 32766) +----> starting rpc.idmapd ----> starting rpc.nfsd on port 2049 with 16 server thread(s) ----> starting rpc.svcgssd ----> all services started normally @@ -51,7 +51,6 @@ Normal, non-debug logging will look something like this: ----> /nfs/htpc-media *(ro,no_subtree_check,insecure,async) ----> /nfs/homes/staff *(rw,no_subtree_check,insecure,no_root_squash,sec=krb5p) ----> /nfs/homes/ehough *(rw,no_subtree_check,insecure,no_root_squash,sec=krb5p) -----> /nfs/backup/duplicacy *(rw,no_subtree_check,insecure,sec=krb5p,all_squash,anonuid=0,anongid=0) ----> list of container ports that should be exposed: ----> 111 (TCP and UDP) ----> 2049 (TCP and UDP) @@ -75,7 +74,7 @@ Debug output will be much more detailed, and it may be very helpful when diagnos ----> log level set to DEBUG ----> will use requested rpc.nfsd thread count of 16 ----> building /etc/exports from environment variables -----> collected 4 valid export(s) from NFS_EXPORT_* environment variables +----> collected 3 valid export(s) from NFS_EXPORT_* environment variables ----> kernel module nfs is loaded ----> kernel module nfsd is loaded ----> kernel module rpcsec_gss_krb5 is loaded @@ -90,21 +89,29 @@ mount: mount('rpc_pipefs','/var/lib/nfs/rpc_pipefs','rpc_pipefs',0x00008000,'(nu mount: mount('nfsd','/proc/fs/nfsd','nfsd',0x00008000,'(null)'):0 ----> starting rpcbind ----> starting exportfs -exporting *:/nfs/backup/duplicacy exporting *:/nfs/homes/ehough exporting *:/nfs/homes/staff exporting *:/nfs/htpc-media ----> starting rpc.mountd on port 32767 -----> starting statd on port 32765 (outgoing from port 32766) -----> starting idmapd +----> starting rpc.statd on port 32765 (outgoing from port 32766) +----> starting rpc.idmapd +----> starting rpc.nfsd on port 2049 with 16 server thread(s) +rpc.nfsd: knfsd is currently down +rpc.nfsd: Writing version string to kernel: -2 +3 +4 +4.1 +4.2 +rpc.nfsd: Created AF_INET TCP socket. +rpc.nfsd: Created AF_INET UDP socket. +rpc.nfsd: Created AF_INET6 TCP socket. +rpc.nfsd: Created AF_INET6 UDP socket. rpc.idmapd: Setting log level to 11 +----> starting rpc.svcgssd rpc.idmapd: libnfsidmap: using domain: hough.matis rpc.idmapd: libnfsidmap: Realms list: 'HOUGH.MATIS' rpc.idmapd: libnfsidmap: processing 'Method' list rpc.idmapd: static_getpwnam: name 'nfs/blue@HOUGH.MATIS' mapped to 'root' rpc.idmapd: static_getpwnam: localname 'melissa' for 'melissa@HOUGH.MATIS' not found rpc.idmapd: static_getpwnam: name 'ehough@HOUGH.MATIS' mapped to 'ehough' +libtirpc: debug level 3 rpc.idmapd: static_getgrnam: group 'nfs/blue@HOUGH.MATIS' mapped to 'root' rpc.idmapd: static_getgrnam: local group 'melissa' for 'melissa@HOUGH.MATIS' not found rpc.idmapd: static_getgrnam: group 'ehough@HOUGH.MATIS' mapped to 'ehough' @@ -112,14 +119,6 @@ rpc.idmapd: libnfsidmap: loaded plugin /usr/lib/libnfsidmap/static.so for method rpc.idmapd: Expiration time is 600 seconds. rpc.idmapd: Opened /proc/net/rpc/nfs4.nametoid/channel rpc.idmapd: Opened /proc/net/rpc/nfs4.idtoname/channel -----> starting rpc.nfsd on port 2049 with 16 server thread(s) -rpc.nfsd: knfsd is currently down -rpc.nfsd: Writing version string to kernel: -2 +3 +4 +4.1 +4.2 -rpc.nfsd: Created AF_INET TCP socket. -rpc.nfsd: Created AF_INET UDP socket. -rpc.nfsd: Created AF_INET6 TCP socket. -rpc.nfsd: Created AF_INET6 UDP socket. -----> starting rpc.svcgssd entering poll ----> all services started normally @@ -128,7 +127,6 @@ entering poll ================================================================== ----> list of enabled NFS protocol versions: 4.2, 4.1, 4, 3 ----> list of container exports: -----> /nfs/backup/duplicacy *(rw,sync,wdelay,hide,nocrossmnt,insecure,root_squash,all_squash,no_subtree_check,secure_locks,acl,no_pnfs,anonuid=0,anongid=0,sec=krb5p,rw,insecure,root_squash,all_squash) ----> /nfs/homes/ehough *(rw,sync,wdelay,hide,nocrossmnt,insecure,no_root_squash,no_all_squash,no_subtree_check,secure_locks,acl,no_pnfs,anonuid=65534,anongid=65534,sec=krb5p,rw,insecure,no_root_squash,no_all_squash) ----> /nfs/homes/staff *(rw,sync,wdelay,hide,nocrossmnt,insecure,no_root_squash,no_all_squash,no_subtree_check,secure_locks,acl,no_pnfs,anonuid=65534,anongid=65534,sec=krb5p,rw,insecure,no_root_squash,no_all_squash) ----> /nfs/htpc-media *(ro,async,wdelay,hide,nocrossmnt,insecure,root_squash,no_all_squash,no_subtree_check,secure_locks,acl,no_pnfs,anonuid=65534,anongid=65534,sec=sys,ro,insecure,root_squash,no_all_squash) @@ -141,6 +139,14 @@ entering poll ================================================================== READY AND WAITING FOR NFS CLIENT CONNECTIONS ================================================================== +rpc.statd: Version 2.3.2 starting +rpc.statd: Flags: No-Daemon Log-STDERR TI-RPC +rpc.statd: Failed to read /var/lib/nfs/state: Address in use +rpc.statd: Initializing NSM state +rpc.statd: Local NSM state number: 3 +rpc.statd: Failed to open /proc/sys/fs/nfs/nsm_local_state: Read-only file system +rpc.statd: Running as root. chown /var/lib/nfs to choose different user +rpc.statd: Waiting for client connections leaving poll handling null request svcgssd_limit_krb5_enctypes: Calling gss_set_allowable_enctypes with 7 enctypes from the kernel diff --git a/entrypoint.sh b/entrypoint.sh index 29aac3b..778c64e 100755 --- a/entrypoint.sh +++ b/entrypoint.sh @@ -434,7 +434,7 @@ init_exports() { log "$PATH_FILE_ETC_EXPORTS is baked into the image" fi - # fallback to environment variables + # fall back to environment variables else local count_valid_exports=0 @@ -576,9 +576,9 @@ boot_helper_start_non_daemon() { local -r bg_pid=$! - # somewhat arbitrary assumption that if the process isn't dead already, it will die within 1/20 of a second. for our + # somewhat arbitrary assumption that if the process isn't dead already, it will die within a millisecond. for our # purposes this works just fine, but if someone has a better solution please open a PR. - sleep .05 + sleep .001 kill -0 $bg_pid 2> /dev/null on_failure stop "$process failed" } @@ -660,7 +660,7 @@ boot_main_idmapd() { func=boot_helper_start_non_daemon fi - $func 'starting idmapd' $PATH_BIN_IDMAPD "${args[@]}" + $func 'starting rpc.idmapd' $PATH_BIN_IDMAPD "${args[@]}" } boot_main_statd() { @@ -691,7 +691,7 @@ boot_main_statd() { func=boot_helper_start_non_daemon fi - $func "starting statd on port $port_in (outgoing from port $port_out)" $PATH_BIN_STATD "${args[@]}" + $func "starting rpc.statd on port $port_in (outgoing from port $port_out)" $PATH_BIN_STATD "${args[@]}" } boot_main_nfsd() {