From 6803be0cfc127a511818c0307679121ebc6cb04e Mon Sep 17 00:00:00 2001 From: Chris Evich Date: Thu, 29 Apr 2021 12:10:58 -0400 Subject: [PATCH] Update to F34 and U2104 Also modernize main library to resemble what's used in other containers-org repositories. Lastly, update hack/get_ci_vm.sh to use the new shared container image. Signed-off-by: Chris Evich --- .cirrus.yml | 10 +- contrib/cirrus/lib.sh | 101 +++++++--------- hack/get_ci_vm.sh | 269 +++++++++--------------------------------- 3 files changed, 102 insertions(+), 278 deletions(-) diff --git a/.cirrus.yml b/.cirrus.yml index 836bf659aa..81e3ef869c 100644 --- a/.cirrus.yml +++ b/.cirrus.yml @@ -17,15 +17,15 @@ env: #### #### Cache-image names to test with (double-quotes around names are critical) ### - FEDORA_NAME: "fedora-33" - PRIOR_FEDORA_NAME: "fedora-32" - UBUNTU_NAME: "ubuntu-2010" - PRIOR_UBUNTU_NAME: "ubuntu-2004" + FEDORA_NAME: "fedora-34" + PRIOR_FEDORA_NAME: "fedora-33" + UBUNTU_NAME: "ubuntu-2104" + PRIOR_UBUNTU_NAME: "ubuntu-2010" # GCE project where images live IMAGE_PROJECT: "libpod-218412" # VM Image built in containers/automation_images - _BUILT_IMAGE_SUFFIX: "c5744859501821952" + _BUILT_IMAGE_SUFFIX: "c6032583541653504" FEDORA_CACHE_IMAGE_NAME: "fedora-${_BUILT_IMAGE_SUFFIX}" PRIOR_FEDORA_CACHE_IMAGE_NAME: "prior-fedora-${_BUILT_IMAGE_SUFFIX}" UBUNTU_CACHE_IMAGE_NAME: "ubuntu-${_BUILT_IMAGE_SUFFIX}" diff --git a/contrib/cirrus/lib.sh b/contrib/cirrus/lib.sh index 7c8b94272c..d1519965d2 100755 --- a/contrib/cirrus/lib.sh +++ b/contrib/cirrus/lib.sh @@ -3,26 +3,28 @@ # Library of common, shared utility functions. This file is intended # to be sourced by other scripts, not called directly. -# Global details persist here -source /etc/environment # not always loaded under all circumstances +# BEGIN Global export of all variables +set -a # Due to differences across platforms and runtime execution environments, # handling of the (otherwise) default shell setup is non-uniform. Rather # than attempt to workaround differences, simply force-load/set required # items every time this library is utilized. -source /etc/profile -source /etc/environment USER="$(whoami)" -export HOME="$(getent passwd $USER | cut -d : -f 6)" -[[ -n "$UID" ]] || UID=$(getent passwd $USER | cut -d : -f 3) -GID=$(getent passwd $USER | cut -d : -f 4) - -# During VM Image build, the 'containers/automation' installation -# was performed. The final step of installation sets the library -# location $AUTOMATION_LIB_PATH in /etc/environment or in the -# default shell profile depending on distribution. +HOME="$(getent passwd $USER | cut -d : -f 6)" +# Some platforms set and make this read-only +[[ -n "$UID" ]] || \ + UID=$(getent passwd $USER | cut -d : -f 3) + +# Automation library installed at image-build time, +# defining $AUTOMATION_LIB_PATH in this file. +if [[ -r "/etc/automation_environment" ]]; then + source /etc/automation_environment +fi +# shellcheck disable=SC2154 if [[ -n "$AUTOMATION_LIB_PATH" ]]; then - source $AUTOMATION_LIB_PATH/common_lib.sh + # shellcheck source=/usr/share/automation/lib/common_lib.sh + source $AUTOMATION_LIB_PATH/common_lib.sh else ( echo "WARNING: It does not appear that containers/automation was installed." @@ -33,37 +35,34 @@ fi # Essential default paths, many are overridden when executing under Cirrus-CI # others are duplicated here, to assist in debugging. -export GOPATH="${GOPATH:-/var/tmp/go}" -if type -P go &> /dev/null -then - # required for go 1.12+ - export GOCACHE="${GOCACHE:-$HOME/.cache/go-build}" - eval "$(go env)" - # required by make and other tools - export $(go env | cut -d '=' -f 1) - - # Ensure compiled tooling is reachable - export PATH="$PATH:$GOPATH/bin" -fi +GOPATH="${GOPATH:-/var/tmp/go}" +GOCACHE="${GOCACHE:-$GOPATH/cache/go-build}" +# called processes like `make` and other tools need these vars. +eval "$(go env)" CIRRUS_WORKING_DIR="${CIRRUS_WORKING_DIR:-$GOPATH/src/github.com/containers/storage}" -export GOSRC="${GOSRC:-$CIRRUS_WORKING_DIR}" -export PATH="$HOME/bin:$GOPATH/bin:/usr/local/bin:$PATH" +GOSRC="${GOSRC:-$CIRRUS_WORKING_DIR}" +PATH="$HOME/bin:$GOPATH/bin:/usr/local/bin:$PATH" SCRIPT_BASE=${GOSRC}/contrib/cirrus -cd $GOSRC -if type -P git &> /dev/null -then - CIRRUS_CHANGE_IN_REPO=${CIRRUS_CHANGE_IN_REPO:-$(git show-ref --hash=8 HEAD || date +%s)} -else # pick something unique and obviously not from Cirrus - CIRRUS_CHANGE_IN_REPO=${CIRRUS_CHANGE_IN_REPO:-no_git_$(date +%s)} -fi - -export CI="${CI:-false}" +CI="${CI:-false}" CIRRUS_CI="${CIRRUS_CI:-false}" +DEST_BRANCH="${DEST_BRANCH:-master}" CONTINUOUS_INTEGRATION="${CONTINUOUS_INTEGRATION:-false}" CIRRUS_REPO_NAME=${CIRRUS_REPO_NAME:-storage} -CIRRUS_BASE_SHA=${CIRRUS_BASE_SHA:-unknown$(date +%s)} # difficult to reliably discover -CIRRUS_BUILD_ID=${CIRRUS_BUILD_ID:-$RANDOM$(date +%s)} # must be short and unique +# Cirrus only sets $CIRRUS_BASE_SHA properly for PRs, but $EPOCH_TEST_COMMIT +# needs to be set from this value in order for `make validate` to run properly. +# When running get_ci_vm.sh, most $CIRRUS_xyz variables are empty. Attempt +# to accomidate both branch and get_ci_vm.sh testing by discovering the base +# branch SHA value. +if [[ -z "$CIRRUS_BASE_SHA" ]] && [[ -z "$CIRRUS_TAG" ]] +then # Operating on a branch, or under `get_ci_vm.sh` + CIRRUS_BASE_SHA=$(git rev-parse ${UPSTREAM_REMOTE:-origin}/$DEST_BRANCH) +elif [[ -z "$CIRRUS_BASE_SHA" ]] +then # Operating on a tag + CIRRUS_BASE_SHA=$(git rev-parse HEAD) +fi +# The starting place for linting and code validation +EPOCH_TEST_COMMIT="$CIRRUS_BASE_SHA" # Unsafe env. vars for display SECRET_ENV_RE='(IRCID)|(ACCOUNT)|(^GC[EP]..+)|(SSH)' @@ -76,13 +75,13 @@ OS_RELEASE_VER="$(source /etc/os-release; echo $VERSION_ID | tr -d '.')" OS_REL_VER="${OS_RELEASE_ID}-${OS_RELEASE_VER}" # Working with dnf + timeout/retry -export SHORT_DNFY='lilto dnf -y' -export LONG_DNFY='bigto dnf -y' +SHORT_DNFY='lilto dnf -y' +LONG_DNFY='bigto dnf -y' # Working with apt under Debian/Ubuntu automation is a PITA, make it easy # Avoid some ways of getting stuck waiting for user input -export DEBIAN_FRONTEND=noninteractive +DEBIAN_FRONTEND=noninteractive # Short-cut for frequently used base command -export SUDOAPTGET='sudo -E apt-get -q --yes' +SUDOAPTGET='sudo -E apt-get -q --yes' # Short list of packages or quick-running command SHORT_APTGET="lilto $SUDOAPTGET" # Long list / long-running command @@ -92,6 +91,9 @@ LONG_APTGET="bigto $SUDOAPTGET" RPMS_CONFLICTING="gcc-go" DEBS_CONFLICTING="" +# END Global export of all variables +set +a + bad_os_id_ver() { die "Unknown/Unsupported distro. $OS_RELEASE_ID and/or version $OS_RELEASE_VER for $(basename $0)" } @@ -120,18 +122,3 @@ install_bats_from_git(){ mkdir -p ~/.parallel touch ~/.parallel/will-cite } - -showrun() { - if [[ "$1" == "--background" ]] - then - shift - # Properly escape any nested spaces, so command can be copy-pasted - msg '+ '$(printf " %q" "$@")' &' - "$@" & - msg -e "${RED}${NOR}" - else - msg '--------------------------------------------------' - msg '+ '$(printf " %q" "$@") > /dev/stderr - "$@" - fi -} diff --git a/hack/get_ci_vm.sh b/hack/get_ci_vm.sh index 3e17210840..8c2c76bf6d 100755 --- a/hack/get_ci_vm.sh +++ b/hack/get_ci_vm.sh @@ -1,225 +1,62 @@ #!/usr/bin/env bash -set -e - -RED="\e[1;36;41m" -YEL="\e[1;33;44m" -NOR="\e[0m" -USAGE_WARNING=" -${YEL}WARNING: This will not work without local sudo access to run podman,${NOR} - ${YEL}and prior authorization to use the storage GCP project. Also,${NOR} - ${YEL}possession of the proper ssh private key is required.${NOR} -" -# TODO: Many/most of these values should come from .cirrus.yml -ZONE="us-central1-b" -CPUS="2" -MEMORY="4Gb" -DISK="200" -PROJECT="storage-240716" -GOSRC="/var/tmp/go/src/github.com/containers/storage" -GCLOUD_IMAGE=${GCLOUD_IMAGE:-quay.io/cevich/gcloud_centos:latest} -GCLOUD_SUDO=${GCLOUD_SUDO-sudo} -SSHUSER="root" - -# Shared tmp directory between container and us -TMPDIR=$(mktemp -d --tmpdir $(basename $0)_tmpdir_XXXXXX) - -STORAGEROOT=$(realpath "$(dirname $0)/../") -# else: Assume $PWD is the root of the storage repository -[[ "$STORAGEROOT" != "/" ]] || STORAGEROOT=$PWD - -# Command shortcuts save some typing (assumes $STORAGEROOT is subdir of $HOME) -PGCLOUD="$GCLOUD_SUDO podman run -it --rm -e AS_ID=$UID -e AS_USER=$USER --security-opt label=disable -v $TMPDIR:$HOME -v $HOME/.config/gcloud:$HOME/.config/gcloud -v $HOME/.config/gcloud/ssh:$HOME/.ssh -v $STORAGEROOT:$STORAGEROOT $GCLOUD_IMAGE --configuration=storage --project=$PROJECT" -SCP_CMD="$PGCLOUD compute scp" - +# +# For help and usage information, simply execute the script w/o any arguments. +# +# This script is intended to be run by Red Hat storage developers who need +# to debug problems specifically related to Cirrus-CI automated testing. +# It requires that you have been granted prior access to create VMs in +# google-cloud. For non-Red Hat contributors, VMs are available as-needed, +# with supervision upon request. -showrun() { - if [[ "$1" == "--background" ]] - then - shift - # Properly escape any nested spaces, so command can be copy-pasted - echo '+ '$(printf " %q" "$@")' &' > /dev/stderr - "$@" & - echo -e "${RED}${NOR}" - else - echo '+ '$(printf " %q" "$@") > /dev/stderr - "$@" - fi -} - -cleanup() { - RET=$? - set +e - wait - - # set GCLOUD_DEBUG to leave tmpdir behind for postmortem - test -z "$GCLOUD_DEBUG" && rm -rf $TMPDIR - - # Not always called from an exit handler, but should always exit when called - exit $RET -} -trap cleanup EXIT - -delvm() { - echo -e "\n" - echo -e "\n${YEL}Offering to Delete $VMNAME ${RED}(Might take a minute or two)${NOR}" - echo -e "\n${YEL}Note: It's safe to answer N, then re-run script again later.${NOR}" - showrun $CLEANUP_CMD # prompts for Yes/No - cleanup -} - -show_usage() { - echo -e "\n${RED}ERROR: $1${NOR}" - echo -e "${YEL}Usage: $(basename $0) ${NOR}" - echo "" - if [[ -r ".cirrus.yml" ]] - then - echo -e "${YEL}Some possible image_name values (from .cirrus.yml):${NOR}" - image_hints - echo "" - fi - exit 1 -} - -get_env_vars() { - # Deal with both YAML and embedded shell-like substitutions in values - # if substitution fails, fall back to printing naked env. var as-is. - python3 -c ' -import yaml,re -env=yaml.load(open(".cirrus.yml"), Loader=yaml.SafeLoader)["env"] -dollar_env_var=re.compile(r"\$(\w+)") -dollarcurly_env_var=re.compile(r"\$\{(\w+)\}") -class ReIterKey(dict): - def __missing__(self, key): - # Cirrus-CI provides some runtime-only env. vars. Avoid - # breaking this hack-script if/when any are present in YAML - return "${0}".format(key) -rep=r"{\1}" # Convert env vars markup to -> str.format_map(re_iter_key) markup -out=ReIterKey() -for k,v in env.items(): - v=str(v) - if "ENCRYPTED" not in v: - out[k]=dollar_env_var.sub(rep, dollarcurly_env_var.sub(rep, v)) -for k,v in out.items(): - print("{0}=\"{1}\"".format(k, v.format_map(out))) - ' -} - -image_hints() { - get_env_vars | fgrep '_CACHE_IMAGE_NAME' | awk -F "=" '{print $2}' -} - -parse_args(){ - echo -e "$USAGE_WARNING" +set -e - if [[ "$USER" =~ "root" ]] - then - show_usage "This script must be run as a regular user." - fi +SCRIPT_FILEPATH=$(realpath "${BASH_SOURCE[0]}") +SCRIPT_DIRPATH=$(dirname "$SCRIPT_FILEPATH") +REPO_DIRPATH=$(realpath "$SCRIPT_DIRPATH/../") - ENVS="$(get_env_vars | tr [:space:] ' ')" - IMAGE_NAME="$1" - if [[ -z "$IMAGE_NAME" ]] - then - show_usage "No image-name specified." +# Help detect if we were called by get_ci_vm container +GET_CI_VM="${GET_CI_VM:-0}" +in_get_ci_vm() { + if ((GET_CI_VM==0)); then + echo "Error: $1 is not intended for use in this context" + exit 2 fi - - ENVS="$ENVS TEST_DRIVER=\"vfs\"" - SETUP_CMD="env $ENVS $GOSRC/contrib/cirrus/setup.sh" - VMNAME="${VMNAME:-${USER}-${IMAGE_NAME}}" - CREATE_CMD="$PGCLOUD compute instances create --zone=$ZONE --image-project=libpod-218412 --image=${IMAGE_NAME} --custom-cpu=$CPUS --custom-memory=$MEMORY --boot-disk-size=$DISK --labels=in-use-by=$USER $VMNAME" - SSH_CMD="$PGCLOUD compute ssh $SSHUSER@$VMNAME" - CLEANUP_CMD="$PGCLOUD compute instances delete --zone $ZONE --delete-disks=all $VMNAME" } -##### main - -[[ "${STORAGEROOT%%${STORAGEROOT##$HOME}}" == "$HOME" ]] || \ - show_usage "Repo clone must be sub-dir of $HOME" - -cd "$STORAGEROOT" - -parse_args "$@" - -# Ensure mount-points and data directories exist on host as $USER. Also prevents -# permission-denied errors during cleanup() b/c `sudo podman` created mount-points -# owned by root. -mkdir -p $TMPDIR/${STORAGEROOT##$HOME} -mkdir -p $TMPDIR/.ssh -mkdir -p {$HOME,$TMPDIR}/.config/gcloud/ssh -chmod 700 {$HOME,$TMPDIR}/.config/gcloud/ssh $TMPDIR/.ssh - -cd $STORAGEROOT - -# Attempt to determine if named 'storage' gcloud configuration exists -showrun $PGCLOUD info > $TMPDIR/gcloud-info -if egrep -q "Account:.*None" $TMPDIR/gcloud-info -then - echo -e "\n${YEL}WARNING: Can't find gcloud configuration for 'storage', running init.${NOR}" - echo -e " ${RED}Please choose '#1: Re-initialize' and 'login' if asked.${NOR}" - echo -e " ${RED}Please set Compute Region and Zone (if asked) to 'us-central1-b'.${NOR}" - echo -e " ${RED}DO NOT set any password for the generated ssh key.${NOR}" - showrun $PGCLOUD init --project=$PROJECT --console-only --skip-diagnostics - - # Verify it worked (account name == someone@example.com) - $PGCLOUD info > $TMPDIR/gcloud-info-after-init - if egrep -q "Account:.*None" $TMPDIR/gcloud-info-after-init - then - echo -e "${RED}ERROR: Could not initialize 'storage' configuration in gcloud.${NOR}" - exit 5 - fi - - # If this is the only config, make it the default to avoid persistent warnings from gcloud - [[ -r "$HOME/.config/gcloud/configurations/config_default" ]] || \ - ln "$HOME/.config/gcloud/configurations/config_storage" \ - "$HOME/.config/gcloud/configurations/config_default" +# get_ci_vm APIv1 container entrypoint calls into this script +# to obtain required repo. specific configuration options. +if [[ "$1" == "--config" ]]; then + in_get_ci_vm "$1" + cat < /dev/stderr + ./contrib/cirrus/setup.sh +else + # Create and access VM for specified Cirrus-CI task + mkdir -p $HOME/.config/gcloud/ssh + podman run -it --rm \ + --tz=local \ + -e NAME="$USER" \ + -e SRCDIR=/src \ + -e GCLOUD_ZONE="$GCLOUD_ZONE" \ + -e DEBUG="${DEBUG:-0}" \ + -v $REPO_DIRPATH:/src:O \ + -v $HOME/.config/gcloud:/root/.config/gcloud:z \ + -v $HOME/.config/gcloud/ssh:/root/.ssh:z \ + quay.io/libpod/get_ci_vm:latest "$@" fi - -# Couldn't make rsync work with gcloud's ssh wrapper: ssh-keys generated on the fly -TARBALL=$VMNAME.tar.bz2 -echo -e "\n${YEL}Packing up local repository into a tarball.${NOR}" -showrun --background tar cjf $TMPDIR/$TARBALL --warning=no-file-changed --exclude-vcs-ignores -C $STORAGEROOT . - -trap delvm INT # Allow deleting VM if CTRL-C during create -# This fails if VM already exists: permit this usage to re-init -echo -e "\n${YEL}Trying to create a VM named $VMNAME\n${RED}(might take a minute/two. Errors ignored).${NOR}" -showrun $CREATE_CMD || true # allow re-running commands below when "delete: N" - -# Any subsequent failure should prompt for VM deletion -trap delvm EXIT - -echo -e "\n${YEL}Retrying for 30s for ssh port to open (may give some errors)${NOR}" -trap 'COUNT=9999' INT -ATTEMPTS=10 -for (( COUNT=1 ; COUNT <= $ATTEMPTS ; COUNT++ )) -do - if $SSH_CMD --command "true"; then break; else sleep 3s; fi -done -if (( COUNT > $ATTEMPTS )) -then - echo -e "\n${RED}Failed${NOR}" - exit 7 -fi -echo -e "${YEL}Got it${NOR}" - -echo -e "\n${YEL}Removing and re-creating $GOSRC on $VMNAME.${NOR}" -showrun $SSH_CMD --command "rm -rf $GOSRC" -showrun $SSH_CMD --command "mkdir -p $GOSRC" - -echo -e "\n${YEL}Transferring tarball to $VMNAME.${NOR}" -wait -showrun $SCP_CMD $HOME/$TARBALL $SSHUSER@$VMNAME:/tmp/$TARBALL - -echo -e "\n${YEL}Unpacking tarball into $GOSRC on $VMNAME.${NOR}" -showrun $SSH_CMD --command "tar xjf /tmp/$TARBALL -C $GOSRC" - -echo -e "\n${YEL}Removing tarball on $VMNAME.${NOR}" -showrun $SSH_CMD --command "rm -f /tmp/$TARBALL" - -echo -e "\n${YEL}Executing environment setup${NOR}" -showrun $SSH_CMD --command "$SETUP_CMD" - -VMIP=$($PGCLOUD compute instances describe $VMNAME --format='get(networkInterfaces[0].accessConfigs[0].natIP)') - -echo -e "\n${YEL}Connecting to $VMNAME${NOR}\nPublic IP Address: $VMIP\n${RED}(option to delete VM upon logout).${NOR}\n" -showrun $SSH_CMD -- -t "cd $GOSRC && exec env $ENVS bash -il"