From a5cb23b98e63958dbf4855de2776239ca72d184c Mon Sep 17 00:00:00 2001
From: Chris Evich <cevich@redhat.com>
Date: Thu, 29 Apr 2021 12:10:58 -0400
Subject: [PATCH] Update to F34 and U2104

Also modernize main library to resemble what's used in other
containers-org repositories.  Lastly, update hack/get_ci_vm.sh to use
the new shared container image.

Signed-off-by: Chris Evich <cevich@redhat.com>
---
 .cirrus.yml           |  10 +-
 contrib/cirrus/lib.sh | 101 +++++++---------
 hack/get_ci_vm.sh     | 269 +++++++++---------------------------------
 3 files changed, 102 insertions(+), 278 deletions(-)

diff --git a/.cirrus.yml b/.cirrus.yml
index 836bf659aa..81e3ef869c 100644
--- a/.cirrus.yml
+++ b/.cirrus.yml
@@ -17,15 +17,15 @@ env:
     ####
     #### Cache-image names to test with (double-quotes around names are critical)
     ###
-    FEDORA_NAME: "fedora-33"
-    PRIOR_FEDORA_NAME: "fedora-32"
-    UBUNTU_NAME: "ubuntu-2010"
-    PRIOR_UBUNTU_NAME: "ubuntu-2004"
+    FEDORA_NAME: "fedora-34"
+    PRIOR_FEDORA_NAME: "fedora-33"
+    UBUNTU_NAME: "ubuntu-2104"
+    PRIOR_UBUNTU_NAME: "ubuntu-2010"
 
     # GCE project where images live
     IMAGE_PROJECT: "libpod-218412"
     # VM Image built in containers/automation_images
-    _BUILT_IMAGE_SUFFIX: "c5744859501821952"
+    _BUILT_IMAGE_SUFFIX: "c6032583541653504"
     FEDORA_CACHE_IMAGE_NAME: "fedora-${_BUILT_IMAGE_SUFFIX}"
     PRIOR_FEDORA_CACHE_IMAGE_NAME: "prior-fedora-${_BUILT_IMAGE_SUFFIX}"
     UBUNTU_CACHE_IMAGE_NAME: "ubuntu-${_BUILT_IMAGE_SUFFIX}"
diff --git a/contrib/cirrus/lib.sh b/contrib/cirrus/lib.sh
index 7c8b94272c..d1519965d2 100755
--- a/contrib/cirrus/lib.sh
+++ b/contrib/cirrus/lib.sh
@@ -3,26 +3,28 @@
 # Library of common, shared utility functions.  This file is intended
 # to be sourced by other scripts, not called directly.
 
-# Global details persist here
-source /etc/environment  # not always loaded under all circumstances
+# BEGIN Global export of all variables
+set -a
 
 # Due to differences across platforms and runtime execution environments,
 # handling of the (otherwise) default shell setup is non-uniform.  Rather
 # than attempt to workaround differences, simply force-load/set required
 # items every time this library is utilized.
-source /etc/profile
-source /etc/environment
 USER="$(whoami)"
-export HOME="$(getent passwd $USER | cut -d : -f 6)"
-[[ -n "$UID" ]] || UID=$(getent passwd $USER | cut -d : -f 3)
-GID=$(getent passwd $USER | cut -d : -f 4)
-
-# During VM Image build, the 'containers/automation' installation
-# was performed.  The final step of installation sets the library
-# location $AUTOMATION_LIB_PATH in /etc/environment or in the
-# default shell profile depending on distribution.
+HOME="$(getent passwd $USER | cut -d : -f 6)"
+# Some platforms set and make this read-only
+[[ -n "$UID" ]] || \
+    UID=$(getent passwd $USER | cut -d : -f 3)
+
+# Automation library installed at image-build time,
+# defining $AUTOMATION_LIB_PATH in this file.
+if [[ -r "/etc/automation_environment" ]]; then
+    source /etc/automation_environment
+fi
+# shellcheck disable=SC2154
 if [[ -n "$AUTOMATION_LIB_PATH" ]]; then
-    source $AUTOMATION_LIB_PATH/common_lib.sh
+        # shellcheck source=/usr/share/automation/lib/common_lib.sh
+        source $AUTOMATION_LIB_PATH/common_lib.sh
 else
     (
     echo "WARNING: It does not appear that containers/automation was installed."
@@ -33,37 +35,34 @@ fi
 
 # Essential default paths, many are overridden when executing under Cirrus-CI
 # others are duplicated here, to assist in debugging.
-export GOPATH="${GOPATH:-/var/tmp/go}"
-if type -P go &> /dev/null
-then
-    # required for go 1.12+
-    export GOCACHE="${GOCACHE:-$HOME/.cache/go-build}"
-    eval "$(go env)"
-    # required by make and other tools
-    export $(go env | cut -d '=' -f 1)
-
-    # Ensure compiled tooling is reachable
-    export PATH="$PATH:$GOPATH/bin"
-fi
+GOPATH="${GOPATH:-/var/tmp/go}"
+GOCACHE="${GOCACHE:-$GOPATH/cache/go-build}"
+# called processes like `make` and other tools need these vars.
+eval "$(go env)"
 CIRRUS_WORKING_DIR="${CIRRUS_WORKING_DIR:-$GOPATH/src/github.com/containers/storage}"
-export GOSRC="${GOSRC:-$CIRRUS_WORKING_DIR}"
-export PATH="$HOME/bin:$GOPATH/bin:/usr/local/bin:$PATH"
+GOSRC="${GOSRC:-$CIRRUS_WORKING_DIR}"
+PATH="$HOME/bin:$GOPATH/bin:/usr/local/bin:$PATH"
 SCRIPT_BASE=${GOSRC}/contrib/cirrus
 
-cd $GOSRC
-if type -P git &> /dev/null
-then
-    CIRRUS_CHANGE_IN_REPO=${CIRRUS_CHANGE_IN_REPO:-$(git show-ref --hash=8 HEAD || date +%s)}
-else # pick something unique and obviously not from Cirrus
-    CIRRUS_CHANGE_IN_REPO=${CIRRUS_CHANGE_IN_REPO:-no_git_$(date +%s)}
-fi
-
-export CI="${CI:-false}"
+CI="${CI:-false}"
 CIRRUS_CI="${CIRRUS_CI:-false}"
+DEST_BRANCH="${DEST_BRANCH:-master}"
 CONTINUOUS_INTEGRATION="${CONTINUOUS_INTEGRATION:-false}"
 CIRRUS_REPO_NAME=${CIRRUS_REPO_NAME:-storage}
-CIRRUS_BASE_SHA=${CIRRUS_BASE_SHA:-unknown$(date +%s)}  # difficult to reliably discover
-CIRRUS_BUILD_ID=${CIRRUS_BUILD_ID:-$RANDOM$(date +%s)}  # must be short and unique
+# Cirrus only sets $CIRRUS_BASE_SHA properly for PRs, but $EPOCH_TEST_COMMIT
+# needs to be set from this value in order for `make validate` to run properly.
+# When running get_ci_vm.sh, most $CIRRUS_xyz variables are empty. Attempt
+# to accomidate both branch and get_ci_vm.sh testing by discovering the base
+# branch SHA value.
+if [[ -z "$CIRRUS_BASE_SHA" ]] && [[ -z "$CIRRUS_TAG" ]]
+then  # Operating on a branch, or under `get_ci_vm.sh`
+    CIRRUS_BASE_SHA=$(git rev-parse ${UPSTREAM_REMOTE:-origin}/$DEST_BRANCH)
+elif [[ -z "$CIRRUS_BASE_SHA" ]]
+then  # Operating on a tag
+    CIRRUS_BASE_SHA=$(git rev-parse HEAD)
+fi
+# The starting place for linting and code validation
+EPOCH_TEST_COMMIT="$CIRRUS_BASE_SHA"
 
 # Unsafe env. vars for display
 SECRET_ENV_RE='(IRCID)|(ACCOUNT)|(^GC[EP]..+)|(SSH)'
@@ -76,13 +75,13 @@ OS_RELEASE_VER="$(source /etc/os-release; echo $VERSION_ID | tr -d '.')"
 OS_REL_VER="${OS_RELEASE_ID}-${OS_RELEASE_VER}"
 
 # Working with dnf + timeout/retry
-export SHORT_DNFY='lilto dnf -y'
-export LONG_DNFY='bigto dnf -y'
+SHORT_DNFY='lilto dnf -y'
+LONG_DNFY='bigto dnf -y'
 # Working with apt under Debian/Ubuntu automation is a PITA, make it easy
 # Avoid some ways of getting stuck waiting for user input
-export DEBIAN_FRONTEND=noninteractive
+DEBIAN_FRONTEND=noninteractive
 # Short-cut for frequently used base command
-export SUDOAPTGET='sudo -E apt-get -q --yes'
+SUDOAPTGET='sudo -E apt-get -q --yes'
 # Short list of packages or quick-running command
 SHORT_APTGET="lilto $SUDOAPTGET"
 # Long list / long-running command
@@ -92,6 +91,9 @@ LONG_APTGET="bigto $SUDOAPTGET"
 RPMS_CONFLICTING="gcc-go"
 DEBS_CONFLICTING=""
 
+# END Global export of all variables
+set +a
+
 bad_os_id_ver() {
     die "Unknown/Unsupported distro. $OS_RELEASE_ID and/or version $OS_RELEASE_VER for $(basename $0)"
 }
@@ -120,18 +122,3 @@ install_bats_from_git(){
     mkdir -p ~/.parallel
     touch ~/.parallel/will-cite
 }
-
-showrun() {
-    if [[ "$1" == "--background" ]]
-    then
-        shift
-        # Properly escape any nested spaces, so command can be copy-pasted
-        msg '+ '$(printf " %q" "$@")' &'
-        "$@" &
-        msg -e "${RED}<backgrounded>${NOR}"
-    else
-        msg '--------------------------------------------------'
-        msg '+ '$(printf " %q" "$@") > /dev/stderr
-        "$@"
-    fi
-}
diff --git a/hack/get_ci_vm.sh b/hack/get_ci_vm.sh
index 3e17210840..8c2c76bf6d 100755
--- a/hack/get_ci_vm.sh
+++ b/hack/get_ci_vm.sh
@@ -1,225 +1,62 @@
 #!/usr/bin/env bash
 
-set -e
-
-RED="\e[1;36;41m"
-YEL="\e[1;33;44m"
-NOR="\e[0m"
-USAGE_WARNING="
-${YEL}WARNING: This will not work without local sudo access to run podman,${NOR}
-         ${YEL}and prior authorization to use the storage GCP project. Also,${NOR}
-         ${YEL}possession of the proper ssh private key is required.${NOR}
-"
-# TODO: Many/most of these values should come from .cirrus.yml
-ZONE="us-central1-b"
-CPUS="2"
-MEMORY="4Gb"
-DISK="200"
-PROJECT="storage-240716"
-GOSRC="/var/tmp/go/src/github.com/containers/storage"
-GCLOUD_IMAGE=${GCLOUD_IMAGE:-quay.io/cevich/gcloud_centos:latest}
-GCLOUD_SUDO=${GCLOUD_SUDO-sudo}
-SSHUSER="root"
-
-# Shared tmp directory between container and us
-TMPDIR=$(mktemp -d --tmpdir $(basename $0)_tmpdir_XXXXXX)
-
-STORAGEROOT=$(realpath "$(dirname $0)/../")
-# else: Assume $PWD is the root of the storage repository
-[[ "$STORAGEROOT" != "/" ]] || STORAGEROOT=$PWD
-
-# Command shortcuts save some typing (assumes $STORAGEROOT is subdir of $HOME)
-PGCLOUD="$GCLOUD_SUDO podman run -it --rm -e AS_ID=$UID -e AS_USER=$USER --security-opt label=disable -v $TMPDIR:$HOME -v $HOME/.config/gcloud:$HOME/.config/gcloud -v $HOME/.config/gcloud/ssh:$HOME/.ssh -v $STORAGEROOT:$STORAGEROOT $GCLOUD_IMAGE --configuration=storage --project=$PROJECT"
-SCP_CMD="$PGCLOUD compute scp"
-
+#
+# For help and usage information, simply execute the script w/o any arguments.
+#
+# This script is intended to be run by Red Hat storage developers who need
+# to debug problems specifically related to Cirrus-CI automated testing.
+# It requires that you have been granted prior access to create VMs in
+# google-cloud.  For non-Red Hat contributors, VMs are available as-needed,
+# with supervision upon request.
 
-showrun() {
-    if [[ "$1" == "--background" ]]
-    then
-        shift
-        # Properly escape any nested spaces, so command can be copy-pasted
-        echo '+ '$(printf " %q" "$@")' &' > /dev/stderr
-        "$@" &
-        echo -e "${RED}<backgrounded>${NOR}"
-    else
-        echo '+ '$(printf " %q" "$@") > /dev/stderr
-        "$@"
-    fi
-}
-
-cleanup() {
-    RET=$?
-    set +e
-    wait
-
-    # set GCLOUD_DEBUG to leave tmpdir behind for postmortem
-    test -z "$GCLOUD_DEBUG" && rm -rf $TMPDIR
-
-    # Not always called from an exit handler, but should always exit when called
-    exit $RET
-}
-trap cleanup EXIT
-
-delvm() {
-    echo -e "\n"
-    echo -e "\n${YEL}Offering to Delete $VMNAME ${RED}(Might take a minute or two)${NOR}"
-    echo -e "\n${YEL}Note: It's safe to answer N, then re-run script again later.${NOR}"
-    showrun $CLEANUP_CMD  # prompts for Yes/No
-    cleanup
-}
-
-show_usage() {
-    echo -e "\n${RED}ERROR: $1${NOR}"
-    echo -e "${YEL}Usage: $(basename $0) <image_name>${NOR}"
-    echo ""
-    if [[ -r ".cirrus.yml" ]]
-    then
-        echo -e "${YEL}Some possible image_name values (from .cirrus.yml):${NOR}"
-        image_hints
-        echo ""
-    fi
-    exit 1
-}
-
-get_env_vars() {
-    # Deal with both YAML and embedded shell-like substitutions in values
-    # if substitution fails, fall back to printing naked env. var as-is.
-    python3 -c '
-import yaml,re
-env=yaml.load(open(".cirrus.yml"), Loader=yaml.SafeLoader)["env"]
-dollar_env_var=re.compile(r"\$(\w+)")
-dollarcurly_env_var=re.compile(r"\$\{(\w+)\}")
-class ReIterKey(dict):
-    def __missing__(self, key):
-        # Cirrus-CI provides some runtime-only env. vars.  Avoid
-        # breaking this hack-script if/when any are present in YAML
-        return "${0}".format(key)
-rep=r"{\1}"  # Convert env vars markup to -> str.format_map(re_iter_key) markup
-out=ReIterKey()
-for k,v in env.items():
-    v=str(v)
-    if "ENCRYPTED" not in v:
-        out[k]=dollar_env_var.sub(rep, dollarcurly_env_var.sub(rep, v))
-for k,v in out.items():
-    print("{0}=\"{1}\"".format(k, v.format_map(out)))
-    '
-}
-
-image_hints() {
-    get_env_vars | fgrep '_CACHE_IMAGE_NAME' | awk -F "=" '{print $2}'
-}
-
-parse_args(){
-    echo -e "$USAGE_WARNING"
+set -e
 
-    if [[ "$USER" =~ "root" ]]
-    then
-        show_usage "This script must be run as a regular user."
-    fi
+SCRIPT_FILEPATH=$(realpath "${BASH_SOURCE[0]}")
+SCRIPT_DIRPATH=$(dirname "$SCRIPT_FILEPATH")
+REPO_DIRPATH=$(realpath "$SCRIPT_DIRPATH/../")
 
-    ENVS="$(get_env_vars | tr [:space:] ' ')"
-    IMAGE_NAME="$1"
-    if [[ -z "$IMAGE_NAME" ]]
-    then
-        show_usage "No image-name specified."
+# Help detect if we were called by get_ci_vm container
+GET_CI_VM="${GET_CI_VM:-0}"
+in_get_ci_vm() {
+    if ((GET_CI_VM==0)); then
+        echo "Error: $1 is not intended for use in this context"
+        exit 2
     fi
-
-    ENVS="$ENVS TEST_DRIVER=\"vfs\""
-    SETUP_CMD="env $ENVS $GOSRC/contrib/cirrus/setup.sh"
-    VMNAME="${VMNAME:-${USER}-${IMAGE_NAME}}"
-    CREATE_CMD="$PGCLOUD compute instances create --zone=$ZONE --image-project=libpod-218412 --image=${IMAGE_NAME} --custom-cpu=$CPUS --custom-memory=$MEMORY --boot-disk-size=$DISK --labels=in-use-by=$USER $VMNAME"
-    SSH_CMD="$PGCLOUD compute ssh $SSHUSER@$VMNAME"
-    CLEANUP_CMD="$PGCLOUD compute instances delete --zone $ZONE --delete-disks=all $VMNAME"
 }
 
-##### main
-
-[[ "${STORAGEROOT%%${STORAGEROOT##$HOME}}" == "$HOME" ]] || \
-    show_usage "Repo clone must be sub-dir of $HOME"
-
-cd "$STORAGEROOT"
-
-parse_args "$@"
-
-# Ensure mount-points and data directories exist on host as $USER.  Also prevents
-# permission-denied errors during cleanup() b/c `sudo podman` created mount-points
-# owned by root.
-mkdir -p $TMPDIR/${STORAGEROOT##$HOME}
-mkdir -p $TMPDIR/.ssh
-mkdir -p {$HOME,$TMPDIR}/.config/gcloud/ssh
-chmod 700 {$HOME,$TMPDIR}/.config/gcloud/ssh $TMPDIR/.ssh
-
-cd $STORAGEROOT
-
-# Attempt to determine if named 'storage' gcloud configuration exists
-showrun $PGCLOUD info > $TMPDIR/gcloud-info
-if egrep -q "Account:.*None" $TMPDIR/gcloud-info
-then
-    echo -e "\n${YEL}WARNING: Can't find gcloud configuration for 'storage', running init.${NOR}"
-    echo -e "         ${RED}Please choose '#1: Re-initialize' and 'login' if asked.${NOR}"
-    echo -e "         ${RED}Please set Compute Region and Zone (if asked) to 'us-central1-b'.${NOR}"
-    echo -e "         ${RED}DO NOT set any password for the generated ssh key.${NOR}"
-    showrun $PGCLOUD init --project=$PROJECT --console-only --skip-diagnostics
-
-    # Verify it worked (account name == someone@example.com)
-    $PGCLOUD info > $TMPDIR/gcloud-info-after-init
-    if egrep -q "Account:.*None" $TMPDIR/gcloud-info-after-init
-    then
-        echo -e "${RED}ERROR: Could not initialize 'storage' configuration in gcloud.${NOR}"
-        exit 5
-    fi
-
-    # If this is the only config, make it the default to avoid persistent warnings from gcloud
-    [[ -r "$HOME/.config/gcloud/configurations/config_default" ]] || \
-        ln "$HOME/.config/gcloud/configurations/config_storage" \
-           "$HOME/.config/gcloud/configurations/config_default"
+# get_ci_vm APIv1 container entrypoint calls into this script
+# to obtain required repo. specific configuration options.
+if [[ "$1" == "--config" ]]; then
+    in_get_ci_vm "$1"
+    cat <<EOF
+DESTDIR="/var/tmp/go/src/github.com/containers/storage"
+UPSTREAM_REPO="https://github.com/containers/storage.git"
+GCLOUD_PROJECT="storage-240716"
+GCLOUD_IMGPROJECT="libpod-218412"
+GCLOUD_CFG="storage"
+GCLOUD_ZONE="${GCLOUD_ZONE:-us-central1-b}"
+GCLOUD_CPUS="2"
+GCLOUD_MEMORY="4Gb"
+GCLOUD_DISK="200"
+EOF
+elif [[ "$1" == "--setup" ]]; then
+    in_get_ci_vm "$1"
+    # get_ci_vm container entrypoint calls us with this option on the
+    # Cirrus-CI environment instance, to perform repo.-specific setup.
+    cd $REPO_DIRPATH
+    echo "+ Running environment setup" > /dev/stderr
+    ./contrib/cirrus/setup.sh
+else
+    # Create and access VM for specified Cirrus-CI task
+    mkdir -p $HOME/.config/gcloud/ssh
+    podman run -it --rm \
+        --tz=local \
+        -e NAME="$USER" \
+        -e SRCDIR=/src \
+        -e GCLOUD_ZONE="$GCLOUD_ZONE" \
+        -e DEBUG="${DEBUG:-0}" \
+        -v $REPO_DIRPATH:/src:O \
+        -v $HOME/.config/gcloud:/root/.config/gcloud:z \
+        -v $HOME/.config/gcloud/ssh:/root/.ssh:z \
+        quay.io/libpod/get_ci_vm:latest "$@"
 fi
-
-# Couldn't make rsync work with gcloud's ssh wrapper: ssh-keys generated on the fly
-TARBALL=$VMNAME.tar.bz2
-echo -e "\n${YEL}Packing up local repository into a tarball.${NOR}"
-showrun --background tar cjf $TMPDIR/$TARBALL --warning=no-file-changed --exclude-vcs-ignores -C $STORAGEROOT .
-
-trap delvm INT  # Allow deleting VM if CTRL-C during create
-# This fails if VM already exists: permit this usage to re-init
-echo -e "\n${YEL}Trying to create a VM named $VMNAME\n${RED}(might take a minute/two.  Errors ignored).${NOR}"
-showrun $CREATE_CMD || true # allow re-running commands below when "delete: N"
-
-# Any subsequent failure should prompt for VM deletion
-trap delvm EXIT
-
-echo -e "\n${YEL}Retrying for 30s for ssh port to open (may give some errors)${NOR}"
-trap 'COUNT=9999' INT
-ATTEMPTS=10
-for (( COUNT=1 ; COUNT <= $ATTEMPTS ; COUNT++ ))
-do
-    if $SSH_CMD --command "true"; then break; else sleep 3s; fi
-done
-if (( COUNT > $ATTEMPTS ))
-then
-    echo -e "\n${RED}Failed${NOR}"
-    exit 7
-fi
-echo -e "${YEL}Got it${NOR}"
-
-echo -e "\n${YEL}Removing and re-creating $GOSRC on $VMNAME.${NOR}"
-showrun $SSH_CMD --command "rm -rf $GOSRC"
-showrun $SSH_CMD --command "mkdir -p $GOSRC"
-
-echo -e "\n${YEL}Transferring tarball to $VMNAME.${NOR}"
-wait
-showrun $SCP_CMD $HOME/$TARBALL $SSHUSER@$VMNAME:/tmp/$TARBALL
-
-echo -e "\n${YEL}Unpacking tarball into $GOSRC on $VMNAME.${NOR}"
-showrun $SSH_CMD --command "tar xjf /tmp/$TARBALL -C $GOSRC"
-
-echo -e "\n${YEL}Removing tarball on $VMNAME.${NOR}"
-showrun $SSH_CMD --command "rm -f /tmp/$TARBALL"
-
-echo -e "\n${YEL}Executing environment setup${NOR}"
-showrun $SSH_CMD --command "$SETUP_CMD"
-
-VMIP=$($PGCLOUD compute instances describe $VMNAME --format='get(networkInterfaces[0].accessConfigs[0].natIP)')
-
-echo -e "\n${YEL}Connecting to $VMNAME${NOR}\nPublic IP Address: $VMIP\n${RED}(option to delete VM upon logout).${NOR}\n"
-showrun $SSH_CMD -- -t "cd $GOSRC && exec env $ENVS bash -il"