Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Feature/explicit updates #286

Merged
merged 3 commits into from
Jun 4, 2024
Merged

Feature/explicit updates #286

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
c9f071c
remove extra dependencies from ciab install stage
msherman64 May 23, 2024
c6d8429
use install_deps instead of auto-updating
msherman64 May 23, 2024
18c4646
use git submodule to pin kolla-ansible version
msherman64 May 23, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 3 additions & 0 deletions .gitmodules
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,3 @@
[submodule "src/kolla-ansible"]
path = src/kolla-ansible
url = https://github.com/chameleoncloud/kolla-ansible
155 changes: 83 additions & 72 deletions cc-ansible
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,25 +3,16 @@
set -o errexit
set -o nounset
set -o pipefail
# set -o xtrace

DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null && pwd)"
VIRTUALENV="$(realpath "${VIRTUALENV:-"$DIR"/venv}")"

FORCE_UPDATES="${FORCE_UPDATES:-no}"
if [[ ! -d "$VIRTUALENV" ]]; then
echo "Creating virtualenv at $VIRTUALENV ..."
python3 -m venv "$VIRTUALENV"
"$VIRTUALENV/bin/pip" install --upgrade pip
FORCE_UPDATES=yes
if [ -f "${VIRTUALENV}/bin/activate" ]; then
# shellcheck source=/dev/null
source "${VIRTUALENV}/bin/activate"
# sourcing this will set the VIRTUAL_ENV, PATH, and PYTHON_HOME env vars
fi

set +o nounset
# shellcheck disable=1090 # TODO use standard VIRTUAL_ENV and PATH method
source "$(realpath "$VIRTUALENV/bin/activate")"
set -o nounset

CHECK_UPDATES=yes
declare -a POSARGS=()
declare -a TRAPS=()
cleanup() {
Expand All @@ -45,19 +36,13 @@
# Add proper flag support for --check as an option for dry-runs
export EXTRA_OPTS="${EXTRA_OPTS:-} --check"
;;
decrypt_passwords|edit_passwords|help|init|view_passwords)
decrypt_passwords|edit_passwords|help|install_deps|init|view_passwords)
# Special subcommand!
command="$key"
;;
-h|--help)
command=help
;;
--no-update)
CHECK_UPDATES=no
;;
--force-update)
FORCE_UPDATES=yes
;;
*)
POSARGS+=("$key")
;;
Expand All @@ -70,6 +55,56 @@
# Subcommands
#

install_deps() {
# install kolla-ansible, perform updates if necessary.

# create virtualenv
python3 -m venv "$VIRTUALENV"
# shellcheck source=/dev/null
source "${VIRTUALENV}/bin/activate"

# upgrade pip and build tools
pip install --upgrade \
pip \
setuptools \
wheel

local pip_requirements="$DIR/requirements.txt"
pip install -r "${pip_requirements}"

# update Ansible Galaxy roles and collections
local galaxy_requirements="$DIR/requirements.yml"
local galaxy_role_path="$DIR/galaxy.ansible.com/ansible_roles/"
ansible-galaxy role install --force -p "$galaxy_role_path" -r "$galaxy_requirements"
local galaxy_collection_path="$DIR/galaxy.ansible.com/"
ansible-galaxy collection install --force -p "$galaxy_collection_path" -r "$galaxy_requirements"

local submodule_target_commit="$(git submodule status --cached src/kolla-ansible/)"
Copy link

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

⚠️ [shellcheck] reported by reviewdog 🐶
Declare and assign separately to avoid masking return values. SC2155


echo "Checking if git submodules need updating"
if [ -z "$(git status --porcelain src/kolla-ansible)" ]; then
git submodule update --init src/kolla-ansible
else
echo "WARNING: you've made local changes to src/kolla-ansible, running install_deps will check out commit ${submodule_target_commit}"
echo "To proceed, either run 'git submodule update --init src/kolla-ansible', or commit the new submodule hash to chi-in-a-box"
git status src/kolla-ansible
return 1
fi

pip install \
-r requirements.txt \
--config-settings editable_mode=strict \
-e src/kolla-ansible

# Update/install yq, adding it to the venv bin path
YQ_VERSION=4.9.6
if [[ "$(type -t yq)" != "file" ]]; then
YQ_BINARY="yq_linux_amd64"
wget https://github.com/mikefarah/yq/releases/download/v${YQ_VERSION}/${YQ_BINARY}.tar.gz -O - \
| tar xz && mv ${YQ_BINARY} "$VIRTUALENV/bin/yq"
fi
}

init() {
local site_config="$CC_ANSIBLE_SITE"

Expand Down Expand Up @@ -115,8 +150,8 @@

tmpfile="$(mktemp)"
_edit_passwords_cleanup() {
rm -f "$tmpfile"

Check warning on line 153 in cc-ansible

View workflow job for this annotation

GitHub Actions / check cc-ansible 

[shellcheck] reported by reviewdog 🐶
Command appears to be unreachable. Check usage (or ignore if invoked indirectly). [SC2317](https://github.com/koalaman/shellcheck/wiki/SC2317)

Raw Output:
./cc-ansible:153:5:info:Command appears to be unreachable. Check usage (or ignore if invoked indirectly). [SC2317](https://github.com/koalaman/shellcheck/wiki/SC2317)
rm -f "$passwords_file_chksum"

Check warning on line 154 in cc-ansible

View workflow job for this annotation

GitHub Actions / check cc-ansible 

[shellcheck] reported by reviewdog 🐶
Command appears to be unreachable. Check usage (or ignore if invoked indirectly). [SC2317](https://github.com/koalaman/shellcheck/wiki/SC2317)

Raw Output:
./cc-ansible:154:5:info:Command appears to be unreachable. Check usage (or ignore if invoked indirectly). [SC2317](https://github.com/koalaman/shellcheck/wiki/SC2317)
}
TRAPS+=(_edit_passwords_cleanup)

Expand Down Expand Up @@ -187,10 +222,26 @@
exit 1
}

# install_deps creates the virtualenv. Exit early if it's not set up
if [ "${command:-}" == "install_deps" ]; then
:
elif [ -z "${VIRTUAL_ENV+x}" ]; then
# VIRTUAL_ENV is present if we have sourced a venv/bin/activate
cat <<ERRMSG
Error: virtualenv is not set up. Please create it by running:
./cc-ansible install_deps
ERRMSG
exit 1
else
echo "Using venv ${VIRTUAL_ENV}"
fi

# On init, there is no requirement that the site config directory be defined,
# because one of init's tasks is creating this directory. Set a default.
if [[ "${command:-}" == "init" ]]; then
if [[ "${command:-}" == "install_deps" ]]; then
# if we're installing deps, this is a no-op, but prevents unset variables
CC_ANSIBLE_SITE="${CC_ANSIBLE_SITE:-$DIR/site-config}"
elif [[ "${command:-}" == "init" ]]; then
CC_ANSIBLE_SITE="${CC_ANSIBLE_SITE:-$DIR/site-config}"
elif [[ -z "${CC_ANSIBLE_SITE:-}" ]]; then
cat <<ERRMSG
Expand All @@ -212,65 +263,25 @@
set -a; source "$(realpath "$CC_ANSIBLE_ENV")"; set +a
fi

# Automatically update dependencies
if [[ "$CHECK_UPDATES" == "yes" || "$FORCE_UPDATES" == "yes" ]]; then
# Update base pip packages
pip_requirements="$DIR/requirements.txt"
pip_requirements_chksum="$VIRTUALENV/requirements.txt.sha256"
if [[ "$FORCE_UPDATES" == "yes" || ! -f "$pip_requirements_chksum" ]] || \
! sha256sum --quiet --check "$pip_requirements_chksum"; then
pip install --upgrade --force-reinstall -r "$pip_requirements"
sha256sum "$pip_requirements" > "$pip_requirements_chksum"
find_kolla_ansible_base_dir () {
kolla_direct_url="$(find ${VIRTUAL_ENV}/lib/python*/site-packages/ -wholename '*kolla_ansible*.dist-info/direct_url.json' -print -quit 1> /dev/null 2>&1)"
msherman64 marked this conversation as resolved.
Show resolved Hide resolved
msherman64 marked this conversation as resolved.
Show resolved Hide resolved
if test -n "${kolla_direct_url}"; then
# Editable install in local path
direct_url="$(yq eval '.url' ${kolla_direct_url})"
msherman64 marked this conversation as resolved.
Show resolved Hide resolved
msherman64 marked this conversation as resolved.
Show resolved Hide resolved
BASEDIR="${direct_url#file:\/\/}"
else
BASEDIR="${VIRTUAL_ENV}/share/kolla-ansible"
fi

# Update Ansible Galaxy roles
galaxy_collection_path="$DIR/galaxy.ansible.com/"
galaxy_role_path="$DIR/galaxy.ansible.com/ansible_roles/"
galaxy_role_requirements="$DIR/requirements.yml"
galaxy_role_requirements_chksum="$galaxy_role_path/requirements.yml.sha256"
if [[ "$FORCE_UPDATES" == "yes" || ! -f "$galaxy_role_requirements_chksum" ]] || \
! sha256sum --quiet --check "$galaxy_role_requirements_chksum"; then
ansible-galaxy role install --force -p "$galaxy_role_path" -r "$galaxy_role_requirements"
ansible-galaxy collection install --force -p "$galaxy_collection_path" -r "$galaxy_role_requirements"
sha256sum "$galaxy_role_requirements" > "$galaxy_role_requirements_chksum"
fi

kolla_ansible_remote=https://github.com/chameleoncloud/kolla-ansible.git
kolla_ansible_checkout=${KOLLA_ANSIBLE_BRANCH:-chameleoncloud/xena}
kolla_ansible_gitref="$VIRTUALENV/kolla-ansible.gitref"
kolla_ansible_egglink="$VIRTUALENV/src/kolla-ansible"
if [[ "$FORCE_UPDATES" == "yes" || ! -f "$kolla_ansible_gitref" || ! -d "$kolla_ansible_egglink" ]] || \
! diff -q >/dev/null \
"$kolla_ansible_gitref" \
<(cd "$kolla_ansible_egglink"; git fetch; git show-ref -s -d origin/"$kolla_ansible_checkout"); then
pushd "$VIRTUALENV" || ( echo "pushd error!" && exit 1 )
pip install -e git+"$kolla_ansible_remote"@"$kolla_ansible_checkout"#egg=kolla-ansible
popd || ( echo "popd error!" && exit 1 )
# [jca 2020-01-30] TODO:
# Ensure the /share folder is placed; this is not copied when using the "develop" setup.py method.
# This is a bit weird, perhaps there is some way to pass an additional flag to pip to make it
# copy this even though it's installing as source. We use source install to keep track of the Git revision.
mkdir -p "$VIRTUALENV/share" && ln -sf "$kolla_ansible_egglink" "$VIRTUALENV/share/kolla-ansible"
(cd "$kolla_ansible_egglink"; git rev-parse HEAD > "$kolla_ansible_gitref")
fi

# Update/install yq
YQ_VERSION=4.9.6
if [[ "$(type -t yq)" != "file" ]]; then
YQ_BINARY="yq_linux_amd64"
wget https://github.com/mikefarah/yq/releases/download/v${YQ_VERSION}/${YQ_BINARY}.tar.gz -O - \
| tar xz && mv ${YQ_BINARY} "$VIRTUALENV/bin/yq"
fi
fi

ansible_path="$VIRTUALENV/share/kolla-ansible/ansible"
echo "${BASEDIR}/ansible"
}

# Handle subcommands
if [[ -n "${command:-}" ]]; then
$command "$@"
exit $?
fi

ansible_path="$(find_kolla_ansible_base_dir)"
if [[ -n "${CC_ANSIBLE_PLAYBOOK:-}" ]]; then
echo "**********************************************************************"
echo "* Playbook override detected! This playbook will be executed within *"
Expand Down
9 changes: 9 additions & 0 deletions docs/setup-guides/production-baremetal/quickstart.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,15 @@ update-alternatives --install /usr/bin/python python /usr/bin/python3 1
{% endtab %}
{% endtabs %}

### Initialize cc-ansible's virtualenv and deps

* cc-ansible will install all of its own dependencies into a python virtualenv, located inside the chi-in-a-box git checkout directory. You'll need to do this before running other `cc-ansible` subcommands.

```bash
./cc-ansible install_deps
```


### Initialize the site configuration

1. Check out this repository:
Expand Down
18 changes: 1 addition & 17 deletions requirements.txt
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,20 +1,4 @@
# Xena release of Kolla-ansible requires ansible release between 2.10 and below 5.x,
# This is because it depends on ansible-core 2.11.x
# See https://docs.openstack.org/releasenotes/kolla-ansible/xena.html#relnotes-13-0-0-stable-xena-upgrade-notes
ansible>=2.10,<5

# ensure the version of pyopenssl installed is compatible with cryptography. May be resolved after ansible 5.x
# https://github.com/pyca/pyopenssl/issues/1114
pyopenssl

# Jinja2 3.1 breaks filters, removes py3.6 support
# See https://jinja.palletsprojects.com/en/3.1.x/changes/#version-3-1-0
jinja2<3.1.0
docker
kubernetes

# Client Tools
python-openstackclient
python-doniclient
python-ironicclient
git+https://github.com/chameleoncloud/python-blazarclient@chameleoncloud/xena
ansible>=2.10,<5
1 change: 1 addition & 0 deletions src/kolla-ansible
Open in desktop
Submodule kolla-ansible added at 9c519e
Loading