Skip to content

Commit

Permalink
Merge pull request #1286 from stackhpc/2023.1-zed-merge
Browse files Browse the repository at this point in the history
2023.1: zed merge
  • Loading branch information
markgoddard authored Sep 12, 2024
2 parents e474210 + acc31e9 commit 20c34c9
Show file tree
Hide file tree
Showing 10 changed files with 240 additions and 7 deletions.
9 changes: 7 additions & 2 deletions .github/path-filters.yml
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
# This file is a list of path filters for the PR workflow in .github/workflows/stackhpc-pull-request.yml.
aio:
aio: &aio
- '.automation'
- '.automation.conf/config.sh'
- '.automation.conf/tempest/load-lists/default'
Expand All @@ -20,6 +20,11 @@ aio:
- 'kayobe-env'
- 'requirements.txt'
- 'terraform/aio/**'
check-tags:
check-tags: &check-tags
- '.github/workflows/stackhpc-check-tags.yml'
- 'etc/kayobe/kolla-image-tags.yml'
- 'etc/kayobe/pulp.yml'
- 'tools/kolla-images.py'
build-kayobe-image:
- *aio
- *check-tags
9 changes: 9 additions & 0 deletions .github/workflows/stackhpc-check-tags.yml
Original file line number Diff line number Diff line change
Expand Up @@ -46,6 +46,15 @@ jobs:
run: |
docker image pull $KAYOBE_IMAGE
- name: Check kolla-images.py image map and tag hierarchy
run: |
docker run -t --rm \
-v $(pwd):/stack/kayobe-automation-env/src/kayobe-config \
-e KAYOBE_ENVIRONMENT -e KAYOBE_VAULT_PASSWORD -e KAYOBE_AUTOMATION_SSH_PRIVATE_KEY \
$KAYOBE_IMAGE \
/stack/kayobe-automation-env/src/kayobe-config/.automation/pipeline/playbook-run.sh \
'$KAYOBE_CONFIG_PATH/ansible/check-kolla-images-py.yml'
- name: Check container image tags
run: |
docker run -t --rm \
Expand Down
3 changes: 2 additions & 1 deletion .github/workflows/stackhpc-pull-request.yml
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,7 @@ jobs:
if: github.repository == 'stackhpc/stackhpc-kayobe-config'
outputs:
aio: ${{ steps.changes.outputs.aio }}
build-kayobe-image: ${{ steps.changes.outputs.build-kayobe-image }}
check-tags: ${{ steps.changes.outputs.check-tags }}
steps:
- name: GitHub Checkout
Expand Down Expand Up @@ -74,7 +75,7 @@ jobs:
- check-changes
uses: ./.github/workflows/stackhpc-build-kayobe-image.yml
with:
if: ${{ needs.check-changes.outputs.aio == 'true' }}
if: ${{ needs.check-changes.outputs.build-kayobe-image == 'true' }}
if: github.repository == 'stackhpc/stackhpc-kayobe-config'

check-tags:
Expand Down
35 changes: 35 additions & 0 deletions etc/kayobe/ansible/check-kolla-images-py.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,35 @@
---
- name: Check kolla-images.py image map and tag hierarchy
hosts: localhost
gather_facts: false
tasks:
- name: Create a temporary directory
ansible.builtin.tempfile:
state: directory
suffix: kolla-ansible
register: tempdir_result

- name: Clone Kolla Ansible repository
ansible.builtin.git:
repo: "{{ stackhpc_kolla_ansible_source_url }}"
version: "{{ stackhpc_kolla_ansible_source_version }}"
dest: "{{ tempdir_result.path }}"

- name: Check image mapping
ansible.builtin.command:
cmd: >-
{{ kayobe_config_path }}/../../tools/kolla-images.py
check-image-map
--kolla-ansible-path {{ tempdir_result.path }}
- name: Check tag hierarchy
ansible.builtin.command:
cmd: >-
{{ kayobe_config_path }}/../../tools/kolla-images.py
check-hierarchy
--kolla-ansible-path {{ tempdir_result.path }}
- name: Remove temporary directory
ansible.builtin.file:
path: "{{ tempdir_result.path }}"
state: absent
1 change: 1 addition & 0 deletions etc/kayobe/ansible/check-tags.yml
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@

- name: Check whether tags exist in Pulp container registry
hosts: localhost
gather_facts: false
tasks:
- name: Query images and tags
command:
Expand Down
2 changes: 1 addition & 1 deletion etc/kayobe/ansible/growroot.yml
Original file line number Diff line number Diff line change
Expand Up @@ -75,7 +75,7 @@
vars:
pv: "{{ pvs.stdout | from_json }}"
disk_tmp: "{{ pv.report[0].pv[0].pv_name[:-1] }}"
disk: "{{ disk_tmp[:-1] if disk_tmp[-1] == 'p' and disk_tmp[:9] == '/dev/nvme' else disk_tmp }}"
disk: "{{ disk_tmp[:-1] if pv.report[0].pv[0].pv_name | regex_search('[a-z0-9]+[0-9]+p[0-9]+') else disk_tmp }}"
part_num: "{{ pv.report[0].pv[0].pv_name[-1] }}"
become: true
failed_when: "growpart.rc != 0 and 'NOCHANGE' not in growpart.stdout"
Expand Down
3 changes: 3 additions & 0 deletions etc/kayobe/kolla-image-tags.yml
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,9 @@ kolla_image_tags:
ironic:
rocky-9: 2023.1-rocky-9-20240906T144646
ubuntu-jammy: 2023.1-ubuntu-jammy-20240906T144646
ironic_dnsmasq:
rocky-9: 2023.1-rocky-9-20240709T132012
ubuntu-jammy: 2023.1-ubuntu-jammy-20240621T104542
kolla_toolbox:
rocky-9: 2023.1-rocky-9-20240809T102431
letsencrypt:
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,5 @@
---
fixes:
- |
Fixes a regression when using ``growroot.yml`` and software raid where the
playbook would fail to identify the correct disk.
64 changes: 61 additions & 3 deletions tools/kolla-images.py
Original file line number Diff line number Diff line change
Expand Up @@ -39,15 +39,20 @@

# Maps a Kolla image to a list of containers that use the image.
IMAGE_TO_CONTAINERS_EXCEPTIONS: Dict[str, List[str]] = {
"dnsmasq": [
"ironic_dnsmasq",
],
"haproxy": [
"glance_tls_proxy",
"haproxy",
"neutron_tls_proxy",
],
"mariadb-server": [
"mariadb",
"mariabackup",
],
"neutron-eswitchd": [
"neutron-mlnx-agent": [
"neutron_eswitchd",
"neutron_mlnx_agent",
],
"neutron-metadata-agent": [
Expand All @@ -58,6 +63,15 @@
"nova_super_conductor",
"nova_conductor",
],
"openvswitch-db-server": [
"openvswitch_db",
],
"ovn-nb-db-server": [
"ovn_nb_db",
],
"ovn-sb-db-server": [
"ovn_sb_db",
],
"prometheus-v2-server": [
"prometheus_server",
],
Expand Down Expand Up @@ -96,6 +110,9 @@ def parse_args() -> argparse.Namespace:
parser.add_argument("--base-distros", default=",".join(SUPPORTED_BASE_DISTROS), choices=SUPPORTED_BASE_DISTROS)
subparsers = parser.add_subparsers(dest="command", required=True)

subparser = subparsers.add_parser("check-image-map", help="Check image mapping against kolla-ansible")
subparser.add_argument("--kolla-ansible-path", required=True, help="Path to kolla-ansible repostory checked out to correct branch")

subparser = subparsers.add_parser("check-hierarchy", help="Check tag variable hierarchy against kolla-ansible")
subparser.add_argument("--kolla-ansible-path", required=True, help="Path to kolla-ansible repostory checked out to correct branch")

Expand All @@ -114,7 +131,7 @@ def parse_args() -> argparse.Namespace:
return parser.parse_args()


def get_abs_path(relative_path: str) -> str:
def get_abs_path(relative_path: str) -> pathlib.Path:
"""Return the absolute path of a file in SKC."""
script_path = pathlib.Path(inspect.getfile(inspect.currentframe()))
return script_path.parent.parent / relative_path
Expand Down Expand Up @@ -277,6 +294,45 @@ def check_tags(base_distros: List[str], kolla_image_tags: KollaImageTags, regist
sys.exit(1)


def check_image_map(kolla_ansible_path: str):
"""Check the image mapping against Kolla Ansible variables.
The *_image variables in Kolla Ansible define the mapping between
containers and images. Ensure that the mapping defined in this script
matches the one in Kolla Ansible.
"""
supported_images = read_images("etc/kayobe/pulp.yml")
assert supported_images
# Build a map from container to image name.
cmd = """git grep -h '^[a-z0-9_]*_image:' ansible/roles/*/defaults/main.yml"""
image_map_str = subprocess.check_output(cmd, shell=True, cwd=os.path.realpath(kolla_ansible_path))
image_map = yaml.safe_load(image_map_str)
image_var_re = re.compile(r"^([a-z0-9_]+)_image$")
image_map = {
image_var_re.match(image_var).group(1): image.split("/")[-1]
for image_var, image in image_map.items()
}
# Filter out unsupported images.
image_map = {
container: image
for container, image in image_map.items()
if image in supported_images
}
assert image_map
errors = []
# Check that our mapping is correct.
for container, image in image_map.items():
containers = get_containers(image)
if container not in containers:
errors.append((container, image))
if errors:
print("Errors:")
for tag_var, image in errors:
print(f"Expected {tag_var} container to use {image} image")
if errors:
sys.exit(1)


def check_hierarchy(kolla_ansible_path: str):
"""Check the tag variable hierarchy against Kolla Ansible variables."""
cmd = """git grep -h '^[a-z0-9_]*_tag:' ansible/roles/*/defaults/main.yml"""
Expand Down Expand Up @@ -352,7 +408,9 @@ def main():

validate(kolla_image_tags)

if args.command == "check-hierarchy":
if args.command == "check-image-map":
check_image_map(args.kolla_ansible_path)
elif args.command == "check-hierarchy":
check_hierarchy(args.kolla_ansible_path)
elif args.command == "check-tags":
check_tags(base_distros, kolla_image_tags, args.registry, args.namespace)
Expand Down
116 changes: 116 additions & 0 deletions tools/merge.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,116 @@
#!/usr/bin/python3

DESCRIPTION = """
This script merges one release branch of SKC into another.
Example 1: Merge stackhpc/yoga into stackhpc/zed:
merge.py yoga zed
Example 2: Merge the branch created in example 1 into stackhpc/2023.1:
merge.py zed 2023.1 zed-yoga-merge
Example 3: Continue after manually resolving merge conflicts seen in example 2:
merge.py zed 2023.1 zed-yoga-merge --continue
"""

import argparse
import os
from subprocess import check_call, check_output
import sys


def command(cmd):
print("Running:", cmd)
check_call(cmd)


def parse_args():
parser = argparse.ArgumentParser(description=DESCRIPTION, formatter_class=argparse.RawDescriptionHelpFormatter)
#"Merge one branch of SKC into the next")
parser.add_argument("previous", type=str, help="The previous version")
parser.add_argument("current", type=str, help="The current version")
parser.add_argument("previous_branch", type=str, nargs="?", default=None, help="Optional branch to use as the previous release. Allows merging multiple branches in parallel.")
parser.add_argument("--continue", dest="cont", action="store_true", help="Continue after merge conflicts have been resolved.")
parser.add_argument("--remote", type=str, default="origin", help="Git remote")
return parser.parse_args()


def fetch(args):
command(["git", "fetch", args.remote])


def checkout(args):
merge_branch = f"{args.current}-{args.previous}-merge"
current_branch = f"{args.remote}/stackhpc/{args.current}"
command(["git", "checkout", "-B", merge_branch, current_branch])


def update_submodules():
command(["git", "submodule", "update"])


def merge_in_progress():
repo_root = check_output(["git", "rev-parse", "--show-toplevel"])
repo_root = repo_root.decode().strip()
return os.path.isfile(os.path.join(repo_root, ".git", "MERGE_HEAD"))


def uncommitted_changes():
unstaged = check_output(["git", "diff"])
staged = check_output(["git", "diff", "--cached"])
return unstaged or staged


def continue_merge():
if merge_in_progress():
command(["git", "merge", "--continue"])
else:
print("No merge in progress")


def merge(args):
if args.previous_branch:
previous_branch = args.previous_branch
else:
previous_branch = f"{args.remote}/stackhpc/{args.previous}"
commit_message = f"Merge stackhpc/{args.previous} into stackhpc/{args.current}"
command(["git", "merge", previous_branch, "-m", commit_message])


def show_diff(args):
print("Proposed changes:")
current_branch = f"{args.remote}/stackhpc/{args.current}"
command(["git", "diff", current_branch])


def create_pr(args):
current_branch = f"stackhpc/{args.current}"
pr_title = f"{args.current}: {args.previous} merge"
command(["gh", "pr", "create", "-f", "-a", "@me", "-B", current_branch, "-t", pr_title])


def main():
args = parse_args()
if args.cont:
continue_merge()
else:
if merge_in_progress():
print("Merge in progress - did you miss the --continue argument?")
sys.exit(1)
if uncommitted_changes():
print("You have uncommitted changes - aborting")
sys.exit(1)
fetch(args)
checkout(args)
update_submodules()
merge(args)
show_diff(args)
create_pr(args)


if __name__ == "__main__":
main()

0 comments on commit 20c34c9

Please sign in to comment.