Update CI pulp plugin_template (#1170)

No-Issue

.ci/ansible/Containerfile.j2

@@ -15,7 +15,7 @@ RUN pip3 install \
 {{ " " }}django-storages[boto3] git+https://github.com/fabricio-aguiar/botocore.git@fix-100-continue
 {%- endif -%}
 {%- if azure_test | default(false) -%}
-{{ " " }}django-storages[azure]>=1.12.2
+{{ " " }}django-storages[azure]>=1.12.2 "azure-storage-blob<12.10.0"
 {%- endif -%}
 {%- for item in plugins -%}
 {%- if item.name == "pulp-certguard" -%}

.ci/scripts/changelog.py

@@ -0,0 +1,68 @@
+import re
+import os
+import requests
+from packaging.version import Version
+from git import Repo
+
+repo = Repo(os.getcwd())
+heads = repo.git.ls_remote("--heads", "https://github.com/pulp/galaxy_ng.git").split("\n")
+branches = [h.split("/")[-1] for h in heads if re.search(r"^([0-9]+)\.([0-9]+)$", h.split("/")[-1])]
+branches.sort(key=lambda ver: Version(ver), reverse=True)
+
+
+def get_changelog(branch):
+    """
+    Get changelog file for a given branch.
+
+    """
+    return requests.get(
+        f"https://raw.githubusercontent.com/pulp/galaxy_ng/{branch}/CHANGES.rst"
+    ).text
+
+
+def get_changelog_releases(changelog):
+    """
+    Get all versions in changelog.
+
+    """
+    versions = re.findall(r"([0-9]+)\.([0-9]+)\.([0-9]+) \(", changelog)
+    return {".".join(v) for v in versions}
+
+
+def get_changelog_entry(changelog, version):
+    """
+    Get changelog entry for a given version.
+
+    """
+    entries = changelog.split(f"{version} (")[1].split("=====\n")
+    header = f"{version} ({entries[0]}=====\n"
+    text = "\n\n\n".join(entries[1].split("\n\n\n")[0:-1])
+    return header + text + "\n\n\n"
+
+
+main_changelog = get_changelog("master")
+main_entries = get_changelog_releases(main_changelog)
+entries_list = list(main_entries)
+to_add = {}
+for branch in branches:
+    changelog = get_changelog(branch)
+    entries = get_changelog_releases(changelog)
+    for entry in entries.difference(main_entries):
+        description = get_changelog_entry(changelog, entry)
+        entries_list.append(entry)
+        print(description)
+        to_add[entry] = description
+
+entries_list.sort(key=lambda ver: Version(ver), reverse=True)
+for version in sorted(to_add, key=lambda ver: Version(ver)):
+    next_version = entries_list[entries_list.index(version) + 1]
+    new_changelog = main_changelog.split(f"{next_version} (")[0] + to_add[version]
+    new_changelog = new_changelog + f"{next_version} ("
+    new_changelog = new_changelog + main_changelog.split(f"{next_version} (")[1]
+    main_changelog = new_changelog
+
+with open("CHANGES.rst", "w") as f:
+    f.write(main_changelog)
+
+if to_add:
+    repo.git.commit("-m", "Update Changelog\n\n[noissue]", "CHANGES.rst")

.github/template_gitref

@@ -1 +1 @@
-2021.08.26-85-ga3934fa
+2021.08.26-90-gdb75a8f

.github/workflows/ci.yml

@@ -64,7 +64,7 @@ jobs:
         run: flake8 --config flake8.cfg
 
       - name: Run extra lint checks
-        run: "[ -x .ci/scripts/extra_linting.sh ] && .ci/scripts/extra_linting.sh"
+        run: "[ ! -x .ci/scripts/extra_linting.sh ] || .ci/scripts/extra_linting.sh"
 
       # check for any files unintentionally left out of MANIFEST.in
       - name: Check manifest

.github/workflows/nightly.yml

@@ -152,6 +152,49 @@ jobs:
           docker exec pulp cat /etc/yum.repos.d/* || true
           docker exec pulp pip3 list
 
+  changelog:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+        with:
+          # by default, it uses a depth of 1
+          # this fetches all history so that we can read each commit
+          fetch-depth: 0
+
+      - uses: actions/setup-python@v2
+        with:
+          python-version: "3.8"
+
+      - name: Install python dependencies
+        run: |
+          echo ::group::PYDEPS
+          pip install gitpython requests packaging
+          echo ::endgroup::
+
+      - name: Configure Git with ansible name and email
+        run: |
+          git config --global user.name 'ansible'
+          git config --global user.email '[email protected]'
+
+      - name: Changelog history
+        run: python .ci/scripts/changelog.py
+
+      - name: Create Pull Request
+        uses: peter-evans/create-pull-request@v3
+        with:
+          token: ${{ secrets.RELEASE_TOKEN }}
+          committer: ansible <[email protected]>
+          author: ansible <[email protected]>
+          title: 'Update Changelog'
+          body: 'No-Issue'
+          branch: 'changelog/update'
+          base: master
+          commit-message: |
+            Update Changelog
+
+            No-Issue
+          delete-branch: true
+
   publish:
     runs-on: ubuntu-latest
     needs: test

.github/workflows/scripts/install.sh

@@ -167,7 +167,8 @@ sudo update-ca-certificates
 echo ::endgroup::
 
 if [ "$TEST" = "azure" ]; then
-  cat /usr/local/share/ca-certificates/azcert.crt >> /opt/az/lib/python3.6/site-packages/certifi/cacert.pem
+  AZCERTIFI=$(/opt/az/bin/python3 -c 'import certifi; print(certifi.where())')
+  cat /usr/local/share/ca-certificates/azcert.crt >> $AZCERTIFI
   cat /usr/local/share/ca-certificates/azcert.crt | cmd_stdin_prefix tee -a /usr/local/lib/python3.8/site-packages/certifi/cacert.pem > /dev/null
   cat /usr/local/share/ca-certificates/azcert.crt | cmd_stdin_prefix tee -a /etc/pki/tls/cert.pem > /dev/null
   AZURE_STORAGE_CONNECTION_STRING='DefaultEndpointsProtocol=https;AccountName=devstoreaccount1;AccountKey=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==;BlobEndpoint=https://ci-azurite:10000/devstoreaccount1;'