Move from travis to Github Actions CI (#873)

Workflow is now managed by pulp/plugin-template

Issue: AAH-460

.travis/Containerfile.j2 → .ci/ansible/Containerfile.j2

@@ -1,4 +1,4 @@
-FROM {{ ci_base | default("pulp/pulp-ci-centos:latest") }}
+FROM {{ ci_base | default("pulp/pulp-ci-centos:" + pulp_container_tag) }}
 
 # Add source directories to container
 {% for item in plugins %}
@@ -23,7 +23,8 @@ RUN pip3 install \
 
 RUN mkdir -p /etc/nginx/pulp/
 {% for item in plugins %}
-RUN ln /usr/local/lib/python3.8/site-packages/{{ item.name }}/app/webserver_snippets/nginx.conf /etc/nginx/pulp/{{ item.name }}.conf || true
+RUN export plugin_path="$(pip3 show {{ item.name }} | sed -n -e 's/Location: //p')/{{ item.name }}" && \
+    ln $plugin_path/app/webserver_snippets/nginx.conf /etc/nginx/pulp/{{ item.name }}.conf || true
 {% endfor %}
 
 ENTRYPOINT ["/init"]

.travis/build_container.yaml → .ci/ansible/build_container.yaml

@@ -16,10 +16,10 @@
       # build. So that PR branches can be used via relative paths.
       #
       # We default to using the docker build / podman buildah cache, for
-      # 1-off-builds and Travis CI purposes (which has no cache across CI runs.)
+      # 1-off-builds and CI purposes (which has no cache across CI runs.)
       # Run build.yaml with -e cache=false if your builds are using outdated
       # layers.
-      command: "docker build --network host --no-cache={{ not cache | default(true) | bool }} -t {{ image.name }}:{{ image.tag }} -f {{ playbook_dir }}/Containerfile ../.."
+      command: "docker build --network host --no-cache={{ not cache | default(true) | bool }} -t {{ image.name }}:{{ image.tag }} -f {{ playbook_dir }}/Containerfile ../../.."
 
     - name: "Clean image cache"
       docker_prune:

.travis/settings.py.j2 → .ci/ansible/settings.py.j2

@@ -1,9 +1,9 @@
-CONTENT_ORIGIN = "http://pulp:80"
-ANSIBLE_API_HOSTNAME = "http://pulp:80"
-ANSIBLE_CONTENT_HOSTNAME = "http://pulp:80/pulp/content"
+CONTENT_ORIGIN = "{{ pulp_scheme }}://pulp:{{ 443 if pulp_scheme == 'https' else 80 }}"
+ANSIBLE_API_HOSTNAME = "{{ pulp_scheme }}://pulp:{{ 443 if pulp_scheme == 'https' else 80 }}"
+ANSIBLE_CONTENT_HOSTNAME = "{{ pulp_scheme }}://pulp:{{ 443 if pulp_scheme == 'https' else 80 }}/pulp/content"
 PRIVATE_KEY_PATH = "/etc/pulp/certs/token_private_key.pem"
 PUBLIC_KEY_PATH = "/etc/pulp/certs/token_public_key.pem"
-TOKEN_SERVER = "http://pulp:80/token"
+TOKEN_SERVER = "{{ pulp_scheme }}://pulp:{{ 443 if pulp_scheme == 'https' else 80 }}/token/"
 TOKEN_SIGNATURE_ALGORITHM = "ES256"
 
 {% if pulp_settings %}

.travis/start_container.yaml → .ci/ansible/start_container.yaml

@@ -71,7 +71,8 @@
         - name: "Wait for Pulp"
           uri:
             url: "http://pulp/pulp/api/v3/status/"
-            follow_redirects: none
+            follow_redirects: all
+            validate_certs: no
           register: result
           until: result.status == 200
           retries: 12

.ci/assets/httpie/config.json

@@ -0,0 +1,7 @@
+{
+    "default_options": [
+      "--ignore-stdin",
+      "--pretty=format",
+      "--traceback"
+    ]
+}

.ci/assets/release_requirements.txt

@@ -0,0 +1,4 @@
+bump2version
+gitpython
+python-redmine
+towncrier

.ci/scripts/check_gettext.sh

@@ -0,0 +1,22 @@
+#!/bin/bash
+
+# WARNING: DO NOT EDIT!
+#
+# This file was generated by plugin_template, and is managed by it. Please use
+# './plugin-template --github galaxy_ng' to update this file.
+#
+# For more info visit https://github.com/pulp/plugin_template
+
+# make sure this script runs at the repo root
+cd "$(dirname "$(realpath -e "$0")")"/../..
+
+set -uv
+
+# check for imports not from pulpcore.plugin. exclude tests
+MATCHES=$(grep -n -r --include \*.py "_(f")
+
+if [ $? -ne 1 ]; then
+  printf "\nERROR: Detected mix of f-strings and gettext:\n"
+  echo "$MATCHES"
+  exit 1
+fi

.travis/check_pulpcore_imports.sh → .ci/scripts/check_pulpcore_imports.sh

@@ -3,18 +3,17 @@
 # WARNING: DO NOT EDIT!
 #
 # This file was generated by plugin_template, and is managed by it. Please use
-# './plugin-template --travis galaxy_ng' to update this file.
+# './plugin-template --github galaxy_ng' to update this file.
 #
 # For more info visit https://github.com/pulp/plugin_template
 
+# make sure this script runs at the repo root
+cd "$(dirname "$(realpath -e "$0")")"/../..
+
 set -uv
 
 # check for imports not from pulpcore.plugin. exclude tests
-MATCHES=$(grep -n -r --include \*.py "from pulpcore.*import" . \
-    | grep -v "tests\|plugin" \
-    | grep -v "pulpcore.app.*viewsets" \
-    | grep -v "pulpcore\.app.*admin" \
-    | grep -v "ProgressReportSerializer")
+MATCHES=$(grep -n -r --include \*.py "from pulpcore.*import" . | grep -v "tests\|plugin"| grep -v "pulpcore.app.*viewsets"| grep -v "pulpcore\.app.*admin"| grep -v "ProgressReportSerializer")
 
 if [ $? -ne 1 ]; then
   printf "\nERROR: Detected bad imports from pulpcore:\n"

.ci/scripts/cherrypick.sh

@@ -0,0 +1,69 @@
+#!/bin/bash
+
+# WARNING: DO NOT EDIT!
+#
+# This file was generated by plugin_template, and is managed by it. Please use
+# './plugin-template --github galaxy_ng' to update this file.
+#
+# For more info visit https://github.com/pulp/plugin_template
+
+set -e
+
+if [ ! -d CHANGES ]; then
+  echo "Error: no CHANGES directory detected. This script must be run from the project root."
+  exit 1
+fi
+
+if [ $# -lt 3 ]
+then
+  echo "Usage: .ci/scripts/cherrypick.sh [commit-hash] [original-issue-id] [backport-issue-id]"
+  echo "   ex: .ci/scripts/cherrypick.sh abcd1234 1234 4567"
+  echo ""
+  echo "Note: make sure you are on a fork of the release branch before running this script."
+  exit
+fi
+
+commit="$(git rev-parse $1)"
+issue="$2"
+backport="$3"
+commit_message=$(git log --format=%B -n 1 $commit)
+
+if ! echo $commit_message | tr '[:upper:]' '[:lower:]' | grep -q "\[noissue\]"
+then
+  if ! echo $commit_message | tr '[:upper:]' '[:lower:]' | grep -q -E "(fixes|closes).*#$issue"
+  then
+    echo "Error: issue $issue not detected in commit message." && exit 1
+  fi
+fi
+
+if [ "$4" = "--continue" ]
+then
+  echo "Continue after manually resolving conflicts..."
+elif [ "$4" = "" ]
+then
+  if ! git cherry-pick --no-commit "$commit"
+  then
+    echo "Please resolve and add merge conflicts and restart this command with appended '--continue'."
+    exit 1
+  fi
+else
+  exit 1
+fi
+
+for file in $(find CHANGES -name "$issue.*")
+do
+  newfile="${file/$issue/$backport}"
+  git mv "$file" "$newfile"
+  sed -i -e "\$a (backported from #$issue)" "$newfile"
+  git add "$newfile"
+done
+
+commit_message="$(printf "$commit_message" | sed -E 's/(fixes|closes)/backports/i')"
+commit_message="$commit_message
+
+fixes #$backport
+
+(cherry picked from commit $commit)"
+git commit -m "$commit_message"
+
+printf "\nSuccessfully backported commit $1.\n"

.ci/scripts/redmine.py

@@ -0,0 +1,35 @@
+# WARNING: DO NOT EDIT!
+#
+# This file was generated by plugin_template, and is managed by it. Please use
+# './plugin-template --github galaxy_ng' to update this file.
+#
+# For more info visit https://github.com/pulp/plugin_template
+
+import os
+import sys
+
+from redminelib import Redmine
+
+REDMINE_API_KEY = os.environ["REDMINE_API_KEY"]
+REDMINE_QUERY_URL = sys.argv[1]
+MILESTONE_URL = sys.argv[2]
+RELEASE = sys.argv[3]
+CLOSED_CURRENTRELEASE = 11
+
+redmine = Redmine(REDMINE_QUERY_URL.split("issues")[0], key=REDMINE_API_KEY)
+query_issues = REDMINE_QUERY_URL.split("=")[-1].split(",")
+milestone_name = redmine.version.get(MILESTONE_URL.split("/")[-1].split(".")[0]).name
+if milestone_name != RELEASE:
+    raise RuntimeError(f"Milestone name, '{milestone_name}', does not match version, '{RELEASE}'.")
+
+to_update = []
+for issue in query_issues:
+    status = redmine.issue.get(int(issue)).status.name
+    if "CLOSE" not in status and status != "MODIFIED":
+        raise ValueError("One or more issues are not MODIFIED")
+    if status == "MODIFIED":  # Removing the already closed
+        to_update.append(int(issue))
+
+for issue in to_update:
+    print(f"Closing #{issue}")
+    redmine.issue.update(issue, status_id=CLOSED_CURRENTRELEASE)

.ci/scripts/schema.py

@@ -0,0 +1,23 @@
+"""
+Customizing OpenAPI validation.
+
+OpenAPI requires paths to start with slashes:
+https://spec.openapis.org/oas/v3.0.3#patterned-fields
+
+But some pulp paths start with curly brackets e.g. {artifact_href}
+This script modifies drf-spectacular schema validation to accept slashes and curly brackets.
+"""
+import json
+from drf_spectacular.validation import JSON_SCHEMA_SPEC_PATH
+
+with open(JSON_SCHEMA_SPEC_PATH) as fh:
+    openapi3_schema_spec = json.load(fh)
+
+properties = openapi3_schema_spec["definitions"]["Paths"]["patternProperties"]
+# Making OpenAPI validation to accept paths starting with / and {
+if "^\\/|{" not in properties:
+    properties["^\\/|{"] = properties["^\\/"]
+    del properties["^\\/"]
+
+with open(JSON_SCHEMA_SPEC_PATH, "w") as fh:
+    json.dump(openapi3_schema_spec, fh)

.ci/scripts/update_redmine.sh

@@ -0,0 +1,26 @@
+#!/bin/bash
+
+# WARNING: DO NOT EDIT!
+#
+# This file was generated by plugin_template, and is managed by it. Please use
+# './plugin-template --github galaxy_ng' to update this file.
+#
+# For more info visit https://github.com/pulp/plugin_template
+
+# make sure this script runs at the repo root
+cd "$(dirname "$(realpath -e "$0")")"/../..
+
+set -euv
+
+export COMMIT_MSG=$(git log --format=%B --no-merges -1)
+export RELEASE=$(echo $COMMIT_MSG | awk '{print $2}')
+export MILESTONE_URL=$(echo $COMMIT_MSG | grep -o "Redmine Milestone: .*" | awk '{print $3}')
+export REDMINE_QUERY_URL=$(echo $COMMIT_MSG | grep -o "Redmine Query: .*" | awk '{print $3}')
+
+echo "Releasing $RELEASE"
+echo "Milestone URL: $MILESTONE_URL"
+echo "Query: $REDMINE_QUERY_URL"
+
+pip install python-redmine httpie
+
+python3 .ci/scripts/redmine.py $REDMINE_QUERY_URL $MILESTONE_URL $RELEASE

.travis/validate_commit_message.py → .ci/scripts/validate_commit_message.py

@@ -1,26 +1,32 @@
 # WARNING: DO NOT EDIT!
 #
-# This file was generated by plugin_template, and is managed by bootstrap.py. Please use
-# bootstrap.py to update this file.
+# This file was generated by plugin_template, and is managed by it. Please use
+# './plugin-template --github galaxy_ng' to update this file.
 #
 # For more info visit https://github.com/pulp/plugin_template
 
-import glob
 import re
-import requests
 import subprocess
 import sys
+from pathlib import Path
+
+
 
-KEYWORDS = ["fixes", "closes", "re", "ref"]
 NO_ISSUE = "[noissue]"
+CHANGELOG_EXTS = [".feature", ".bugfix", ".doc", ".removal", ".misc", ".deprecation"]
+
+
+KEYWORDS = ["fixes", "closes", "re", "ref"]
 STATUSES = ["NEW", "ASSIGNED", "POST", "MODIFIED"]
+REDMINE_URL = "https://pulp.plan.io"
 
 sha = sys.argv[1]
+project = ""
 message = subprocess.check_output(["git", "log", "--format=%B", "-n 1", sha]).decode("utf-8")
 
 
 def __check_status(issue):
-    response = requests.get("https://pulp.plan.io/issues/{}.json".format(issue))
+    response = requests.get(f"{REDMINE_URL}/issues/{issue}.json")
     response.raise_for_status()
     bug_json = response.json()
     status = bug_json["issue"]["status"]["name"]
@@ -30,27 +36,38 @@ def __check_status(issue):
             "{statuses}.".format(issue=issue, status=status, statuses=", ".join(STATUSES))
         )
 
+    if project:
+        project_id = bug_json["issue"]["project"]["id"]
+        project_json = requests.get(f"{REDMINE_URL}/projects/{project_id}.json").json()
+        if project_json["project"]["identifier"] != project:
+            sys.exit(f"Error: issue {issue} is not in the {project} project.")
+
 
 def __check_changelog(issue):
-    if len(glob.glob(f"CHANGES/**/{issue}.*", recursive=True)) < 1:
+    matches = list(Path("CHANGES").rglob(f"{issue}.*"))
+
+    if len(matches) < 1:
         sys.exit(f"Could not find changelog entry in CHANGES/ for {issue}.")
+    for match in matches:
+        if match.suffix not in CHANGELOG_EXTS:
+            sys.exit(f"Invalid extension for changelog entry '{match}'.")
 
 
 print("Checking commit message for {sha}.".format(sha=sha[0:7]))
 
 # validate the issue attached to the commit
-if NO_ISSUE in message:
-    print("Commit {sha} has no issue attached. Skipping issue check".format(sha=sha[0:7]))
-else:
-    regex = r"(?:{keywords})[\s:]+#(\d+)".format(keywords=("|").join(KEYWORDS))
-    pattern = re.compile(regex)
+regex = r"(?:{keywords})[\s:]+#(\d+)".format(keywords=("|").join(KEYWORDS))
+pattern = re.compile(regex, re.IGNORECASE)
 
-    issues = pattern.findall(message)
+issues = pattern.findall(message)
 
-    if issues:
-        for issue in pattern.findall(message):
-            __check_status(issue)
-            __check_changelog(issue)
+if issues:
+    for issue in pattern.findall(message):
+        __check_status(issue)
+        __check_changelog(issue)
+else:
+    if NO_ISSUE in message:
+        print("Commit {sha} has no issues but is tagged {tag}.".format(sha=sha[0:7], tag=NO_ISSUE))
     else:
         sys.exit(
             "Error: no attached issues found for {sha}. If this was intentional, add "

.github/post-job-template.yml.j2

@@ -0,0 +1,18 @@
+update_manifest:
+
+    runs-on: ubuntu-latest
+    needs: test
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v2
+        with:
+          ref: {{ "${{ github.event.after }}" }}  # for PR avoids checking out merge commit
+          fetch-depth: 0  # include all history
+
+      - name: Update Manifest for Ansible Hub UI
+        env:
+          GITHUB_PULL_REQUEST: {{ "${{ github.event.number }}" }}
+          GITHUB_BRANCH: {{ "${{ github.head_ref }}" }}
+          GITHUB_REPO_SLUG: {{ "${{ github.repository }}" }}
+        run: .github/workflows/scripts/update_manifest.sh
+        shell: bash

.github/pre-job-template.yml.j2

@@ -0,0 +1,17 @@
+check_commit:
+
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v2
+        with:
+          ref: {{ "${{ github.event.after }}" }}  # for PR avoids checking out merge commit
+          fetch-depth: 0  # include all history
+
+      - name: Run script to validate commits for both pull request and a push
+        env:
+          GITHUB_PR_COMMITS_URL: {{ "${{ github.event.pull_request.commits_url }}" }}
+          START_COMMIT: {{ "${{ github.event.before }}" }}
+          END_COMMIT: {{ "${{ github.event.after }}" }}
+        run: |
+          python .ci/scripts/validate_commit_message_custom.py

.github/template_gitref

@@ -0,0 +1 @@
+2021.04.08-88-ga67f224-dirty