From c70e994fefd26f9f35027e88a2da7b5d6642f721 Mon Sep 17 00:00:00 2001 From: Casper Welzel Andersen Date: Mon, 30 Oct 2023 11:57:29 +0100 Subject: [PATCH 1/8] Support GitHub Enterprise Add a new input parameter: gh_rest_api_base_url It defaults to https://api.github.com. If set, it should include the path `/api/v3` - if not, it will be added. Use $GITHUB_SERVER_URL otherwise throughout. --- README.md | 1 + action.yml | 4 ++++ entrypoint.sh | 6 ++--- push_action/run.py | 50 ++++++++++++++++++++++++++++++++++++++--- push_action/utils.py | 7 ++++-- push_action/validate.py | 40 +++++++++++++++++++++++++++++++++ 6 files changed, 100 insertions(+), 8 deletions(-) create mode 100644 push_action/validate.py diff --git a/README.md b/README.md index 3a4de3b..5361507 100644 --- a/README.md +++ b/README.md @@ -155,6 +155,7 @@ All input names in **bold** are _required_. | `unprotect_reviews` | Momentarily remove pull request review protection from target branch.
**Note**: One needs administrative access to the repository to be able to use this feature. This means two things need to match up: The PAT must represent a user with administrative rights, and these rights need to be granted to the usage scope of the PAT. | `False` | | `debug` | Set `set -x` in `entrypoint.sh` when running the action. This is for debugging the action. | `False` | | `path` | A path to the working directory of the action. This should be relative to the `$GITHUB_WORKSPACE`. | `.` | +| `gh_rest_api_base_url` | The base URL for the GitHub REST API. This is useful for GitHub Enterprise users.
Note, `/api/v3` will be appended to this value if it does not already exist. See the note [here](https://docs.github.com/en/enterprise-server@3.10/rest/quickstart?apiVersion=2022-11-28&tool=curl#using-curl-commands-in-github-actions). | `https://api.github.com` | ## License diff --git a/action.yml b/action.yml index 695904c..3e6f16f 100644 --- a/action.yml +++ b/action.yml @@ -48,6 +48,10 @@ inputs: description: 'A path to the working directory of the action. This should be relative to the $GITHUB_WORKSPACE.' required: false default: '.' + gh_rest_api_base_url: + description: 'The base URL for the GitHub REST API. This is useful for GitHub Enterprise users. Note, `/api/v3` will be appended to this value if it does not already exist. See the note here: https://docs.github.com/en/enterprise-server@3.10/rest/quickstart?apiVersion=2022-11-28&tool=curl#using-curl-commands-in-github-actions.' + required: false + default: 'https://api.github.com' runs: using: 'docker' image: 'Dockerfile' diff --git a/entrypoint.sh b/entrypoint.sh index b1d07d6..aa63df2 100755 --- a/entrypoint.sh +++ b/entrypoint.sh @@ -102,9 +102,9 @@ git config --global --add safe.directory ${PWD} # Retrieve target repository echo -e "\nFetching the latest information from '${GITHUB_REPOSITORY}' ..." -git config --local --name-only --get-regexp "http\.https\:\/\/github\.com\/\.extraheader" && git config --local --unset-all "http.https://github.com/.extraheader" || : -git submodule foreach --recursive 'git config --local --name-only --get-regexp "http\.https\:\/\/github\.com\/\.extraheader" && git config --local --unset-all "http.https://github.com/.extraheader" || :' -git remote set-url origin https://${GITHUB_ACTOR}:${INPUT_TOKEN}@github.com/${GITHUB_REPOSITORY}.git +git config --local --name-only --get-regexp "http\.https\:\/\/github\.com\/\.extraheader" && git config --local --unset-all "http.${GITHUB_SERVER_URL}/.extraheader" || : +git submodule foreach --recursive 'git config --local --name-only --get-regexp "http\.https\:\/\/github\.com\/\.extraheader" && git config --local --unset-all "http.${GITHUB_SERVER_URL}/.extraheader" || :' +git remote set-url origin $(push-action create_origin_url) git fetch --unshallow -tp origin || : echo "Fetching the latest information from '${GITHUB_REPOSITORY}' ... DONE!" diff --git a/push_action/run.py b/push_action/run.py index e19088d..bff7eb0 100644 --- a/push_action/run.py +++ b/push_action/run.py @@ -6,16 +6,16 @@ 1) Get required statuses for branch (GitHub Actions jobs / third party status checks) from: - https://api.github.com/repos/:owner/:repo/branches/:branch + {input_gh_rest_api_base_url}/repos/:owner/:repo/branches/:branch protection -> required_status_checks -> contexts 2) Get GitHub Actions runs for specific workflow: - https://api.github.com/repos/:owner/:repo/actions/workflows/:workflow_id/runs + {input_gh_rest_api_base_url}/repos/:owner/:repo/actions/workflows/:workflow_id/runs :workflow_id can also be :workflow_file_name (e.g., 'main.yml') Get :run_id from this 3) Get names and statuses of jobs in specific run: - https://api.github.com/repos/:owner/:repo/actions/runs/:run_id/jobs + {input_gh_rest_api_base_url}/repos/:owner/:repo/actions/runs/:run_id/jobs Match found required GitHub Actions runs found in 1) 4) Wait and do 3) again until required GitHub Actions jobs have "status": "completed" @@ -29,6 +29,7 @@ import sys from time import sleep, time from typing import TYPE_CHECKING +from urllib.parse import urlsplit from push_action.cache import IN_MEMORY_CACHE from push_action.utils import ( @@ -197,6 +198,46 @@ def protected_branch(branch: str) -> str: return "protected" if response["protected"] else "" +def compile_origin_url() -> str: + """Compile the git remote 'origin' URL for the repository.""" + compiled_url = "" + + for required_env_vars in [ + "GITHUB_SERVER_URL", + "GITHUB_REPOSITORY", + "GITHUB_ACTOR", + "INPUT_TOKEN", + ]: + if required_env_vars not in os.environ: + raise RuntimeError( + f"Required rnvironment variable {required_env_vars} is not set." + ) + + base_url = os.getenv("GITHUB_SERVER_URL", "https://github.com") + split_base_url = urlsplit(base_url) + + if not split_base_url.scheme or split_base_url.netloc: + raise RuntimeError( + f"Could not determine scheme and netloc from GITHUB_SERVER_URL: {base_url}" + ) + + # Add scheme + compiled_url += f"{split_base_url.scheme}://" + + # Add username and token + compiled_url += os.getenv("GITHUB_ACTOR", "") + compiled_url += ":" + compiled_url += os.getenv("INPUT_TOKEN", "") + + # Add netloc + compiled_url += f"@{split_base_url.netloc}" + + # Add path (repository) + compiled_url += f"/{os.getenv('GITHUB_REPOSITORY', '')}.git" + + return compiled_url + + def main() -> None: """Main function to run this module""" # Handle inputs @@ -247,6 +288,7 @@ def main() -> None: "unprotect_reviews", "protect_reviews", "protected_branch", + "create_origin_url", ], ) @@ -264,6 +306,8 @@ def main() -> None: protect_reviews() elif IN_MEMORY_CACHE["args"].ACTION == "protected_branch": print(protected_branch(IN_MEMORY_CACHE["args"].ref), end="", flush=True) + elif IN_MEMORY_CACHE["args"].ACTION == "create_origin_url": + print(compile_origin_url(), end="", flush=True) else: raise RuntimeError(f"Unknown ACTIONS {IN_MEMORY_CACHE['args'].ACTION!r}") except Exception as exc: # pylint: disable=broad-except diff --git a/push_action/utils.py b/push_action/utils.py index 2ee3dd9..23df9a4 100644 --- a/push_action/utils.py +++ b/push_action/utils.py @@ -17,13 +17,15 @@ import requests from push_action.cache import IN_MEMORY_CACHE +from push_action.validate import validate_rest_api_base_url if TYPE_CHECKING: - from typing import Callable, List, Optional, Union + from typing import Callable, List, Union REQUEST_TIMEOUT = 10 # in seconds -API_V3_BASE = "https://api.github.com" +API_V3_BASE = validate_rest_api_base_url(os.getenv("INPUT_GH_REST_API_BASE_URL", "")) +API_VERSION = "2022-11-28" class RepoRole(Enum): @@ -73,6 +75,7 @@ def api_request( headers={ "Authorization": f"Bearer {IN_MEMORY_CACHE['args'].token}", "Accept": "application/vnd.github.v3+json", + "X-GitHub-Api-Version": API_VERSION, }, timeout=REQUEST_TIMEOUT, **kwargs, diff --git a/push_action/validate.py b/push_action/validate.py new file mode 100644 index 0000000..c2535f8 --- /dev/null +++ b/push_action/validate.py @@ -0,0 +1,40 @@ +"""Validate inputs.""" +from urllib.parse import urlsplit + +GITHUB_FREE_REST_API_BASE_URL = "https://api.github.com" +"""Base URL for GitHub Free API.""" + +GITHUB_ENTERPRISE_API_PREFIX = "/api/v3" +"""Prefix for GitHub Enterprise API URLs. +See the note for more information here: +https://docs.github.com/en/enterprise-server@3.10/rest/quickstart?apiVersion=2022-11-28&tool=curl#using-curl-commands-in-github-actions. +""" + + +def validate_rest_api_base_url(base_url: str) -> str: + """Validate and parse the `gh_rest_api_base_url` input.""" + split_base_url = urlsplit(base_url) + + if not split_base_url.scheme or not split_base_url.netloc: + raise ValueError( + "Invalid URL provided for `gh_rest_api_base_url` input (missing scheme " + "and/or netloc)." + ) + + compiled_url = split_base_url.scheme + "://" + split_base_url.netloc + + if compiled_url == GITHUB_FREE_REST_API_BASE_URL: + return compiled_url + + url_path = split_base_url.path.rstrip("/") + + if url_path and not split_base_url.path.endswith(GITHUB_ENTERPRISE_API_PREFIX): + raise ValueError( + "Invalid URL provided for `gh_rest_api_base_url` input (path must end with " + f"{GITHUB_ENTERPRISE_API_PREFIX!r})." + ) + + if not url_path: + compiled_url += GITHUB_ENTERPRISE_API_PREFIX + + return compiled_url From 493fcfe4522030ead0f93ee6490987a4d8cb07e8 Mon Sep 17 00:00:00 2001 From: Casper Welzel Andersen Date: Mon, 30 Oct 2023 12:07:02 +0100 Subject: [PATCH 2/8] Use "@Q" for regexp editions of gh server url --- entrypoint.sh | 4 ++-- push_action/run.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/entrypoint.sh b/entrypoint.sh index aa63df2..e266c92 100755 --- a/entrypoint.sh +++ b/entrypoint.sh @@ -102,8 +102,8 @@ git config --global --add safe.directory ${PWD} # Retrieve target repository echo -e "\nFetching the latest information from '${GITHUB_REPOSITORY}' ..." -git config --local --name-only --get-regexp "http\.https\:\/\/github\.com\/\.extraheader" && git config --local --unset-all "http.${GITHUB_SERVER_URL}/.extraheader" || : -git submodule foreach --recursive 'git config --local --name-only --get-regexp "http\.https\:\/\/github\.com\/\.extraheader" && git config --local --unset-all "http.${GITHUB_SERVER_URL}/.extraheader" || :' +git config --local --name-only --get-regexp "http\.${GITHUB_SERVER_URL@Q}\/\.extraheader" && git config --local --unset-all "http.${GITHUB_SERVER_URL}/.extraheader" || : +git submodule foreach --recursive 'git config --local --name-only --get-regexp "http\.${GITHUB_SERVER_URL@Q}\/\.extraheader" && git config --local --unset-all "http.${GITHUB_SERVER_URL}/.extraheader" || :' git remote set-url origin $(push-action create_origin_url) git fetch --unshallow -tp origin || : echo "Fetching the latest information from '${GITHUB_REPOSITORY}' ... DONE!" diff --git a/push_action/run.py b/push_action/run.py index bff7eb0..6bace85 100644 --- a/push_action/run.py +++ b/push_action/run.py @@ -213,7 +213,7 @@ def compile_origin_url() -> str: f"Required rnvironment variable {required_env_vars} is not set." ) - base_url = os.getenv("GITHUB_SERVER_URL", "https://github.com") + base_url = os.getenv("GITHUB_SERVER_URL", "") split_base_url = urlsplit(base_url) if not split_base_url.scheme or split_base_url.netloc: From 97aa1af4fd92755367a0a19e5dbdc399c5a32beb Mon Sep 17 00:00:00 2001 From: Casper Welzel Andersen Date: Mon, 30 Oct 2023 13:06:47 +0100 Subject: [PATCH 3/8] Add required options for push-action --- entrypoint.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/entrypoint.sh b/entrypoint.sh index e266c92..3f6afc1 100755 --- a/entrypoint.sh +++ b/entrypoint.sh @@ -104,7 +104,7 @@ git config --global --add safe.directory ${PWD} echo -e "\nFetching the latest information from '${GITHUB_REPOSITORY}' ..." git config --local --name-only --get-regexp "http\.${GITHUB_SERVER_URL@Q}\/\.extraheader" && git config --local --unset-all "http.${GITHUB_SERVER_URL}/.extraheader" || : git submodule foreach --recursive 'git config --local --name-only --get-regexp "http\.${GITHUB_SERVER_URL@Q}\/\.extraheader" && git config --local --unset-all "http.${GITHUB_SERVER_URL}/.extraheader" || :' -git remote set-url origin $(push-action create_origin_url) +git remote set-url origin $(push-action --token "null" --ref "null" --temp-branch "null" -- create_origin_url) git fetch --unshallow -tp origin || : echo "Fetching the latest information from '${GITHUB_REPOSITORY}' ... DONE!" From 7f848ff86f2eec3db35aff54bd97f3cd999a54ae Mon Sep 17 00:00:00 2001 From: Casper Welzel Andersen Date: Mon, 30 Oct 2023 14:22:05 +0100 Subject: [PATCH 4/8] Ensure if-condition is correctly evaluated --- push_action/run.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/push_action/run.py b/push_action/run.py index 6bace85..52af448 100644 --- a/push_action/run.py +++ b/push_action/run.py @@ -216,7 +216,7 @@ def compile_origin_url() -> str: base_url = os.getenv("GITHUB_SERVER_URL", "") split_base_url = urlsplit(base_url) - if not split_base_url.scheme or split_base_url.netloc: + if not (split_base_url.scheme or split_base_url.netloc): raise RuntimeError( f"Could not determine scheme and netloc from GITHUB_SERVER_URL: {base_url}" ) From 8ef281d2104727e2e7547aef3b03c2a991b188d2 Mon Sep 17 00:00:00 2001 From: Casper Welzel Andersen Date: Mon, 30 Oct 2023 14:43:22 +0100 Subject: [PATCH 5/8] Add logging to push-action --- push_action/__init__.py | 10 ++++++++++ push_action/cache.py | 2 +- push_action/run.py | 8 +++++++- push_action/utils.py | 24 +++++++++++++++++------- 4 files changed, 35 insertions(+), 9 deletions(-) diff --git a/push_action/__init__.py b/push_action/__init__.py index 26a54db..0fd9c8a 100644 --- a/push_action/__init__.py +++ b/push_action/__init__.py @@ -3,6 +3,16 @@ This Python module is a helper module to interact with the GitHub API. It is meant to only be used in the `push-protected` GitHub action. """ +import logging +import os +import sys __version__ = "2.14.0" __author__ = "Casper Welzel Andersen" + + +LOGGER = logging.getLogger("push_action") + +if os.getenv("INPUT_DEBUG", "").lower() in ("true", "1"): + LOGGER.setLevel(logging.DEBUG) + LOGGER.addHandler(logging.StreamHandler(sys.stdout)) diff --git a/push_action/cache.py b/push_action/cache.py index 3766aba..b769c39 100644 --- a/push_action/cache.py +++ b/push_action/cache.py @@ -6,7 +6,7 @@ """ from typing import TYPE_CHECKING -if TYPE_CHECKING: +if TYPE_CHECKING: # pragma: no cover from typing import Any, Iterator diff --git a/push_action/run.py b/push_action/run.py index 52af448..78b8224 100644 --- a/push_action/run.py +++ b/push_action/run.py @@ -25,6 +25,7 @@ """ import argparse import json +import logging import os import sys from time import sleep, time @@ -41,10 +42,13 @@ remove_branch, ) -if TYPE_CHECKING: +if TYPE_CHECKING: # pragma: no cover from typing import Any, Dict +LOGGER = logging.getLogger("push_action.run") + + def wait() -> None: """Wait until status checks have finished""" required_statuses = get_branch_statuses(IN_MEMORY_CACHE["args"].ref) @@ -294,6 +298,8 @@ def main() -> None: IN_MEMORY_CACHE["args"] = parser.parse_args() + LOGGER.debug("Parsed args: %s", IN_MEMORY_CACHE["args"]) + fail = "" try: if IN_MEMORY_CACHE["args"].ACTION == "wait_for_checks": diff --git a/push_action/utils.py b/push_action/utils.py index 23df9a4..bc4e1e7 100644 --- a/push_action/utils.py +++ b/push_action/utils.py @@ -3,6 +3,7 @@ Utility functions for use in the `push_action.run` module. """ from enum import Enum +import logging import os from time import time from typing import TYPE_CHECKING @@ -19,10 +20,13 @@ from push_action.cache import IN_MEMORY_CACHE from push_action.validate import validate_rest_api_base_url -if TYPE_CHECKING: +if TYPE_CHECKING: # pragma: no cover from typing import Callable, List, Union +LOGGER = logging.getLogger("push_action.utils") + + REQUEST_TIMEOUT = 10 # in seconds API_V3_BASE = validate_rest_api_base_url(os.getenv("INPUT_GH_REST_API_BASE_URL", "")) API_VERSION = "2022-11-28" @@ -118,6 +122,12 @@ def api_request( except json.JSONDecodeError as exc: raise RuntimeError(f"Failed to jsonify response.\n{exc!r}") from exc + LOGGER.debug( + "API Call to: %s\nResponse: %s", + url, + response.text if isinstance(response, requests.Response) else response, + ) + return response @@ -189,9 +199,9 @@ def get_workflow_runs(workflow_id: int, new_request: bool = False) -> "List[dict ) workflow_runs = [ - _ - for _ in response.get("workflow_runs", []) - if _.get("head_branch", "") == IN_MEMORY_CACHE["args"].temp_branch + run + for run in response.get("workflow_runs", []) + if run.get("head_branch", "") == IN_MEMORY_CACHE["args"].temp_branch ] if cache_name in IN_MEMORY_CACHE: @@ -250,16 +260,16 @@ def get_required_actions( f"{type(response)}" ) - runs = [] + runs: "List[dict]" = [] for workflow in response["workflows"]: runs.extend(get_workflow_runs(workflow["id"])) - jobs = [] + jobs: "List[dict]" = [] for run in runs: jobs.extend(get_workflow_run_jobs(run["id"])) IN_MEMORY_CACHE[cache_name] = [ - _ for _ in jobs if _.get("name", "") in statuses + job for job in jobs if job.get("name", "") in statuses ] return IN_MEMORY_CACHE[cache_name] From 5bee3c1d430011802fe6f10f35b1be59f8962065 Mon Sep 17 00:00:00 2001 From: Casper Welzel Andersen Date: Mon, 30 Oct 2023 14:44:08 +0100 Subject: [PATCH 6/8] Run protected CI job with debug=true --- .github/workflows/ci_tests.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/ci_tests.yml b/.github/workflows/ci_tests.yml index 53d126b..ac1f1ea 100644 --- a/.github/workflows/ci_tests.yml +++ b/.github/workflows/ci_tests.yml @@ -103,6 +103,7 @@ jobs: token: '${{ secrets.CI_PUSH_TO_PROTECTED_BRANCH }}' branch: protected unprotect_reviews: true + debug: true - name: Pushing to a protected branch without any changes uses: ./ @@ -110,6 +111,7 @@ jobs: token: '${{ secrets.CI_PUSH_TO_PROTECTED_BRANCH }}' ref: refs/heads/protected unprotect_reviews: true + debug: true force-pushing: needs: [protected] From 13975886cde768f80f75b3a5e4eea8b2a2aeb4b5 Mon Sep 17 00:00:00 2001 From: Casper Welzel Andersen Date: Mon, 30 Oct 2023 14:57:35 +0100 Subject: [PATCH 7/8] Use function to escape regexp special chars Implement function from stackoverflow answer: https://stackoverflow.com/a/16951928/12404091 --- entrypoint.sh | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/entrypoint.sh b/entrypoint.sh index 3f6afc1..bbdcd7f 100755 --- a/entrypoint.sh +++ b/entrypoint.sh @@ -12,6 +12,9 @@ esac set -e # Utility functions +ere_quote() { + sed 's/[][\.|$(){}?+*^]/\\&/g' <<< "$*" +} unprotect () { case ${INPUT_UNPROTECT_REVIEWS} in y | Y | yes | Yes | YES | true | True | TRUE | on | On | ON) @@ -102,8 +105,8 @@ git config --global --add safe.directory ${PWD} # Retrieve target repository echo -e "\nFetching the latest information from '${GITHUB_REPOSITORY}' ..." -git config --local --name-only --get-regexp "http\.${GITHUB_SERVER_URL@Q}\/\.extraheader" && git config --local --unset-all "http.${GITHUB_SERVER_URL}/.extraheader" || : -git submodule foreach --recursive 'git config --local --name-only --get-regexp "http\.${GITHUB_SERVER_URL@Q}\/\.extraheader" && git config --local --unset-all "http.${GITHUB_SERVER_URL}/.extraheader" || :' +git config --local --name-only --get-regexp "http\.$(ere_quote "${GITHUB_SERVER_URL}")\/\.extraheader" && git config --local --unset-all "http.${GITHUB_SERVER_URL}/.extraheader" || : +git submodule foreach --recursive 'git config --local --name-only --get-regexp "http\.$(ere_quote "${GITHUB_SERVER_URL}")\/\.extraheader" && git config --local --unset-all "http.${GITHUB_SERVER_URL}/.extraheader" || :' git remote set-url origin $(push-action --token "null" --ref "null" --temp-branch "null" -- create_origin_url) git fetch --unshallow -tp origin || : echo "Fetching the latest information from '${GITHUB_REPOSITORY}' ... DONE!" From 9c9797d40de52760bdd44f9fed83ef193c761078 Mon Sep 17 00:00:00 2001 From: Casper Welzel Andersen Date: Mon, 30 Oct 2023 15:04:30 +0100 Subject: [PATCH 8/8] Stream to stderr not stdout for logging --- push_action/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/push_action/__init__.py b/push_action/__init__.py index 0fd9c8a..932b368 100644 --- a/push_action/__init__.py +++ b/push_action/__init__.py @@ -15,4 +15,4 @@ if os.getenv("INPUT_DEBUG", "").lower() in ("true", "1"): LOGGER.setLevel(logging.DEBUG) - LOGGER.addHandler(logging.StreamHandler(sys.stdout)) + LOGGER.addHandler(logging.StreamHandler(sys.stderr))