diff --git a/actions/shellcheck/extract-shell-from-gh-actions-files b/actions/shellcheck/extract-shell-from-gh-actions-files new file mode 100755 index 0000000..838167e --- /dev/null +++ b/actions/shellcheck/extract-shell-from-gh-actions-files @@ -0,0 +1,155 @@ +#!/usr/bin/env python3 +""" +usage: extract-shell-from-gh-actions-files [-0] [ […]] + +Writes the shell snippets in a GitHub Actions workflow file or composite action +file to a set of separate files. There's one file for each workflow × job × +step or composite action × step (for steps with a "run" block). The produced +file names are printed to stdout. Use the -0 option to terminate them with a +null byte (\\x00) instead of a newline (\\x0a). +""" +from pathlib import Path +from shlex import split as shsplit +import argparse +import re +import sys +import yaml + + +__usage__ = __doc__ + + +def main(output_dir, *files, terminator = "\n"): + if not output_dir: + return fatal(f"no output dir given") + + output_dir = Path(output_dir) + + if not output_dir.is_dir(): + return fatal(f"output path {output_dir!r} is not a directory or does not exist") + + if not files: + return fatal(f"no files given") + + for file in (Path(f) for f in files): + with file.open("r", encoding = "utf-8") as fh: + contents = yaml.safe_load(fh) + + # Workflow + if contents.get("jobs"): + workflow = contents + workflow_output_dir = output_dir / "workflows" / file.name + workflow_output_dir.mkdir(parents = True) + + for job_name, job in workflow.get("jobs", {}).items(): + job_output_dir = workflow_output_dir / f"job-{fssafe(job_name)}" + job_output_dir.mkdir() + + extract_shell_steps(workflow, job, job_output_dir, terminator) + + # Composite action + elif contents.get("runs", {}).get("using") == "composite": + action = contents + action_output_dir = output_dir / "actions" / file.parent.name + action_output_dir.mkdir(parents = True) + + extract_shell_steps({}, {"steps": action.get("runs", {}).get("steps", [])}, action_output_dir, terminator) + + else: + return fatal(f"file {file} does not appear to be a GitHub Actions workflow file or composite action file") + + +def extract_shell_steps(workflow, job, job_output_dir, terminator): + for step_idx, step in enumerate(job.get("steps", []), 1): + step_name = step.get("name", str(step_idx)) + run_shell = step.get("run") + + if run_shell is None: + continue + + shell = shell_name(workflow, job, step) + + if shell not in {"bash", "sh"}: + continue + + # Replace GitHub Actions workflow interpolations (${{ … }}) + # with a placeholder so it doesn't cause false issues reported + # by ShellCheck. + run_shell = re.sub(r'\$\{\{.+?\}\}', "…", run_shell) + + # Use job and step env to hint to ShellCheck what env vars are defined. + # By default it already assumes UPPERCASE vars are always defined, but + # it will warn about lowercase vars that it doesn't know about. + env = {**job.get("env", {}), **step.get("env", {})} + + step_output_file = job_output_dir / f"step-{fssafe(step_name)}" + + with step_output_file.open("w", encoding = "utf-8") as fh: + print(f"#!{shell_cmd(workflow, job, step)}", file = fh) + print(f"# shellcheck disable=SC2096,SC2239", file = fh) + if env: + print("export", *env.keys(), file = fh) + print(run_shell, file = fh) + + print(step_output_file, end = terminator) + + +def shell_name(workflow, job, step): + return Path(shsplit(shell_cmd(workflow, job, step))[0]).stem.lower() + + +def shell_cmd(workflow, job, step): + def default_shell(x): + x.get("defaults", {}).get("run", {}).get("shell") + + shell = step.get("shell") \ + or default_shell(job) \ + or default_shell(workflow) + + # expansions and defaults from + if shell == "bash": + shell = "bash --noprofile --norc -eo pipefail {0}" + + elif shell == "sh": + shell = "sh -e {0}" + + elif not shell: + shell = "pwsh" if runs_on_windows(job) else "bash -e {0}" + + return shell.replace("{0}", "") + + +def runs_on_windows(job): + runs_on = job.get("runs-on") + + # XXX TODO: runs-on may be a list of labels or even a dictionary of {group, + # labels}¹, but we don't handle that here (and can't really know what an + # arbitrary label means anyway). + # -trs, 7 March 2023 + # + # ¹ + if not isinstance(runs_on, str): + return None + + return runs_on.lower().startswith("windows-") + + +def fssafe(name): + return re.sub(r'[^a-zA-Z0-9_-]+', "-", name) + + +def fatal(error): + print(error, file = sys.stderr) + print(__usage__) + return 1 + + +if __name__ == "__main__": + args = sys.argv[1:] + kwargs = {} + + if args and args[0] == "-0": + kwargs["terminator"] = "\0" + del args[0] + + sys.exit(main(*args, **kwargs)) diff --git a/actions/shellcheck/extract-shell-from-gh-workflow b/actions/shellcheck/extract-shell-from-gh-workflow deleted file mode 100755 index 0e35893..0000000 --- a/actions/shellcheck/extract-shell-from-gh-workflow +++ /dev/null @@ -1,118 +0,0 @@ -#!/usr/bin/env python3 -""" -usage: extract-gh-actions-shell [-0] [ […]] - -Writes the shell snippets in a GitHub Actions workflow file to a set of -separate files. There's one file for each workflow × job × step (with a "run" -block). The produced file names are printed to stdout. Use the -0 option to -terminate them with a null byte (\\x00) instead of a newline (\\x0a). -""" -from pathlib import Path -from shlex import split as shsplit -import argparse -import re -import sys -import yaml - - -__usage__ = __doc__ - - -def main(output_dir, *workflow_files, terminator = "\n"): - if not output_dir: - return fatal(f"no output dir given") - - output_dir = Path(output_dir) - - if not output_dir.is_dir(): - return fatal(f"output path {output_dir!r} is not a directory or does not exist") - - if not workflow_files: - return fatal(f"no workflow files given") - - for workflow_file in (Path(w) for w in workflow_files): - with workflow_file.open("r", encoding = "utf-8") as fh: - workflow = yaml.safe_load(fh) - - workflow_output_dir = output_dir / workflow_file.name - workflow_output_dir.mkdir() - - for job_name, job in workflow.get("jobs", {}).items(): - job_output_dir = workflow_output_dir / f"job-{fssafe(job_name)}" - job_output_dir.mkdir() - - for step_idx, step in enumerate(job.get("steps", []), 1): - step_name = step.get("name", str(step_idx)) - run_shell = step.get("run") - - if run_shell is None: - continue - - shell = shell_name(workflow, job, step) - - if shell not in {"bash", "sh"}: - continue - - # Replace GitHub Actions workflow interpolations (${{ … }}) - # with a placeholder so it doesn't cause false issues reported - # by ShellCheck. - run_shell = re.sub(r'\$\{\{.+?\}\}', "…", run_shell) - - step_output_file = job_output_dir / f"step-{fssafe(step_name)}" - - with step_output_file.open("w", encoding = "utf-8") as fh: - print(f"#!/bin/{shell}", file = fh) - print(run_shell, file = fh) - - print(step_output_file, end = terminator) - - -def shell_name(workflow, job, step): - def default_shell(x): - x.get("defaults", {}).get("run", {}).get("shell") - - shell = step.get("shell") \ - or default_shell(job) \ - or default_shell(workflow) - - if shell: - return Path(shsplit(shell)[0]).stem.lower() - else: - # defaults from - return "pwsh" if runs_on_windows(job) else "bash" - - -def runs_on_windows(job): - runs_on = job.get("runs-on") - - # XXX TODO: runs-on may be a list of labels or even a dictionary of {group, - # labels}¹, but we don't handle that here (and can't really know what an - # arbitrary label means anyway). - # -trs, 7 March 2023 - # - # ¹ - if not isinstance(runs_on, str): - return None - - return runs_on.lower().startswith("windows-") - - -def fssafe(name): - return re.sub(r'[^a-zA-Z0-9_-]+', "-", name) - - -def fatal(error): - print(error, file = sys.stderr) - print(__usage__) - return 1 - - -if __name__ == "__main__": - args = sys.argv[1:] - kwargs = {} - - if args and args[0] == "-0": - kwargs["terminator"] = "\0" - del args[0] - - sys.exit(main(*args, **kwargs)) diff --git a/actions/shellcheck/shellcheck b/actions/shellcheck/shellcheck index ad73cea..0733e80 100755 --- a/actions/shellcheck/shellcheck +++ b/actions/shellcheck/shellcheck @@ -19,21 +19,21 @@ main() { find-files() { shell-files - workflow-shell + actions-shell } shell-files() { git grep -lzP '^#!(.*?)(ba)?sh' } -workflow-shell() { - tmpdir="$base/.github/workflows/_shellcheck/" +actions-shell() { + local tmpdir="$base/.github/_shellcheck/" rm -rf "$tmpdir" mkdir "$tmpdir" echo "*" > "$tmpdir/.gitignore" - git ls-files -z :/.github/workflows/'*'.y{a,}ml \ - | xargs -0 "$bin"/extract-shell-from-gh-workflow -0 "$tmpdir" + git ls-files -z :/.github/workflows/'*'.y{a,}ml :'**'/action.y{a,}ml \ + | xargs -0 "$bin"/extract-shell-from-gh-actions-files -0 "$tmpdir" } download() {