From a7bfbc9904e67d915123075091cb27498708b1bf Mon Sep 17 00:00:00 2001 From: Stacky McStackface <95074132+stackable-bot@users.noreply.github.com> Date: Tue, 19 Mar 2024 17:20:27 +0100 Subject: [PATCH] Generated commit to update templated files based on rev 63572a8 in stackabletech/operator-templating repo. (#225) Triggered by: Manual run triggered by: razvan with message [new run-tests script] --- scripts/run-tests | 329 +++++++++++++++++++++++++++++++++++++++++++ scripts/run_tests.sh | 171 +--------------------- 2 files changed, 330 insertions(+), 170 deletions(-) create mode 100755 scripts/run-tests diff --git a/scripts/run-tests b/scripts/run-tests new file mode 100755 index 0000000..6abb97b --- /dev/null +++ b/scripts/run-tests @@ -0,0 +1,329 @@ +#!/bin/env python +# vim: filetype=python syntax=python tabstop=4 expandtab + +import argparse +import collections.abc +import contextlib +import logging +import os +import re +import shutil +import subprocess +import sys +import tempfile + +__version__ = "0.0.1" + +DESCRIPTION = """ +Run integration tests. Call this script from the root of the repository. + +Exits with 0 on success, 1 on failure. + +Requires the following commands to be installed: +* beku +* stackablectl +* kubectl +* kubectl-kuttl + +Examples: + +1. Install operators, run all tests and clean up test namespaces: + + ./scripts/run-tests --parallel 4 + +2. Install operators but for Airflow use version "0.0.0-pr123" instead of "0.0.0-dev" and run all tests as above: + + ./scripts/run-tests --operator airflow=0.0.0-pr123 --parallel 4 + +3. Do not install any operators, run the smoke test suite and keep namespace: + + ./scripts/run-tests --skip-release --skip-delete --test-suite smoke-latest + +4. Run the ldap test(s) from the openshift test suite and keep namespace: + + ./scripts/run-tests --skip-release --skip-delete --test-suite openshift --test ldap +""" + + +class TestRunnerException(Exception): + pass + + +def parse_args(argv: list[str]) -> argparse.Namespace: + """Parse command line args.""" + parser = argparse.ArgumentParser( + description=DESCRIPTION, formatter_class=argparse.RawDescriptionHelpFormatter + ) + parser.add_argument( + "--version", + help="Display application version", + action="version", + version=f"%(prog)s {__version__}", + ) + + parser.add_argument( + "--skip-delete", + help="Do not delete test namespaces.", + action="store_true", + ) + + parser.add_argument( + "--skip-release", + help="Do not install operators.", + action="store_true", + ) + + parser.add_argument( + "--parallel", + help="How many tests to run in parallel. Default 2.", + type=int, + required=False, + default=2, + ) + + parser.add_argument( + "--operator", + help="Patch operator version in release.yaml. Format =", + nargs="*", + type=cli_parse_operator_args, + default=[], + ) + + parser.add_argument( + "--test", + help="Kuttl test to run.", + type=str, + required=False, + ) + + parser.add_argument( + "--test-suite", + help="Name of the test suite to expand. Default: default", + type=str, + required=False, + ) + + parser.add_argument( + "--log-level", + help="Set log level.", + type=cli_log_level, + required=False, + default=logging.INFO, + ) + + return parser.parse_args(argv) + + +def cli_parse_operator_args(args: str) -> tuple[str, str]: + if "=" not in args: + raise argparse.ArgumentTypeError( + f"Invalid operator argument: {args}. Must be in format =" + ) + op, version = args.split("=", maxsplit=1) + return (op, version) + + +def cli_log_level(cli_arg: str) -> int: + match cli_arg: + case "debug": + return logging.DEBUG + case "info": + return logging.INFO + case "error": + return logging.ERROR + case "warning": + return logging.WARNING + case "critical": + return logging.CRITICAL + case _: + raise argparse.ArgumentTypeError("Invalid log level") + + +def have_requirements() -> None: + commands = [ + ("beku", "https://github.com/stackabletech/beku.py"), + ( + "stackablectl", + "https://github.com/stackabletech/stackable-cockpit/blob/main/rust/stackablectl/README.md", + ), + ("kubectl", "https://kubernetes.io/docs/tasks/tools/install-kubectl-linux/"), + ("kubectl-kuttl", "https://kuttl.dev/"), + ] + + err = False + for command, url in commands: + if not shutil.which(command): + logging.error(f'Command "{command}" not found, please install from {url}') + err = True + if err: + raise TestRunnerException() + + +@contextlib.contextmanager +def release_file( + operators: list[tuple[str, str]] = [], +) -> collections.abc.Generator[str, None, None]: + """Patch release.yaml with operator versions if needed. + + If no --operator is set, the default release file is used. + + If an invalid operator name is provided (i.e. one that doesn't exist in the + original release file), a TestRunnerException is raised. + + Yields the name of the (potentially patched) release file. This is a temporary + file that will be deleted when the context manager exits. + """ + + def _patch(): + release_file = os.path.join("tests", "release.yaml") + # Make a copy so we can mutate it without affecting the original + ops_copy = operators.copy() + patched_release = [] + with open(release_file, "r") as f: + patch_version = "" + for line in f: + if patch_version: + line = re.sub(":.+$", f": {patch_version}", line) + patch_version = "" + else: + for op, version in ops_copy: + if op in line: + patch_version = version + ops_copy.remove((op, version)) # found an operator to patch + break + patched_release.append(line) + if ops_copy: + # Some --operator args were not found in the release file. This is + # most likely a typo and CI pipelines should terminate early in such + # cases. + logging.error( + f"Operators {', '.join([op for op, _ in ops_copy])} not found in {release_file}" + ) + raise TestRunnerException() + with tempfile.NamedTemporaryFile( + mode="w", + delete=False, + delete_on_close=False, + prefix="patched", + ) as f: + pcontents = "".join(patched_release) + logging.debug(f"Writing patched release to {f.name}: {pcontents}\n") + f.write(pcontents) + return f.name + + release_file = _patch() + try: + yield release_file + except TestRunnerException as e: + logging.error(f"Caught exception: {e}") + raise + finally: + if "patched" in release_file: + try: + logging.debug(f"Removing patched release file : {release_file}") + os.remove(release_file) + except FileNotFoundError | OSError: + logging.error(f"Failed to delete patched release file: {release_file}") + + +def maybe_install_release(skip_release: bool, release_file: str) -> None: + if skip_release: + logging.debug("Skip release installation") + return + stackablectl_err = "" + try: + stackablectl_cmd = [ + "stackablectl", + "release", + "install", + "--release-file", + release_file, + "tests", + ] + logging.debug(f"Running : {stackablectl_cmd}") + + completed_proc = subprocess.run( + stackablectl_cmd, + capture_output=True, + check=True, + ) + # stackablectl doesn't return a non-zero exit code on failure + # so we need to check stderr for errors + stackablectl_err = completed_proc.stderr.decode("utf-8") + if "error" in stackablectl_err.lower(): + logging.error(stackablectl_err) + logging.error("stackablectl failed") + raise TestRunnerException() + + except subprocess.CalledProcessError: + # in case stackablectl starts returning non-zero exit codes + logging.error(stackablectl_err) + logging.error("stackablectl failed") + raise TestRunnerException() + + +def gen_tests(test_suite: str) -> None: + try: + beku_cmd = [ + "beku", + "--test_definition", + os.path.join("tests", "test-definition.yaml"), + "--kuttl_test", + os.path.join("tests", "kuttl-test.yaml.jinja2"), + "--template_dir", + os.path.join("tests", "templates", "kuttl"), + "--output_dir", + os.path.join("tests", "_work"), + ] + if test_suite: + beku_cmd.extend(["--suite", test_suite]) + + logging.debug(f"Running : {beku_cmd}") + subprocess.run( + beku_cmd, + check=True, + ) + except subprocess.CalledProcessError: + logging.error("beku failed") + raise TestRunnerException() + + +def run_tests(test: str, parallel: int, skip_delete: bool) -> None: + try: + kuttl_cmd = ["kubectl-kuttl", "test"] + if test: + kuttl_cmd.extend(["--test", test]) + if parallel: + kuttl_cmd.extend(["--parallel", str(parallel)]) + if skip_delete: + kuttl_cmd.extend(["--skip-delete"]) + + logging.debug(f"Running : {kuttl_cmd}") + + subprocess.run( + kuttl_cmd, + cwd="tests/_work", + check=True, + ) + except subprocess.CalledProcessError: + logging.error("kuttl failed") + raise TestRunnerException() + + +def main(argv) -> int: + ret = 0 + try: + opts = parse_args(argv[1:]) + logging.basicConfig(encoding="utf-8", level=opts.log_level) + have_requirements() + gen_tests(opts.test_suite) + with release_file(opts.operator) as f: + maybe_install_release(opts.skip_release, f) + run_tests(opts.test, opts.parallel, opts.skip_delete) + except TestRunnerException: + ret = 1 + return ret + + +if __name__ == "__main__": + sys.exit(main(sys.argv)) diff --git a/scripts/run_tests.sh b/scripts/run_tests.sh index 3fc17f6..a80e096 100755 --- a/scripts/run_tests.sh +++ b/scripts/run_tests.sh @@ -1,172 +1,3 @@ #!/usr/bin/env bash -# -# Run the integration test suite for this operator. -# -# If a "tests/release.yaml" file is present, it will install the operators listed -# in the release file first. The name of the test suite in that file must be "tests". -# Since each operator test suite has different dependencies, the "tests/release.yaml" -# file is not included in this repository. -# -# Optionally you can provide a specific test suite to run and even a specific -# test name. -# -# Example 1 - run all tests of the openshift suite. -# -# ./scripts/run_tests.sh --test-suite openshift --parallel 2 -# -# Example 2 - run all tests that contain the word "smoke" in the openshift suite and skip resource deletion. -# -# ./scripts/run_tests.sh \ -# --test-suite openshift \ -# --test smoke \ -# --skip-delete -# -set +e - -DIR_NAME=$(dirname "$0") -REPO_ROOT=$(dirname "$DIR_NAME") -TEST_ROOT="$REPO_ROOT/tests/_work" -RELEASE_FILE="$REPO_ROOT/tests/release.yaml" -STACKABLECTL_SKIP_RELEASE="" -BEKU_TEST_SUITE="" -KUTTL_TEST="" -KUTTL_SKIP_DELETE="" -KUTTL_PARALLEL="" -KUTTL_NAMESPACE="" - -is_installed() { - local command="$1" - local install_url="$2" - - if ! which "$command" >/dev/null 2>&1; then - echo "Command [$command] not found. To install it, please see $install_url" - exit 1 - fi -} - -install_operators() { - if [ -n "$STACKABLECTL_SKIP_RELEASE" ]; then - echo "Skipping operator installation" - return - fi - - if [ -f "$RELEASE_FILE" ]; then - echo "Installing operators with stackablectl version: $(stackablectl --version)" - stackablectl release install --release-file "$RELEASE_FILE" tests - else - echo "No tests/release.yaml found, skipping operator installation" - fi -} - -expand_test_suite() { - # Expand the tests - echo "Running beku version: $(beku --version)" - if [ -z "$BEKU_TEST_SUITE" ]; then - echo "No test suite specified, expanding all tests" - beku - else - echo "Expanding test suite: $BEKU_TEST_SUITE" - beku --suite "$BEKU_TEST_SUITE" - fi -} - -run_tests() { - echo "Running kuttl version: $(kubectl-kuttl --version)" - - local OPTS=("test") - - if [ -n "$KUTTL_SKIP_DELETE" ]; then - OPTS+=("--skip-delete") - fi - - if [ -n "$KUTTL_NAMESPACE" ]; then - OPTS+=("--namespace $KUTTL_NAMESPACE") - - # Create the namespace if it does not exist. - # To avoid an error when the namespace already exists, we use "kubectl describe" - # and if that fails we create the namespace. - kubectl describe namespace "$KUTTL_NAMESPACE" || kubectl create namespace "$KUTTL_NAMESPACE" - fi - - if [ -n "$KUTTL_PARALLEL" ]; then - OPTS+=("--parallel $KUTTL_PARALLEL") - fi - - if [ -n "$KUTTL_TEST" ]; then - OPTS+=("--test=$KUTTL_TEST") - fi - - pushd "$TEST_ROOT" || exit 1 - # Disable SC2068 because we want to pass the array as individual arguments - # and it would break for the "--parallel n" option. - # shellcheck disable=SC2068 - kubectl-kuttl ${OPTS[@]} - local KUTTL_EXIT_CODE=$? - popd || exit 1 - exit $KUTTL_EXIT_CODE -} - -usage() { - cat < Run a test suite from the test_definition.yaml file. Default is all tests. - --test Run a specific test or a set of tests. - --skip-delete Skip resource deletion after the test run. - --parallel Run tests in parallel. Default is to run all tests in parallel. - --skip-release Skip the operator installation. - --namespace Run the tests in the specified namespace. -USAGE -} - -parse_args() { - while [[ "$#" -gt 0 ]]; do - case $1 in - --skip-release) - STACKABLECTL_SKIP_RELEASE="true" - ;; - --skip-delete) - KUTTL_SKIP_DELETE="true" - ;; - --parallel) - KUTTL_PARALLEL="$2" - shift - ;; - --test-suite) - BEKU_TEST_SUITE="$2" - shift - ;; - --test) - KUTTL_TEST="$2" - shift - ;; - --namespace) - KUTTL_NAMESPACE="$2" - shift - ;; - *) - echo "Unknown parameter : $1" - usage - exit 1 - ;; - esac - shift - done -} - -main() { - parse_args "$@" - - is_installed beku "https://github.com/stackabletech/beku.py" - is_installed stackablectl "https://github.com/stackabletech/stackable-cockpit/blob/main/rust/stackablectl/README.md" - is_installed kubectl "https://kubernetes.io/docs/tasks/tools/install-kubectl-linux/" - is_installed kubectl-kuttl "https://kuttl.dev/" - - expand_test_suite - install_operators - run_tests -} - -main "$@" +./scripts/run-tests $@