From 3dc458d8231a48d101d1025e06dcffd91fbedf88 Mon Sep 17 00:00:00 2001 From: Nora Sandler Date: Tue, 25 Jun 2024 21:02:07 +0000 Subject: [PATCH] expand CI --- .github/run_tests.py | 226 ++++++++++++++++++----- .github/workflows/build.yaml | 2 +- .github/workflows/build_and_test.yaml | 2 +- .github/workflows/test_each_chapter.yaml | 2 +- 4 files changed, 179 insertions(+), 53 deletions(-) diff --git a/.github/run_tests.py b/.github/run_tests.py index 359e789..a9f1e87 100755 --- a/.github/run_tests.py +++ b/.github/run_tests.py @@ -1,37 +1,54 @@ #!/usr/bin/env python3 # -*- coding: utf-8 -*- +"""CI script to test the reference implementation for a particular chapter. +Test all combinations of extra credit features and other test options. +This script's main purpose is to test the test suite itself (e.g. make sure +we didn't accidentally use Part II features in what's supposed to be a +Part I-only test) +""" +import argparse import json import platform import subprocess -import sys -# get/sanity check chapter number -cc: str = sys.argv[1] -chapter: int = int(sys.argv[2]) -if chapter < 1 or chapter > 20: +# usage: /run_tests.py /path/to/cc --chapter 1 +# for pre-coalescing chapter 20: ./run_tests.py /path/to/cc --chapter 1 --no-coalesce +parser = argparse.ArgumentParser() +parser.add_argument( + "cc", type=str, nargs="?", default=None, help="Path to compiler under test" +) +parser.add_argument( + "--chapter", + type=str, + help=("Chapter whose implementation we're testing"), +) + +args = parser.parse_args() + +CC: str = args.cc +CHAPTER: int +NO_COALESCING: bool = False +if args.chapter == "20a": + CHAPTER = 20 + NO_COALESCING = True +else: + CHAPTER = int(args.chapter) + +if CHAPTER < 1 or CHAPTER > 20: exit("Bad chapter number") # do we need rosetta? -prefix = "" +ARCH_PREFIX = "" if platform.machine().lower() == "arm64": - prefix = "arch -x86_64 " - -# define the stages we're going to test -stages: list[str] = ["lex", "parse", "tacky", "validate", "codegen", "run"] -if chapter == 1: - # tacky and validate stages not added yet - stages.remove("tacky") - stages.remove("validate") -elif chapter < 5: - # TACKY stage added, validate not - stages.remove("validate") -elif chapter > 18: - # don't test intermediate stages for chapters 19/20 - stages = ["run"] - -# info about extra credit features -ALL_EXTRA_CRED = { + ARCH_PREFIX = "arch -x86_64 " + +# figure out what combinations of extra-credit flags to test. Include: +# --extra-credit (all extra credit features that have been implemented so far) +# none (no extra credit features) +# each combo of extra-credit features covered by any test case in this +# chapter or an earlier one +EXTRA_CRED_CHAPTERS = { "bitwise": 3, "compound": 5, "increment": 5, @@ -43,47 +60,156 @@ def get_idx(flag: str) -> tuple[int, str]: - # sort flags by chapter, use alphabetical order as tiebreaker - return (ALL_EXTRA_CRED[flag], flag) + """Sort flags by chapter, use alphabetical order as tiebreaker + Just for consistency/determinism in test runs + """ + return (EXTRA_CRED_CHAPTERS[flag], flag) -all_implemented_extra_credit: tuple = tuple( - sorted([f for f in ALL_EXTRA_CRED if ALL_EXTRA_CRED[f] <= chapter], key=get_idx) +# all extra credit chapters implemented so far +ALL_EXTRA_CREDIT: tuple = tuple( + sorted( + [f for f in EXTRA_CRED_CHAPTERS if EXTRA_CRED_CHAPTERS[f] <= CHAPTER], + key=get_idx, + ) ) - -# find set of all extra-credit flag combinations we should test -extra_credit_combos: set[tuple] = { +# find set of all extra-credit flag combinations we should test, +# based on test_properties.json, and record appropriate option +# strings to enable each of them (including options passed to test_compiler +# and options passed to the compiler itself +extra_credit_combos: dict[tuple, dict[str, str]] = { # none (empty set) - (), + (): {"test_opts": "", "compiler_opts": ""}, # all - all_implemented_extra_credit, + ALL_EXTRA_CREDIT: { + "test_opts": "--extra-credit", + "compiler_opts": " ".join("--" + f for f in ALL_EXTRA_CREDIT), + }, } +# if the key-value pair "test/case.c": (flag1, flag2, ...) +# appears in test_properties.json, and test/case.c is in the current +# chapter or earlier, add (flag1, flag2, ...) to our set of extra credit combos with open("./test_properties.json", encoding="utf-8") as f: extra_cred_flags = json.load(f)["extra_credit_tests"] for path, flags in extra_cred_flags.items(): # all keys in this dict start with chapter_N/ chapter_num = int(path.split("/")[0].removeprefix("chapter_")) - if chapter_num <= chapter: - extra_credit_combos.add(tuple(sorted(flags, key=get_idx))) + if chapter_num <= CHAPTER: + k = tuple(sorted(flags, key=get_idx)) + opt_string = " ".join("--" + flag for flag in k) + extra_credit_combos[k] = { + "test_opts": opt_string, + "compiler_opts": opt_string, + } + + +# test functions +def run_tests(test_script_opts: str = "", cc_opts: str = "") -> None: + """Invoke test_compiler script with a particular set of options. + Args: + test_script_opts: options to pass test_compiler + cc_opts: options to pass the compiler under test (after -- ) + """ + + if cc_opts: + cc_opt_str = f" -- {cc_opts}" + else: + cc_opt_str = "" + + cmd = f"{ARCH_PREFIX}./test_compiler {CC} --chapter {CHAPTER} {test_script_opts}{cc_opt_str}" + print(cmd, flush=True) + subprocess.run( + cmd, + shell=True, + check=True, + ) -for flags in sorted(extra_credit_combos): - extra_cred_args: str = "" - if flags: - flag_str = " ".join("--" + f for f in flags) - test_compiler_arg = ( - "--extra-credit" if flags == all_implemented_extra_credit else flag_str +def test_normal() -> None: + """Main test method for if we're testing a chapter from part I or II""" + # define the stages we're going to test + stages: list[str] = ["lex", "parse", "tacky", "validate", "codegen", "run"] + if CHAPTER == 1: + # tacky and validate stages not added yet + stages.remove("tacky") + stages.remove("validate") + elif CHAPTER < 5: + # TACKY stage added, validate not + stages.remove("validate") + for _, extra_cred_opts in extra_credit_combos.items(): + test_script_opts = extra_cred_opts["test_opts"] + compiler_opts = extra_cred_opts["compiler_opts"] + for stage in stages: + stage_test_script_opts = f"{test_script_opts} --stage {stage}" + run_tests(test_script_opts=stage_test_script_opts, cc_opts=compiler_opts) + + +def test_chapter_19() -> None: + """Run tests for chapter 19. + Don't test different stages but do run both with and without + --int-only option + """ + for _, extra_cred_opts in extra_credit_combos.items(): + test_script_opts = extra_cred_opts["test_opts"] + compiler_opts = extra_cred_opts["compiler_opts"] + # run with --int-only option + run_tests( + test_script_opts=test_script_opts + " --int-only", + cc_opts=compiler_opts + " --int-only", + ) + # run without it + run_tests(test_script_opts=test_script_opts, cc_opts=compiler_opts) + + +def test_chapter_20a() -> None: + """Run tests for chapter 20 without coalescing. + Don't test different stages but do run both with and without + --int-only option. + """ + for _, extra_cred_opts in extra_credit_combos.items(): + test_script_opts = extra_cred_opts["test_opts"] + compiler_opts = extra_cred_opts["compiler_opts"] + # run with --int-only option + run_tests( + test_script_opts=test_script_opts + " --int-only --no-coalescing", + cc_opts=compiler_opts + " --int-only", ) - extra_cred_args = f"{test_compiler_arg} -- {flag_str}" - print(f"Testing with extra credit features: {flag_str}") - for stage in stages: - print(f"Testing '{stage}' stage") - cmd = f"{prefix}./test_compiler {cc} --chapter {chapter} --stage {stage} {extra_cred_args}" - print(cmd) - subprocess.run( - cmd, - shell=True, - check=True, + # run without it + run_tests( + test_script_opts=test_script_opts + " --no-coalescing", + cc_opts=compiler_opts, ) + + +def test_chapter_20() -> None: + """Run tests for chapter 20 with coalescing. + Don't test different stages but do run both with and without + --int-only option. + """ + for _, extra_cred_opts in extra_credit_combos.items(): + test_script_opts = extra_cred_opts["test_opts"] + compiler_opts = extra_cred_opts["compiler_opts"] + # run with --int-only option + run_tests( + test_script_opts=test_script_opts + " --int-only", + cc_opts=compiler_opts + " --int-only", + ) + # run without it + run_tests(test_script_opts=test_script_opts, cc_opts=compiler_opts) + + +if __name__ == "__main__": + if CHAPTER < 19: + test_normal() + elif CHAPTER == 19: + test_chapter_19() + elif CHAPTER == 20: + if NO_COALESCING: + test_chapter_20a() + else: + test_chapter_20() + else: + exit(f"Bad chapter number {CHAPTER}") + exit(0) diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index d6e230e..7717da5 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -15,7 +15,7 @@ on: inputs: chapter: required: true - type: number + type: string os: type: string required: true diff --git a/.github/workflows/build_and_test.yaml b/.github/workflows/build_and_test.yaml index 9b7ca17..dd8e199 100644 --- a/.github/workflows/build_and_test.yaml +++ b/.github/workflows/build_and_test.yaml @@ -8,7 +8,7 @@ on: type: string chapter: required: true - type: number + type: string branch: type: string required: false diff --git a/.github/workflows/test_each_chapter.yaml b/.github/workflows/test_each_chapter.yaml index b323413..ef8bfdc 100644 --- a/.github/workflows/test_each_chapter.yaml +++ b/.github/workflows/test_each_chapter.yaml @@ -11,7 +11,7 @@ jobs: fail-fast: true matrix: os: [macos-latest, ubuntu-latest] - chapter: [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20] + chapter: ["1", "2", "3", "4", "5", "6", "7", "8", "9", "10", "11", "12", "13", "14", "15", "16", "17", "18", "19", "20a","20"] uses: ./.github/workflows/build_and_test.yaml with: chapter: ${{ matrix.chapter }}