Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Cleaner failure reports #62

Open
wants to merge 10 commits into
base: main
Choose a base branch
from
21 changes: 16 additions & 5 deletions test_framework/basic.py
Original file line number Diff line number Diff line change
@@ -151,6 +151,7 @@ def build_error_message(
expected_stdout: str,
actual: subprocess.CompletedProcess[str],
exe_name: str,
source_file: Path,
) -> str:
"""Build the error message for when a compiled test program behaves incorrectly
Called when a unittest assert* message fails
@@ -163,7 +164,7 @@ def build_error_message(
an error message
"""

msg_lines = [f"Incorrect behavior in {exe_name}"]
msg_lines = [f"Incorrect behavior in {exe_name} built from {source_file}"]

# report on incorrect return code
if expected_retcode != actual.returncode:
@@ -310,22 +311,27 @@ def validate_runs(
expected_retcode = expected["return_code"]
expected_stdout = expected.get("stdout", "")

def build_error_message_wrapped():
Copy link
Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Not sure if that's really necessary, but instead of adding the source_file three times I decided to create this closure :)

build_error_message(
expected_retcode, expected_stdout, actual, exe, source_file
)

exe = actual.args[0]
self.assertEqual(
expected_retcode,
actual.returncode,
msg=build_error_message(expected_retcode, expected_stdout, actual, exe),
msg=build_error_message_wrapped(),
)
self.assertEqual(
expected_stdout,
actual.stdout,
msg=build_error_message(expected_retcode, expected_stdout, actual, exe),
msg=build_error_message_wrapped(),
)

# none of our test programs write to stderr
self.assertFalse(
actual.stderr,
msg=build_error_message(expected_retcode, expected_stdout, actual, exe),
msg=build_error_message_wrapped(),
)

def compile_failure(self, source_file: Path) -> None:
@@ -399,8 +405,11 @@ def compile_and_run(self, source_file: Path) -> None:
print_stderr(compile_result)

# run the executable
# TODO cleaner handling if executable doesn't exist? or check that it exists above?
nlsandler marked this conversation as resolved.
Show resolved Hide resolved
exe = source_file.with_suffix("")
self.assertTrue(
exe.exists(), msg=f"Compilation did not produce executable {exe}!"
)

result = subprocess.run(
[exe], check=False, capture_output=True, text=True, timeout=10.0
)
@@ -662,6 +671,7 @@ def make_invalid_tests(
test_name = f"test_{key}"

test_method = make_invalid_test(program)
test_method.__doc__ = str(program.relative_to(TEST_DIR))
tests.append((test_name, test_method))

return tests
@@ -714,6 +724,7 @@ def make_valid_tests(
else:
# for stages besides "run", just test that compilation succeeds
test_method = make_test_valid(program)
test_method.__doc__ = str(program.relative_to(TEST_DIR))
tests.append((test_name, test_method))
return tests

18 changes: 16 additions & 2 deletions test_framework/runner.py
Original file line number Diff line number Diff line change
@@ -11,7 +11,7 @@
from functools import reduce
from operator import ior
from pathlib import Path
from typing import Iterable, Optional, List, Type
from typing import Iterable, Optional, List, Type, Any

import test_framework
import test_framework.basic
@@ -23,6 +23,16 @@
from test_framework.tacky.suite import Optimizations


class MyTextTestResult(unittest.TextTestResult):
Copy link
Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This works fine!

Now the only thing left is naming, I guess. This is a very lazy name... Should we stick to it?

An alternative could be ConciseTextTestResult or using any other of these adjectives like brief, terse, reduced, succinct...

def addFailure(self, test: Any, err: Any) -> None:
super(MyTextTestResult, self).addFailure(test, (err[0], err[1], None))

def getDescription(self, test: unittest.TestCase) -> str:
return test.shortDescription() or super(MyTextTestResult, self).getDescription(
Copy link
Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I decided to fall back on the description instead of putting "MISSING" :)

test
)


def get_optimization_flags(
latest_chapter: int,
optimization_opt: Optional[test_framework.tacky.suite.Optimizations],
@@ -518,7 +528,11 @@ def main() -> int:
unittest.installHandler()

# run it
runner = unittest.TextTestRunner(verbosity=args.verbose, failfast=args.failfast)
runner = unittest.TextTestRunner(
verbosity=args.verbose,
failfast=args.failfast,
resultclass=MyTextTestResult,
)
result = runner.run(test_suite)
if result.wasSuccessful():
return 0