Merge pull request #34 from darrenburns/xfail-output

Outputting expected failures and unexpected passes

setup.py

@@ -3,7 +3,7 @@
 with open("README.md", "r") as fh:
     long_description = fh.read()
 
-version = "0.6.5a0"
+version = "0.7.0a0"
 
 setup(
     name="ward",

tests/test_suite.py

@@ -168,5 +168,5 @@ def all_keys_less_than_length_10(cities):
      .is_instance_of(dict)
      .contains("tokyo")
      .has_length(6)
-     .equals({"edinburgh": "scasdnd", "tokyo": "japan", "london": "england"}))
+     .equals(cities))
 # endregion example

tests/test_util.py

@@ -0,0 +1,30 @@
+from ward import expect
+from ward.test_result import TestResult, TestOutcome
+from ward.util import get_exit_code, ExitCode
+
+
+def test_get_exit_code_returns_success_when_skip_and_pass_and_xfail_present(example_test):
+    test_results = [
+        TestResult(test=example_test, outcome=TestOutcome.PASS),
+        TestResult(test=example_test, outcome=TestOutcome.SKIP),
+        TestResult(test=example_test, outcome=TestOutcome.XFAIL),
+    ]
+    exit_code = get_exit_code(test_results)
+
+    expect(exit_code).equals(ExitCode.SUCCESS)
+
+
+def test_get_exit_code_returns_success_when_no_test_results():
+    exit_code = get_exit_code([])
+
+    expect(exit_code).equals(ExitCode.SUCCESS)
+
+
+def test_get_exit_code_returns_fail_when_xpass_present(example_test):
+    test_results = [
+        TestResult(test=example_test, outcome=TestOutcome.XPASS),
+        TestResult(test=example_test, outcome=TestOutcome.PASS),
+    ]
+    exit_code = get_exit_code(test_results)
+
+    expect(exit_code).equals(ExitCode.FAILED)

ward/run.py

@@ -9,8 +9,8 @@
 from ward.collect import get_info_for_modules, get_tests_in_modules, load_modules
 from ward.fixtures import fixture_registry
 from ward.suite import Suite
-from ward.terminal import ExitCode, SimpleTestResultWrite
-from ward.test_result import TestOutcome
+from ward.terminal import SimpleTestResultWrite
+from ward.util import get_exit_code
 
 init()
 
@@ -48,9 +48,6 @@ def run(path, filter, fail_limit):
     time_taken = default_timer() - start_run
     writer.output_test_result_summary(results, time_taken)
 
-    if any(r.outcome == TestOutcome.FAIL for r in results):
-        exit_code = ExitCode.TEST_FAILED
-    else:
-        exit_code = ExitCode.SUCCESS
+    exit_code = get_exit_code(results)
 
     sys.exit(exit_code.value)

ward/suite.py

@@ -28,14 +28,25 @@ def generate_test_runs(self) -> Generator[TestResult, None, None]:
             try:
                 resolved_fixtures = test.resolve_args(self.fixture_registry)
             except FixtureExecutionError as e:
-                yield TestResult(test, TestOutcome.FAIL, e, message="[Error] " + str(e))
+                yield TestResult(test, TestOutcome.FAIL, e)
                 continue
             try:
                 resolved_vals = {k: fix.resolved_val for (k, fix) in resolved_fixtures.items()}
+
+                # Run the test
                 test(**resolved_vals)
-                yield TestResult(test, TestOutcome.PASS, None, message="")
+
+                # The test has completed without exception and therefore passed
+                if test.marker == WardMarker.XFAIL:
+                    yield TestResult(test, TestOutcome.XPASS, None)
+                else:
+                    yield TestResult(test, TestOutcome.PASS, None)
+
             except Exception as e:
-                yield TestResult(test, TestOutcome.FAIL, e, message="")
+                if test.marker == WardMarker.XFAIL:
+                    yield TestResult(test, TestOutcome.XFAIL, e)
+                else:
+                    yield TestResult(test, TestOutcome.FAIL, e)
             finally:
                 for fixture in resolved_fixtures.values():
                     if fixture.is_generator_fixture:

ward/terminal.py

@@ -2,8 +2,7 @@
 import sys
 import traceback
 from dataclasses import dataclass
-from enum import Enum
-from typing import Generator, List, Optional, Tuple
+from typing import Generator, List, Optional, Dict
 
 from colorama import Fore, Style
 from termcolor import colored
@@ -12,19 +11,14 @@
 from ward.expect import ExpectationFailed
 from ward.suite import Suite
 from ward.test_result import TestOutcome, TestResult
+from ward.util import get_exit_code, ExitCode
 
 
 def truncate(s: str, num_chars: int) -> str:
     suffix = "..." if len(s) > num_chars - 3 else ""
     return s[:num_chars] + suffix
 
 
-class ExitCode(Enum):
-    SUCCESS = 0
-    TEST_FAILED = 1
-    ERROR = 2
-
-
 class TestResultWriterBase:
 
     def __init__(self, suite: Suite):
@@ -110,10 +104,12 @@ def output_single_test_result(self, test_result: TestResult):
             TestOutcome.PASS: "green",
             TestOutcome.SKIP: "blue",
             TestOutcome.FAIL: "red",
+            TestOutcome.XFAIL: "magenta",
+            TestOutcome.XPASS: "yellow",
         }
         colour = outcome_to_colour[test_result.outcome]
         bg = f"on_{colour}"
-        padded_outcome = f" {test_result.outcome.name} "
+        padded_outcome = f" {test_result.outcome.name[:4]} "
         mod_name = lightblack(f"{test_result.test.module.__name__}.")
         print(colored(padded_outcome, color='grey', on_color=bg),
               mod_name + test_result.test.name)
@@ -161,29 +157,44 @@ def output_why_test_failed(self, test_result: TestResult):
         print(Style.RESET_ALL)
 
     def output_test_result_summary(self, test_results: List[TestResult], time_taken: float):
-        num_passed, num_failed, num_skipped = self._get_num_passed_failed_skipped(test_results)
-        print(self.generate_chart(num_passed=num_passed, num_failed=num_failed, num_skipped=num_skipped), "")
-
-        if any(r.outcome == TestOutcome.FAIL for r in test_results):
-            result = colored("FAILED", color='red', attrs=["bold"])
+        outcome_counts = self._get_outcome_counts(test_results)
+        chart = self.generate_chart(
+            num_passed=outcome_counts[TestOutcome.PASS],
+            num_failed=outcome_counts[TestOutcome.FAIL],
+            num_skipped=outcome_counts[TestOutcome.SKIP],
+            num_xfail=outcome_counts[TestOutcome.XFAIL],
+            num_unexp=outcome_counts[TestOutcome.XPASS],
+        )
+        print(chart, "")
+
+        exit_code = get_exit_code(test_results)
+        if exit_code == ExitCode.FAILED:
+            result = colored(exit_code.name, color='red', attrs=["bold"])
         else:
-            result = colored("PASSED", color='green', attrs=["bold"])
+            result = colored(exit_code.name, color='green', attrs=["bold"])
         print(f"{result} in {time_taken:.2f} seconds [ "
-              f"{colored(str(num_failed) + ' failed', color='red')}  "
-              f"{colored(str(num_skipped) + ' skipped', color='blue')}  "
-              f"{colored(str(num_passed) + ' passed', color='green')} ]")
-
-    def generate_chart(self, num_passed, num_failed, num_skipped):
-        pass_pct = num_passed / max(num_passed + num_failed + num_skipped, 1)
-        fail_pct = num_failed / max(num_passed + num_failed + num_skipped, 1)
-        skip_pct = 1.0 - pass_pct - fail_pct
+              f"{colored(str(outcome_counts[TestOutcome.FAIL]) + ' failed', color='red')}  "
+              f"{colored(str(outcome_counts[TestOutcome.XPASS]) + ' xpassed', color='yellow')}  "
+              f"{colored(str(outcome_counts[TestOutcome.XFAIL]) + ' xfailed', color='magenta')}  "
+              f"{colored(str(outcome_counts[TestOutcome.SKIP]) + ' skipped', color='blue')}  "
+              f"{colored(str(outcome_counts[TestOutcome.PASS]) + ' passed', color='green')} ]")
+
+    def generate_chart(self, num_passed, num_failed, num_skipped, num_xfail, num_unexp):
+        num_tests = num_passed + num_failed + num_skipped + num_xfail + num_unexp
+        pass_pct = num_passed / max(num_tests, 1)
+        fail_pct = num_failed / max(num_tests, 1)
+        xfail_pct = num_xfail / max(num_tests, 1)
+        unexp_pct = num_unexp / max(num_tests, 1)
+        skip_pct = 1.0 - pass_pct - fail_pct - xfail_pct - unexp_pct
 
         num_green_bars = int(pass_pct * self.terminal_size.width)
         num_red_bars = int(fail_pct * self.terminal_size.width)
         num_blue_bars = int(skip_pct * self.terminal_size.width)
+        num_yellow_bars = int(unexp_pct * self.terminal_size.width)
+        num_magenta_bars = int(xfail_pct * self.terminal_size.width)
 
         # Rounding to integers could leave us a few bars short
-        num_bars_remaining = self.terminal_size.width - num_green_bars - num_red_bars - num_blue_bars
+        num_bars_remaining = self.terminal_size.width - num_green_bars - num_red_bars - num_blue_bars - num_yellow_bars - num_magenta_bars
         if num_bars_remaining and num_green_bars:
             num_green_bars += 1
             num_bars_remaining -= 1
@@ -196,21 +207,28 @@ def generate_chart(self, num_passed, num_failed, num_skipped):
             num_blue_bars += 1
             num_bars_remaining -= 1
 
-        assert num_bars_remaining == 0
+        if num_bars_remaining and num_yellow_bars:
+            num_yellow_bars += 1
+            num_bars_remaining -= 1
 
-        if self.terminal_size.width - num_green_bars - num_red_bars == 1:
-            num_green_bars += 1
+        if num_bars_remaining and num_magenta_bars:
+            num_magenta_bars += 1
+            num_bars_remaining -= 1
 
         return (colored("F" * num_red_bars, color="red", on_color="on_red") +
+                colored("U" * num_yellow_bars, color="yellow", on_color="on_yellow") +
+                colored("x" * num_magenta_bars, color="magenta", on_color="on_magenta") +
                 colored("s" * num_blue_bars, color="blue", on_color="on_blue") +
                 colored("." * num_green_bars, color="green", on_color="on_green"))
 
     def output_test_run_post_failure_summary(self, test_results: List[TestResult]):
         pass
 
-    def _get_num_passed_failed_skipped(self, test_results: List[TestResult]) -> Tuple[int, int, int]:
-        num_passed = len([r for r in test_results if r.outcome == TestOutcome.PASS])
-        num_failed = len([r for r in test_results if r.outcome == TestOutcome.FAIL])
-        num_skipped = len([r for r in test_results if r.outcome == TestOutcome.SKIP])
-
-        return num_passed, num_failed, num_skipped
+    def _get_outcome_counts(self, test_results: List[TestResult]) -> Dict[TestOutcome, int]:
+        return {
+            TestOutcome.PASS: len([r for r in test_results if r.outcome == TestOutcome.PASS]),
+            TestOutcome.FAIL: len([r for r in test_results if r.outcome == TestOutcome.FAIL]),
+            TestOutcome.SKIP: len([r for r in test_results if r.outcome == TestOutcome.SKIP]),
+            TestOutcome.XFAIL: len([r for r in test_results if r.outcome == TestOutcome.XFAIL]),
+            TestOutcome.XPASS: len([r for r in test_results if r.outcome == TestOutcome.XPASS]),
+        }

ward/test_result.py

@@ -9,12 +9,13 @@ class TestOutcome(Enum):
     PASS = auto()
     FAIL = auto()
     SKIP = auto()
-    XFAIL = auto()
+    XFAIL = auto()  # expected fail
+    XPASS = auto()  # unexpected pass
 
 
 @dataclass
 class TestResult:
     test: Test
     outcome: TestOutcome
-    error: Optional[Exception]
-    message: str
+    error: Optional[Exception] = None
+    message: str = ""

ward/util.py

@@ -0,0 +1,18 @@
+from enum import Enum
+from typing import Iterable
+
+from ward.test_result import TestOutcome, TestResult
+
+
+class ExitCode(Enum):
+    SUCCESS = 0
+    FAILED = 1
+    ERROR = 2
+
+
+def get_exit_code(results: Iterable[TestResult]) -> ExitCode:
+    if any(r.outcome == TestOutcome.FAIL or r.outcome == TestOutcome.XPASS for r in results):
+        exit_code = ExitCode.FAILED
+    else:
+        exit_code = ExitCode.SUCCESS
+    return exit_code