Catch exception in tokenization subsystem per line, log CRITICAL (#223)

Catch exception in tokenization subsystem per line, log CRITICAL

Reviewed-by: https://github.com/apps/ansible-zuul

ansible_navigator/ui_framework/colorize.py

@@ -11,11 +11,14 @@
 import functools
 
 from itertools import chain
-from ..tm_tokenize.grammars import Grammars
-from ..tm_tokenize.tokenize import tokenize
+
 from .curses_defs import CursesLine
 from .curses_defs import CursesLinePart
 
+from ..tm_tokenize.grammars import Grammars
+from ..tm_tokenize.tokenize import tokenize
+
+
 CURSES_STYLES = {
     0: None,
     1: getattr(curses, "A_BOLD", None),
@@ -104,10 +107,27 @@ def render(self, doc, scope):
             lines = []
             for line_idx, line in enumerate(doc.splitlines()):
                 first_line = line_idx == 0
-                state, regions = tokenize(compiler, state, line, first_line)
-                lines.append((regions, line))
-            return columns_and_colors(lines, self._schema)
-        res = [[{"column": 0, "chars": l, "color": None}] for l in doc.splitlines()]  # noqa: E741
+                try:
+                    state, regions = tokenize(compiler, state, line, first_line)
+                except Exception as exc:  # pylint: disable=broad-except
+                    self._logger.critical(
+                        (
+                            "An unexpected error occured within the tokenization"
+                            " subsystem.  Please log an issue with the following:"
+                        )
+                    )
+                    self._logger.critical(
+                        "  Err: '%s', Scope: '%s', Line follows....", str(exc), scope
+                    )
+                    self._logger.critical("  '%s'", line)
+                    self._logger.critical("  The current content will be rendered without color")
+                    break
+                else:
+                    lines.append((regions, line))
+            else:
+                return columns_and_colors(lines, self._schema)
+
+        res = [[{"column": 0, "chars": doc_line, "color": None}] for doc_line in doc.splitlines()]
         return res
 
 

tests/unit/ui_framework/test_colorize.py

@@ -0,0 +1,54 @@
+"""tests for colorize
+"""
+import json
+import os
+
+from unittest.mock import patch
+
+from ansible_navigator.ui_framework.colorize import Colorize
+
+from ansible_navigator.yaml import human_dump
+
+SHARE_DIR = os.path.abspath(
+    os.path.join(os.path.basename(__file__), "..", "share", "ansible_navigator")
+)
+
+
+def test_basic_success_json():
+    """Ensure the json string is returned as 1 lines, 5 parts and can be reassembled
+    to the json string"""
+    sample = json.dumps({"test": "data"})
+    result = Colorize(SHARE_DIR).render(doc=sample, scope="source.json")
+    assert len(result) == 1
+    assert len(result[0]) == 5
+    assert "".join(line_part["chars"] for line_part in result[0]) == sample
+
+
+def test_basic_success_yaml():
+    """Ensure the yaml string is returned as 2 lines, with 1 and 3 parts
+    respectively, ensure the parts of the second line can be reaseembled to
+    the second line of the yaml string
+    """
+    sample = human_dump({"test": "data"})
+    result = Colorize(SHARE_DIR).render(doc=sample, scope="source.yaml")
+    assert len(result) == 2
+    assert len(result[0]) == 1
+    assert result[0][0]["chars"] == sample.splitlines()[0]
+    assert len(result[1]) == 3
+    assert "".join(line_part["chars"] for line_part in result[1]) == sample.splitlines()[1]
+
+
+@patch("ansible_navigator.ui_framework.colorize.tokenize")
+def test_graceful_failure(mocked_func, caplog):
+    """Ensure a tokenization error returns the original one line json string
+    w/o color and the log reflects the critical error
+    """
+    mocked_func.side_effect = ValueError()
+    sample = json.dumps({"test": "data"})
+    result = Colorize(SHARE_DIR).render(doc=sample, scope="source.json")
+    assert len(result) == 1
+    assert len(result[0]) == 1
+    assert result[0][0]["chars"] == sample
+    assert result[0][0]["color"] is None
+    assert result[0][0]["column"] == 0
+    assert "rendered without color" in caplog.text