0.2: RELEASE (#3)

* 0.2 - Refactored intents and responses handlers for better logic split
* 0.2 - Stories refactoring
* 0.2 - Icons for trees depending on type of item
* 0.2 - File structure refactoring
* 0.2 - App is working again after refactoring
* 0.2 - nlu.json files support, formating

.gitignore

@@ -4,3 +4,5 @@ export/*
 build/*
 rasa_storyteller.egg-info/*
 dist/*
+.mypy_cache/*
+*/export/*

CHANGELOG.md

@@ -0,0 +1,18 @@
+# Changelog
+
+All notable changes to this project will be documented in this file.
+
+The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
+and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
+
+
+## [0.2.0] - 2020-07-01
+### Changed
+- Logic of PySimpleGui UI is split from processing data
+- Working with trees using [AnyTree](https://pypi.org/project/anytree/2.8.0/) is much easier now
+- Import and export for nlu.json file is supported (#2)
+- All code is formatted with [Black](https://black.readthedocs.io/en/stable/index.html#)
+
+## [0.1.0] - 2020-03-23
+### Added
+- Initial release

handlers/Handler.py → backend/handlers/AbstractHandler.py

@@ -1,8 +1,7 @@
 from abc import ABC, abstractmethod
 
 
-class Handler(ABC):
-
+class AbstractHandler(ABC):
     @abstractmethod
     def __init__(self, filename, *args):
         pass
@@ -14,7 +13,3 @@ def import_data(self):
     @abstractmethod
     def export_data(self):
         pass
-
-    @abstractmethod
-    def sort_alphabetically(self):
-        pass

backend/handlers/Exporter.py

@@ -0,0 +1,41 @@
+import json
+import shutil
+
+import yaml
+
+
+class Exporter(object):
+    def __init__(
+        self, nlu, resp, stories, nlu_file_md, nlu_file_json, domain_file, stories_file
+    ):
+        self.nlu = nlu
+        self.resp = resp
+        self.stories = stories
+        self.nlu_file_md = nlu_file_md
+        self.nlu_file_json = nlu_file_json
+        self.domain_file = domain_file
+        self.stories_file = stories_file
+
+    def export(self):
+        nlu_data = self.nlu.export_data()
+        resp_data = self.resp.export_data()
+        stories_data = self.stories.export_data()
+
+        with open(self.nlu_file_md, "w", encoding="utf-8") as nf:
+            nlu_data["result"].seek(0)
+            shutil.copyfileobj(nlu_data["result"], nf)
+
+        with open(self.nlu_file_json, "w", encoding="utf-8") as nfj:
+            nfj.write(json.dumps(nlu_data["result_json"], ensure_ascii=False, indent=4))
+
+        domain_data = {
+            "actions": resp_data["actions"],
+            "intents": nlu_data["intents"],
+            "responses": resp_data["responses"],
+        }
+        with open(self.domain_file, "w", encoding="utf-8") as df:
+            df.write(yaml.safe_dump(domain_data, allow_unicode=True))
+
+        with open(self.stories_file, "w", encoding="utf-8") as sf:
+            stories_data.seek(0)
+            shutil.copyfileobj(stories_data, sf)

backend/handlers/ItemsWithExamplesHandler.py

@@ -0,0 +1,89 @@
+from abc import ABC
+
+from PySimpleGUI import TreeData
+from anytree import Node
+from anytree.search import find
+
+from backend.handlers.AbstractHandler import AbstractHandler
+from backend.models.BaseNode import BaseNode, BaseItem
+from backend.models.Intent import IntentNode
+from common.constants import QUESTION_ICON, ANSWER_ICON
+
+
+class ItemsWithExamplesHandler(AbstractHandler, ABC):
+    def __init__(self, filename, *args):
+        super().__init__(filename, *args)
+        self.filename = filename
+        self.items = []
+        self.tree = Node("root", text="")
+        self.parent_nodes_class = BaseNode
+        self.parent_object_class = BaseItem
+        self.child_nodes_class = Node
+
+    def add_to_items(self, key):
+        if key in self.items:
+            raise ValueError(f"There is already item with key {key} in this list.")
+        else:
+            self.items.append(key)
+
+    def export_to_pysg_tree(self):
+        parent_icon = (
+            QUESTION_ICON if self.parent_nodes_class == IntentNode else ANSWER_ICON
+        )
+        tree_data = TreeData()
+        for item in self.tree.children:
+            tree_data.Insert(
+                parent="",
+                key=item.item.name,
+                text=item.item.name,
+                values=[],
+                icon=parent_icon,
+            )
+            for example in item.children:
+                tree_data.Insert(
+                    parent=item.item.name,
+                    key=example.name,
+                    text=example.name,
+                    values=[],
+                    icon=ANSWER_ICON,
+                )
+        return tree_data
+
+    def add_node_with_kids(self, parent_name, *kids):
+        item = self.parent_object_class(name=parent_name)
+        current_parent = self.parent_nodes_class(item, parent=self.tree)
+        self.add_to_items(parent_name)
+        for kid in kids:
+            self.child_nodes_class(name=kid, parent=current_parent)
+            self.add_to_items(kid)
+
+    def add_example_to_node(self, parent, text):
+        selected_node = find(self.tree, lambda node: node.name == parent, maxlevel=3)
+        if isinstance(selected_node, self.parent_nodes_class):
+            parent = selected_node
+        elif isinstance(selected_node, self.child_nodes_class):
+            parent = selected_node.parent
+        else:
+            raise ValueError("Can't find item to add example.")
+        self.child_nodes_class(name=text, parent=parent)
+        self.add_to_items(text)
+
+    def update_node_value(self, node, new_value):
+        node = find(self.tree, lambda n: n.name == node, maxlevel=3)
+        self.items.remove(node.name)
+        self.add_to_items(new_value)
+        node.name = new_value
+        if isinstance(node, self.parent_nodes_class):
+            node.item.name = new_value
+            for story_item in node.item.story_tree:
+                story_item.name = new_value
+
+    def remove_node(self, node):
+        node = find(self.tree, lambda n: n.name == node, maxlevel=3)
+        if not hasattr(node, "item") or not node.item.story_tree:
+            node.parent = None
+            self.items.remove(node.name)
+            for kid in node.children:
+                self.items.remove(kid.name)
+        else:
+            raise ValueError("Item is used in stories and can't be removed.")

backend/handlers/NLUHandler.py

@@ -0,0 +1,77 @@
+import json
+from io import StringIO  # Python3
+
+import markdown_generator as mg
+from anytree.search import find
+
+from backend.handlers.ItemsWithExamplesHandler import ItemsWithExamplesHandler
+from backend.models.Intent import IntentNode, Intent, IntentExample
+
+
+class NLUHandler(ItemsWithExamplesHandler):
+    def __init__(self, filename, *args):
+        super().__init__(filename, *args)
+        self.parent_object_class = Intent
+        self.parent_nodes_class = IntentNode
+        self.child_nodes_class = IntentExample
+
+    def import_data(self):
+        current_intent = None
+        try:
+            with open(self.filename, "r", encoding="utf-8") as df:
+                nlu = json.loads(df.read())
+                for example in nlu["rasa_nlu_data"]["common_examples"]:
+                    if example["intent"] not in self.items:
+                        current_intent = IntentNode(
+                            Intent(name=example["intent"]), parent=self.tree
+                        )
+                        self.add_to_items(example["intent"])
+                    else:
+                        current_intent = find(
+                            self.tree,
+                            lambda node: node.name == example["intent"],
+                            maxlevel=2,
+                        )
+                    IntentExample(name=example["text"], parent=current_intent)
+                    self.add_to_items(example["text"])
+
+        except json.JSONDecodeError:  # suggest it's markdown
+            with open(self.filename, "r", encoding="utf-8") as df:
+                nlu = df.readlines()
+                for line in nlu:
+                    if line.startswith("## intent:"):
+                        heading = line.split("## intent:")[1].strip()
+                        current_intent = IntentNode(
+                            Intent(name=heading), parent=self.tree
+                        )
+                        self.add_to_items(heading)
+                    if line.startswith("- "):
+                        text_example = line.split("- ")[1].strip()
+                        IntentExample(name=text_example, parent=current_intent)
+                        self.add_to_items(text_example)
+
+    def export_data(self):
+        result = StringIO()
+        intents = []
+        writer = mg.Writer(result)
+        result_json = {
+            "rasa_nlu_data": {
+                "common_examples": [],
+                "regex_features": [],
+                "lookup_tables": [],
+                "entity_synonyms": [],
+            }
+        }
+
+        for intent in self.tree.children:
+            intents.append(intent.item.name)
+            writer.write_heading(f"intent:{intent.item.name}", 2)
+            examples_list = [example.name.strip() for example in intent.children]
+            examples_md = [f"- {example}" for example in examples_list]
+            writer.writelines(examples_md)
+            for example in examples_list:
+                result_json["rasa_nlu_data"]["common_examples"].append(
+                    {"intent": intent.item.name, "example": example}
+                )
+
+        return {"result": result, "result_json": result_json, "intents": intents}

backend/handlers/ResponseHandler.py

@@ -0,0 +1,37 @@
+import yaml
+
+from backend.handlers.ItemsWithExamplesHandler import ItemsWithExamplesHandler
+from backend.models.Response import Response, ResponseNode, ResponseExample
+
+
+class ResponseHandler(ItemsWithExamplesHandler):
+    def __init__(self, filename, *args):
+        super().__init__(filename, *args)
+        self.parent_object_class = Response
+        self.parent_nodes_class = ResponseNode
+        self.child_nodes_class = ResponseExample
+
+    def import_data(self):
+        with open(self.filename, "r", encoding="utf-8") as domain_file:
+            domain_data = yaml.safe_load(domain_file.read())
+            for response, texts in domain_data["responses"].items():
+                response_name = response.split("utter_")[-1].strip()
+                current_response = ResponseNode(
+                    Response(name=response_name), parent=self.tree
+                )
+                self.add_to_items(response_name)
+                for text in texts:
+                    ResponseExample(name=text["text"], parent=current_response)
+                    self.add_to_items(text["text"])
+
+    def export_data(self):
+        responses = []
+        result = {}
+        for response in self.tree.children:
+            responses.append(f"utter_{response.item.name}")
+            result[f"utter_{response.name}"] = []
+            for kid in response.children:
+                result[f"utter_{response.item.name}"].append(
+                    {"text": str(kid.name.strip())}
+                )
+        return {"responses": result, "actions": responses}