unittests/end2end (#140)

.github/workflows/coverage.yml

@@ -28,9 +28,14 @@ jobs:
       run: |
         sudo apt install libssl-dev libfann-dev portaudio19-dev libpulse-dev
         pip install -r requirements/test.txt
+        pip install ./test/end2end/session/skill-ovos-hello-world
+        pip install ./test/end2end/session/skill-ovos-fallback-unknown
+        pip install ./test/end2end/session/skill-ovos-fallback-unknownv1
+        pip install ./test/end2end/session/skill-converse_test
     - name: Generate coverage report
       run: |
-        pytest --cov=./ovos_workshop --cov-report=xml
+        pytest --cov=./ovos_workshop --cov-report xml test/unittests
+        pytest --cov-append --cov=ovos_workshop --cov-report xml test/end2end
     - name: Upload coverage to Codecov
       uses: codecov/codecov-action@v3
       with:

.github/workflows/unit_tests.yml

@@ -59,9 +59,14 @@ jobs:
         run: |
           sudo apt install libssl-dev libfann-dev portaudio19-dev libpulse-dev
           pip install -r requirements/test.txt
+          pip install ./test/end2end/session/skill-ovos-hello-world
+          pip install ./test/end2end/session/skill-ovos-fallback-unknown
+          pip install ./test/end2end/session/skill-ovos-fallback-unknownv1
+          pip install ./test/end2end/session/skill-converse_test
       - name: Run unittests
         run: |
           pytest --cov=ovos_workshop --cov-report xml test/unittests
+          pytest --cov-append --cov=ovos_workshop --cov-report xml test/end2end
           # NOTE: additional pytest invocations should also add the --cov-append flag
           #       or they will overwrite previous invocations' coverage reports
           #       (for an example, see OVOS Skill Manager's workflow)

requirements/test.txt

@@ -1,5 +1,4 @@
-ovos-core~=0.0.7
-neon-lang-plugin-libretranslate~=0.2
-adapt-parser~=0.5
+ovos-core>=0.0.8a50
 pytest
 pytest-cov
+ovos-translate-server-plugin

test/end2end/session/minicroft.py

@@ -0,0 +1,69 @@
+from time import sleep
+from ovos_bus_client.session import SessionManager, Session
+from ovos_core.intent_services import IntentService
+from ovos_core.skill_manager import SkillManager
+from ovos_plugin_manager.skills import find_skill_plugins
+from ovos_utils.log import LOG
+from ovos_utils.messagebus import FakeBus
+from ovos_utils.process_utils import ProcessState
+from ovos_workshop.skills.fallback import FallbackSkill
+
+
+class MiniCroft(SkillManager):
+    def __init__(self, skill_ids, *args, **kwargs):
+        bus = FakeBus()
+        super().__init__(bus, *args, **kwargs)
+        self.skill_ids = skill_ids
+        self.intent_service = self._register_intent_services()
+
+    def _register_intent_services(self):
+        """Start up the all intent services and connect them as needed.
+
+        Args:
+            bus: messagebus client to register the services on
+        """
+        service = IntentService(self.bus)
+        # Register handler to trigger fallback system
+        self.bus.on(
+            'mycroft.skills.fallback',
+            FallbackSkill.make_intent_failure_handler(self.bus)
+        )
+        return service
+
+    def load_plugin_skills(self):
+        LOG.info("loading skill plugins")
+        plugins = find_skill_plugins()
+        for skill_id, plug in plugins.items():
+            LOG.debug(skill_id)
+            if skill_id not in self.skill_ids:
+                continue
+            if skill_id not in self.plugin_skills:
+                self._load_plugin_skill(skill_id, plug)
+
+    def run(self):
+        """Load skills and update periodically from disk and internet."""
+        self.status.set_alive()
+
+        self.load_plugin_skills()
+
+        self.status.set_ready()
+
+        LOG.info("Skills all loaded!")
+
+    def stop(self):
+        super().stop()
+        SessionManager.bus = None
+        SessionManager.sessions = {}
+        SessionManager.default_session = SessionManager.sessions["default"] = Session("default")
+
+
+def get_minicroft(skill_id):
+    if isinstance(skill_id, str):
+        skill_id = [skill_id]
+    assert isinstance(skill_id, list)
+    croft1 = MiniCroft(skill_id)
+    croft1.start()
+    while croft1.status.state != ProcessState.READY:
+        sleep(0.2)
+    return croft1
+

test/end2end/session/skill-converse_test/__init__.py

@@ -0,0 +1,82 @@
+from ovos_workshop.decorators import killable_intent
+from ovos_workshop.skills.ovos import OVOSSkill
+from mycroft.skills import intent_file_handler
+from time import sleep
+
+
+class TestAbortSkill(OVOSSkill):
+    """
+    send "mycroft.skills.abort_question" and confirm only get_response is aborted
+    send "mycroft.skills.abort_execution" and confirm the full intent is aborted, except intent3
+    send "my.own.abort.msg" and confirm intent3 is aborted
+    say "stop" and confirm all intents are aborted
+    """
+    def initialize(self):
+        self.stop_called = False
+        self._converse = False
+        self.bus.on("test_activate", self.do_activate)
+        self.bus.on("test_deactivate", self.do_deactivate)
+
+    def do_activate(self, message):
+        self.activate()
+
+    def do_deactivate(self, message):
+        self.deactivate()
+
+    @intent_file_handler("converse_on.intent")
+    def handle_converse_on(self, message):
+        self._converse = True
+        self.speak("on")
+
+    @intent_file_handler("converse_off.intent")
+    def handle_converse_off(self, message):
+        self._converse = False
+        self.speak("off")
+
+    def handle_intent_aborted(self):
+        self.speak("I am dead")
+
+    @intent_file_handler("test_get_response.intent")
+    def handle_test_get_response(self, message):
+        ans = self.get_response("get", num_retries=1)
+        self.speak(ans or "ERROR")
+
+    @intent_file_handler("test_get_response3.intent")
+    def handle_test_get_response3(self, message):
+        ans = self.get_response(num_retries=3)
+        self.speak(ans or "ERROR")
+
+    @killable_intent(callback=handle_intent_aborted)
+    @intent_file_handler("test.intent")
+    def handle_test_abort_intent(self, message):
+        self.stop_called = False
+        self.my_special_var = "changed"
+        while True:
+            sleep(1)
+            self.speak("still here")
+
+    @intent_file_handler("test2.intent")
+    @killable_intent(callback=handle_intent_aborted)
+    def handle_test_get_response_intent(self, message):
+        self.stop_called = False
+        self.my_special_var = "CHANGED"
+        ans = self.get_response("question", num_retries=99999)
+        self.log.debug("get_response returned: " + str(ans))
+        if ans is None:
+            self.speak("question aborted")
+
+    @killable_intent(msg="my.own.abort.msg", callback=handle_intent_aborted)
+    @intent_file_handler("test3.intent")
+    def handle_test_msg_intent(self, message):
+        self.stop_called = False
+        if self.my_special_var != "default":
+            self.speak("someone forgot to cleanup")
+        while True:
+            sleep(1)
+            self.speak("you can't abort me")
+
+    def stop(self):
+        self.stop_called = True
+
+    def converse(self, message):
+        return self._converse

test/end2end/session/skill-converse_test/locale/en-us/converse_off.intent

@@ -0,0 +1 @@
+no

test/end2end/session/skill-converse_test/locale/en-us/converse_on.intent

@@ -0,0 +1 @@
+yes

test/end2end/session/skill-converse_test/locale/en-us/get.dialog

@@ -0,0 +1 @@
+give me an answer

test/end2end/session/skill-converse_test/locale/en-us/question.dialog

@@ -0,0 +1 @@
+this is a question

test/end2end/session/skill-converse_test/locale/en-us/test.intent

@@ -0,0 +1 @@
+test

test/end2end/session/skill-converse_test/locale/en-us/test2.intent

@@ -0,0 +1 @@
+test again

test/end2end/session/skill-converse_test/locale/en-us/test3.intent

@@ -0,0 +1 @@
+one more test

test/end2end/session/skill-converse_test/locale/en-us/test_get_response.intent

@@ -0,0 +1 @@
+test get response

test/end2end/session/skill-converse_test/locale/en-us/test_get_response3.intent

@@ -0,0 +1,2 @@
+3 prompts
+three prompts

test/end2end/session/skill-converse_test/setup.py

@@ -0,0 +1,39 @@
+#!/usr/bin/env python3
+from setuptools import setup
+from os import getenv, path, walk
+
+
+def find_resource_files():
+    resource_base_dirs = ("locale", "ui", "vocab", "dialog", "regex")
+    base_dir = path.dirname(__file__)
+    package_data = ["skill.json"]
+    for res in resource_base_dirs:
+        if path.isdir(path.join(base_dir, res)):
+            for (directory, _, files) in walk(path.join(base_dir, res)):
+                if files:
+                    package_data.append(
+                        path.join(directory.replace(base_dir, "").lstrip('/'),
+                                  '*'))
+    return package_data
+
+
+# skill_id=package_name:SkillClass
+PLUGIN_ENTRY_POINT = 'ovos-tskill-abort.openvoiceos=ovos_tskill_abort:TestAbortSkill'
+
+setup(
+    # this is the package name that goes on pip
+    name='ovos-tskill-abort',
+    version='0.0.1',
+    description='this is a OVOS test skill for the killable_intents decorator',
+    url='https://github.com/OpenVoiceOS/skill-abort-test',
+    author='JarbasAi',
+    author_email='[email protected]',
+    license='Apache-2.0',
+    package_dir={"ovos_tskill_abort": ""},
+    package_data={'ovos_tskill_abort': find_resource_files()},
+    packages=['ovos_tskill_abort'],
+    include_package_data=True,
+    install_requires=["ovos-workshop"],
+    keywords='ovos skill plugin',
+    entry_points={'ovos.plugin.skill': PLUGIN_ENTRY_POINT}
+)

test/end2end/session/skill-ovos-fallback-unknown/__init__.py

@@ -0,0 +1,10 @@
+from ovos_workshop.skills.fallback import FallbackSkill
+from ovos_workshop.decorators import fallback_handler
+
+
+class UnknownSkill(FallbackSkill):
+
+    @fallback_handler(priority=100)
+    def handle_fallback(self, message):
+        self.speak_dialog('unknown')
+        return True

test/end2end/session/skill-ovos-fallback-unknown/locale/en-us/unknown.dialog

@@ -0,0 +1,9 @@
+I'm sorry, I don't understand.
+I don't know what that means.
+I don't understand, but I'm learning new things everyday.
+Sorry, I didn't catch that.
+Sorry, I don't understand.
+I don't understand.
+I'm not sure I understood you.
+You might have to say that a different way.
+Please rephrase your request.

test/end2end/session/skill-ovos-fallback-unknown/setup.py

@@ -0,0 +1,39 @@
+#!/usr/bin/env python3
+from setuptools import setup
+from os import walk, path
+
+URL = "https://github.com/OpenVoiceOS/skill-ovos-fallback-unknown"
+SKILL_CLAZZ = "UnknownSkill"  # needs to match __init__.py class name
+PYPI_NAME = "ovos-skill-fallback-unknown"  # pip install PYPI_NAME
+
+# below derived from github url to ensure standard skill_id
+SKILL_AUTHOR, SKILL_NAME = URL.split(".com/")[-1].split("/")
+SKILL_PKG = SKILL_NAME.lower().replace('-', '_')
+PLUGIN_ENTRY_POINT = f'{SKILL_NAME.lower()}.{SKILL_AUTHOR.lower()}={SKILL_PKG}:{SKILL_CLAZZ}'
+# skill_id=package_name:SkillClass
+
+
+def find_resource_files():
+    resource_base_dirs = ("locale", "ui", "vocab", "dialog", "regex", "skill")
+    base_dir = path.dirname(__file__)
+    package_data = ["*.json"]
+    for res in resource_base_dirs:
+        if path.isdir(path.join(base_dir, res)):
+            for (directory, _, files) in walk(path.join(base_dir, res)):
+                if files:
+                    package_data.append(
+                        path.join(directory.replace(base_dir, "").lstrip('/'),
+                                  '*'))
+    return package_data
+
+
+setup(
+    name=PYPI_NAME,
+    version="0.0.0",
+    package_dir={SKILL_PKG: ""},
+    package_data={SKILL_PKG: find_resource_files()},
+    packages=[SKILL_PKG],
+    include_package_data=True,
+    keywords='ovos skill plugin',
+    entry_points={'ovos.plugin.skill': PLUGIN_ENTRY_POINT}
+)

test/end2end/session/skill-ovos-fallback-unknownv1/__init__.py

@@ -0,0 +1,12 @@
+from ovos_workshop.skills.fallback import FallbackSkillV1
+from ovos_workshop.decorators import fallback_handler
+
+
+# explicitly use class with compat for older cores
+# this is usually auto detected, just done here for unittests
+class UnknownSkill(FallbackSkillV1):
+
+    @fallback_handler(priority=100)
+    def handle_fallback(self, message):
+        self.speak_dialog('unknown')
+        return True

test/end2end/session/skill-ovos-fallback-unknownv1/locale/en-us/unknown.dialog

@@ -0,0 +1,9 @@
+I'm sorry, I don't understand.
+I don't know what that means.
+I don't understand, but I'm learning new things everyday.
+Sorry, I didn't catch that.
+Sorry, I don't understand.
+I don't understand.
+I'm not sure I understood you.
+You might have to say that a different way.
+Please rephrase your request.

test/end2end/session/skill-ovos-fallback-unknownv1/setup.py

@@ -0,0 +1,39 @@
+#!/usr/bin/env python3
+from setuptools import setup
+from os import walk, path
+
+URL = "https://github.com/OpenVoiceOS/skill-ovos-fallback-unknownv1"
+SKILL_CLAZZ = "UnknownSkill"  # needs to match __init__.py class name
+PYPI_NAME = "ovos-skill-fallback-unknown-v1"  # pip install PYPI_NAME
+
+# below derived from github url to ensure standard skill_id
+SKILL_AUTHOR, SKILL_NAME = URL.split(".com/")[-1].split("/")
+SKILL_PKG = SKILL_NAME.lower().replace('-', '_')
+PLUGIN_ENTRY_POINT = f'{SKILL_NAME.lower()}.{SKILL_AUTHOR.lower()}={SKILL_PKG}:{SKILL_CLAZZ}'
+# skill_id=package_name:SkillClass
+
+
+def find_resource_files():
+    resource_base_dirs = ("locale", "ui", "vocab", "dialog", "regex", "skill")
+    base_dir = path.dirname(__file__)
+    package_data = ["*.json"]
+    for res in resource_base_dirs:
+        if path.isdir(path.join(base_dir, res)):
+            for (directory, _, files) in walk(path.join(base_dir, res)):
+                if files:
+                    package_data.append(
+                        path.join(directory.replace(base_dir, "").lstrip('/'),
+                                  '*'))
+    return package_data
+
+
+setup(
+    name=PYPI_NAME,
+    version="0.0.0",
+    package_dir={SKILL_PKG: ""},
+    package_data={SKILL_PKG: find_resource_files()},
+    packages=[SKILL_PKG],
+    include_package_data=True,
+    keywords='ovos skill plugin',
+    entry_points={'ovos.plugin.skill': PLUGIN_ENTRY_POINT}
+)

test/end2end/session/skill-ovos-hello-world/MANIFEST.in

@@ -0,0 +1,7 @@
+recursive-include dialog *
+recursive-include vocab *
+recursive-include locale *
+recursive-include res *
+recursive-include ui *
+include *.json
+include *.txt