unittests/fallback_service

.github/workflows/coverage.yml

@@ -29,6 +29,7 @@ jobs:
         pip install -r requirements/tests.txt
         pip install ./test/unittests/common_query/ovos_tskill_fakewiki
         pip install ./test/end2end/session/skill-ovos-hello-world
+        pip install ./test/end2end/session/skill-ovos-fallback-unknown
     - name: Generate coverage report
       run: |
         pytest --cov=ovos_core --cov-report xml test/unittests

.github/workflows/unit_tests.yml

@@ -56,6 +56,7 @@ jobs:
           pip install -r requirements/tests.txt
           pip install ./test/unittests/common_query/ovos_tskill_fakewiki
           pip install ./test/end2end/session/skill-ovos-hello-world
+          pip install ./test/end2end/session/skill-ovos-fallback-unknown
       - name: Run unittests
         run: |
           pytest --cov=ovos_core --cov-report xml test/unittests

ovos_core/intent_services/fallback_service.py

@@ -165,7 +165,7 @@ def _fallback_range(self, utterances, lang, message, fb_range):
         for skill_id, prio in sorted_handlers:
             result = self.attempt_fallback(utterances, skill_id, lang, message)
             if result:
-                return ovos_core.intent_services.IntentMatch('Fallback', None, {}, None, utterances[0])
+                return ovos_core.intent_services.IntentMatch('Fallback', None, {}, skill_id, utterances[0])
 
         # old style deprecated fallback skill singleton class
         LOG.debug("checking for FallbackSkillsV1")

requirements/requirements.txt

@@ -12,7 +12,7 @@ ovos-plugin-manager<0.1.0, >=0.0.24a9
 ovos-config~=0.0,>=0.0.11a13
 ovos-lingua-franca>=0.4.7
 ovos-backend-client>=0.1.0a12
-ovos-workshop<0.1.0, >=0.0.13a5
+ovos-workshop<0.1.0, >=0.0.13a6
 
 # provides plugins and classic machine learning framework
 ovos-classifiers<0.1.0, >=0.0.0a37

test/end2end/session/minicroft.py

@@ -0,0 +1,66 @@
+from time import sleep
+from ovos_bus_client.session import SessionManager, Session
+from ovos_core.intent_services import IntentService
+from ovos_core.skill_manager import SkillManager
+from ovos_plugin_manager.skills import find_skill_plugins
+from ovos_utils.log import LOG
+from ovos_utils.messagebus import FakeBus
+from ovos_utils.process_utils import ProcessState
+from ovos_workshop.skills.fallback import FallbackSkill
+
+
+class MiniCroft(SkillManager):
+    def __init__(self, skill_ids, *args, **kwargs):
+        bus = FakeBus()
+        super().__init__(bus, *args, **kwargs)
+        self.skill_ids = skill_ids
+        self.intent_service = self._register_intent_services()
+
+    def _register_intent_services(self):
+        """Start up the all intent services and connect them as needed.
+
+        Args:
+            bus: messagebus client to register the services on
+        """
+        service = IntentService(self.bus)
+        # Register handler to trigger fallback system
+        self.bus.on(
+            'mycroft.skills.fallback',
+            FallbackSkill.make_intent_failure_handler(self.bus)
+        )
+        return service
+
+    def load_plugin_skills(self):
+        LOG.info("loading skill plugins")
+        plugins = find_skill_plugins()
+        for skill_id, plug in plugins.items():
+            LOG.debug(skill_id)
+            if skill_id not in self.skill_ids:
+                continue
+            if skill_id not in self.plugin_skills:
+                self._load_plugin_skill(skill_id, plug)
+
+    def run(self):
+        """Load skills and update periodically from disk and internet."""
+        self.status.set_alive()
+
+        self.load_plugin_skills()
+
+        self.status.set_ready()
+
+        LOG.info("Skills all loaded!")
+
+    def stop(self):
+        super().stop()
+        SessionManager.bus = None
+        SessionManager.sessions = {}
+        SessionManager.default_session = SessionManager.sessions["default"] = Session("default")
+
+
+def get_minicroft(skill_id):
+    croft1 = MiniCroft([skill_id])
+    croft1.start()
+    while croft1.status.state != ProcessState.READY:
+        sleep(0.2)
+    return croft1
+

test/end2end/session/skill-ovos-fallback-unknown/__init__.py

@@ -0,0 +1,10 @@
+from ovos_workshop.skills.fallback import FallbackSkill
+from ovos_workshop.decorators import fallback_handler
+
+
+class UnknownSkill(FallbackSkill):
+
+    @fallback_handler(priority=100)
+    def handle_fallback(self, message):
+        self.speak_dialog('unknown')
+        return True

test/end2end/session/skill-ovos-fallback-unknown/locale/en-us/unknown.dialog

@@ -0,0 +1,9 @@
+I'm sorry, I don't understand.
+I don't know what that means.
+I don't understand, but I'm learning new things everyday.
+Sorry, I didn't catch that.
+Sorry, I don't understand.
+I don't understand.
+I'm not sure I understood you.
+You might have to say that a different way.
+Please rephrase your request.

test/end2end/session/skill-ovos-fallback-unknown/setup.py

@@ -0,0 +1,39 @@
+#!/usr/bin/env python3
+from setuptools import setup
+from os import walk, path
+
+URL = "https://github.com/OpenVoiceOS/skill-ovos-fallback-unknown"
+SKILL_CLAZZ = "UnknownSkill"  # needs to match __init__.py class name
+PYPI_NAME = "ovos-skill-fallback-unknown"  # pip install PYPI_NAME
+
+# below derived from github url to ensure standard skill_id
+SKILL_AUTHOR, SKILL_NAME = URL.split(".com/")[-1].split("/")
+SKILL_PKG = SKILL_NAME.lower().replace('-', '_')
+PLUGIN_ENTRY_POINT = f'{SKILL_NAME.lower()}.{SKILL_AUTHOR.lower()}={SKILL_PKG}:{SKILL_CLAZZ}'
+# skill_id=package_name:SkillClass
+
+
+def find_resource_files():
+    resource_base_dirs = ("locale", "ui", "vocab", "dialog", "regex", "skill")
+    base_dir = path.dirname(__file__)
+    package_data = ["*.json"]
+    for res in resource_base_dirs:
+        if path.isdir(path.join(base_dir, res)):
+            for (directory, _, files) in walk(path.join(base_dir, res)):
+                if files:
+                    package_data.append(
+                        path.join(directory.replace(base_dir, "").lstrip('/'),
+                                  '*'))
+    return package_data
+
+
+setup(
+    name=PYPI_NAME,
+    version="0.0.0",
+    package_dir={SKILL_PKG: ""},
+    package_data={SKILL_PKG: find_resource_files()},
+    packages=[SKILL_PKG],
+    include_package_data=True,
+    keywords='ovos skill plugin',
+    entry_points={'ovos.plugin.skill': PLUGIN_ENTRY_POINT}
+)

test/end2end/session/test_fallback.py

@@ -0,0 +1,225 @@
+from time import sleep
+from unittest import TestCase
+
+from ovos_bus_client.message import Message
+from ovos_bus_client.session import SessionManager, Session
+from .minicroft import get_minicroft
+
+
+class TestFallback(TestCase):
+
+    def setUp(self):
+        self.skill_id = "skill-ovos-fallback-unknown.openvoiceos"
+        self.core = get_minicroft(self.skill_id)
+
+    def test_fallback(self):
+        SessionManager.sessions = {}
+        SessionManager.default_session = SessionManager.sessions["default"] = Session("default")
+        SessionManager.default_session.lang = "en-us"
+        messages = []
+
+        def new_msg(msg):
+            nonlocal messages
+            m = Message.deserialize(msg)
+            if m.msg_type in ["ovos.skills.settings_changed"]:
+                return  # skip these, only happen in 1st run
+            messages.append(m)
+            print(len(messages), msg)
+
+        def wait_for_n_messages(n):
+            nonlocal messages
+            while len(messages) < n:
+                sleep(0.1)
+
+        self.core.bus.on("message", new_msg)
+
+        utt = Message("recognizer_loop:utterance",
+                      {"utterances": ["invalid"]},
+                      {"session": SessionManager.default_session.serialize(),  # explicit default sess
+                       "x": "xx"})
+        self.core.bus.emit(utt)
+
+        # confirm all expected messages are sent
+        expected_messages = [
+            "recognizer_loop:utterance",
+            # Converse
+            "skill.converse.ping",
+            "skill.converse.pong",
+            # FallbackV2
+            "ovos.skills.fallback.ping",
+            "ovos.skills.fallback.pong",
+            # skill executing
+            f"ovos.skills.fallback.{self.skill_id}.request",
+            f"ovos.skills.fallback.{self.skill_id}.start",
+            "enclosure.active_skill",
+            "speak",
+            f"ovos.skills.fallback.{self.skill_id}.response",
+            # intent service post fallback
+            "intent.service.skills.activated",
+            f"{self.skill_id}.activate",
+            "ovos.session.update_default"
+        ]
+        wait_for_n_messages(len(expected_messages))
+
+        self.assertEqual(len(expected_messages), len(messages))
+
+        mtypes = [m.msg_type for m in messages]
+        for m in expected_messages:
+            self.assertTrue(m in mtypes)
+
+        # verify that contexts are kept around
+        for m in messages:
+            self.assertEqual(m.context["session"]["session_id"], "default")
+            self.assertEqual(m.context["x"], "xx")
+        # verify active skills is empty until "intent.service.skills.activated"
+        for m in messages[:10]:
+            self.assertEqual(m.context["session"]["session_id"], "default")
+            self.assertEqual(m.context["session"]["active_skills"], [])
+
+        # verify converse ping/pong answer from skill
+        self.assertEqual(messages[1].msg_type, "skill.converse.ping")
+        self.assertEqual(messages[2].msg_type, "skill.converse.pong")
+        self.assertEqual(messages[2].data["skill_id"], self.skill_id)
+        self.assertEqual(messages[2].context["skill_id"], self.skill_id)
+        self.assertFalse(messages[2].data["can_handle"])
+
+        # verify fallback ping/pong answer from skill
+        self.assertEqual(messages[3].msg_type, "ovos.skills.fallback.ping")
+        self.assertEqual(messages[4].msg_type, "ovos.skills.fallback.pong")
+        self.assertEqual(messages[4].data["skill_id"], self.skill_id)
+        self.assertEqual(messages[4].context["skill_id"], self.skill_id)
+        self.assertTrue(messages[4].data["can_handle"])
+
+        # verify skill executes
+        self.assertEqual(messages[5].msg_type, f"ovos.skills.fallback.{self.skill_id}.request")
+        self.assertEqual(messages[5].data["skill_id"], self.skill_id)
+        self.assertEqual(messages[6].msg_type, f"ovos.skills.fallback.{self.skill_id}.start")
+        self.assertEqual(messages[7].msg_type, "enclosure.active_skill")
+        self.assertEqual(messages[7].data["skill_id"], self.skill_id)
+        self.assertEqual(messages[8].msg_type, "speak")
+        self.assertEqual(messages[8].data["meta"]["dialog"], "unknown")
+        self.assertEqual(messages[8].data["meta"]["skill"], self.skill_id)
+        self.assertEqual(messages[9].msg_type, f"ovos.skills.fallback.{self.skill_id}.response")
+        self.assertTrue(messages[9].data["result"])
+        self.assertEqual(messages[9].data["fallback_handler"], "UnknownSkill.handle_fallback")
+
+        # verify skill is activated
+        self.assertEqual(messages[10].msg_type, "intent.service.skills.activated")
+        self.assertEqual(messages[10].data["skill_id"], self.skill_id)
+        self.assertEqual(messages[11].msg_type, f"{self.skill_id}.activate")
+
+        # verify default session is now updated
+        self.assertEqual(messages[12].msg_type, "ovos.session.update_default")
+        self.assertEqual(messages[12].data["session_data"]["session_id"], "default")
+
+        # test second message with no session resumes default active skills
+        messages = []
+        utt = Message("recognizer_loop:utterance",
+                      {"utterances": ["invalid"]})
+        self.core.bus.emit(utt)
+        wait_for_n_messages(len(expected_messages))
+        self.assertEqual(len(expected_messages), len(messages))
+
+        # verify that contexts are kept around
+        for m in messages[1:]:
+            self.assertEqual(m.context["session"]["session_id"], "default")
+            self.assertEqual(m.context["session"]["active_skills"][0][0], self.skill_id)
+
+    def test_fallback_with_session(self):
+        SessionManager.sessions = {}
+        SessionManager.default_session = SessionManager.sessions["default"] = Session("default")
+        SessionManager.default_session.lang = "en-us"
+        messages = []
+
+        sess = Session()
+
+        def new_msg(msg):
+            nonlocal messages
+            m = Message.deserialize(msg)
+            if m.msg_type in ["ovos.skills.settings_changed"]:
+                return  # skip these, only happen in 1st run
+            messages.append(m)
+            print(len(messages), msg)
+
+        def wait_for_n_messages(n):
+            nonlocal messages
+            while len(messages) < n:
+                sleep(0.1)
+
+        self.core.bus.on("message", new_msg)
+
+        utt = Message("recognizer_loop:utterance",
+                      {"utterances": ["invalid"]},
+                      {"session": sess.serialize(),  # explicit sess
+                       "x": "xx"})
+        self.core.bus.emit(utt)
+
+        # confirm all expected messages are sent
+        expected_messages = [
+            "recognizer_loop:utterance",
+            # Converse
+            "skill.converse.ping",
+            "skill.converse.pong",
+            # FallbackV2
+            "ovos.skills.fallback.ping",
+            "ovos.skills.fallback.pong",
+            # skill executing - TODO "mycroft.skill.handler.start" +  "mycroft.skill.handler.complete" should be added
+            f"ovos.skills.fallback.{self.skill_id}.request",
+            f"ovos.skills.fallback.{self.skill_id}.start",
+            "enclosure.active_skill",
+            "speak",
+            f"ovos.skills.fallback.{self.skill_id}.response",
+            # intent service post fallback
+            "intent.service.skills.activated",
+            f"{self.skill_id}.activate"
+        ]
+        wait_for_n_messages(len(expected_messages))
+
+        self.assertEqual(len(expected_messages), len(messages))
+
+        mtypes = [m.msg_type for m in messages]
+        for m in expected_messages:
+            self.assertTrue(m in mtypes)
+
+        # verify that contexts are kept around
+        for m in messages:
+            self.assertEqual(m.context["session"]["session_id"], sess.session_id)
+            self.assertEqual(m.context["x"], "xx")
+
+        # verify converse ping/pong answer from skill
+        self.assertEqual(messages[1].msg_type, "skill.converse.ping")
+        self.assertEqual(messages[2].msg_type, "skill.converse.pong")
+        self.assertEqual(messages[2].data["skill_id"], self.skill_id)
+        self.assertEqual(messages[2].context["skill_id"], self.skill_id)
+        self.assertFalse(messages[2].data["can_handle"])
+
+        # verify fallback ping/pong answer from skill
+        self.assertEqual(messages[3].msg_type, "ovos.skills.fallback.ping")
+        self.assertEqual(messages[4].msg_type, "ovos.skills.fallback.pong")
+        self.assertEqual(messages[4].data["skill_id"], self.skill_id)
+        self.assertEqual(messages[4].context["skill_id"], self.skill_id)
+        self.assertTrue(messages[4].data["can_handle"])
+
+        # verify skill executes
+        self.assertEqual(messages[5].msg_type, f"ovos.skills.fallback.{self.skill_id}.request")
+        self.assertEqual(messages[5].data["skill_id"], self.skill_id)
+        self.assertEqual(messages[6].msg_type, f"ovos.skills.fallback.{self.skill_id}.start")
+        self.assertEqual(messages[7].msg_type, "enclosure.active_skill")
+        self.assertEqual(messages[7].data["skill_id"], self.skill_id)
+        self.assertEqual(messages[8].msg_type, "speak")
+        self.assertEqual(messages[8].data["meta"]["dialog"], "unknown")
+        self.assertEqual(messages[8].data["meta"]["skill"], self.skill_id)
+        self.assertEqual(messages[9].msg_type, f"ovos.skills.fallback.{self.skill_id}.response")
+        self.assertTrue(messages[9].data["result"])
+        self.assertEqual(messages[9].data["fallback_handler"], "UnknownSkill.handle_fallback")
+
+        # verify skill is activated
+        self.assertEqual(messages[10].msg_type, "intent.service.skills.activated")
+        self.assertEqual(messages[10].data["skill_id"], self.skill_id)
+        self.assertEqual(messages[11].msg_type, f"{self.skill_id}.activate")
+
+        # test that active skills list has been updated
+        sess = SessionManager.sessions[sess.session_id]
+        self.assertEqual(sess.active_skills[0][0], self.skill_id)
+        # test that default session remains unchanged
+        self.assertEqual(SessionManager.default_session.active_skills, [])