improve_fallback_timeout

.github/workflows/coverage.yml

@@ -27,6 +27,7 @@ jobs:
           pip install ./test/unittests/common_query/ovos_tskill_fakewiki
           pip install ./test/end2end/skill-ovos-hello-world
           pip install ./test/end2end/skill-ovos-fallback-unknown
+          pip install ./test/end2end/skill-ovos-slow-fallback
           pip install ./test/end2end/skill-converse_test
           pip install ./test/end2end/skill-ovos-schedule
           pip install ./test/end2end/skill-new-stop

.github/workflows/unit_tests.yml

@@ -54,6 +54,7 @@ jobs:
           pip install ./test/unittests/common_query/ovos_tskill_fakewiki
           pip install ./test/end2end/skill-ovos-hello-world
           pip install ./test/end2end/skill-ovos-fallback-unknown
+          pip install ./test/end2end/skill-ovos-slow-fallback
           pip install ./test/end2end/skill-converse_test
           pip install ./test/end2end/skill-ovos-schedule
           pip install ./test/end2end/skill-new-stop

ovos_core/intent_services/fallback_service.py

@@ -139,13 +139,22 @@ def attempt_fallback(self, utterances, skill_id, lang, message):
                                     "utterance": utterances[0],  # backwards compat, we send all transcripts now
                                     "lang": lang})
             result = self.bus.wait_for_response(fb_msg,
-                                                f"ovos.skills.fallback.{skill_id}.response")
+                                                f"ovos.skills.fallback.{skill_id}.response",
+                                                timeout=self.fallback_config.get("max_skill_runtime", 10))
             if result and 'error' in result.data:
                 error_msg = result.data['error']
                 LOG.error(f"{skill_id}: {error_msg}")
                 return False
             elif result is not None:
                 return result.data.get('result', False)
+            else:
+                # abort any ongoing fallback
+                # if skill crashed or returns False, all good
+                # if it is just taking a long time, more than 1 fallback would end up answering
+                self.bus.emit(message.forward("ovos.skills.fallback.force_timeout",
+                                              {"skill_id": skill_id}))
+                LOG.warning(f"{skill_id} took too long to answer, "
+                            f'increasing "max_skill_runtime" in mycroft.conf might help alleviate this issue')
         return False
 
     def _fallback_range(self, utterances, lang, message, fb_range):
@@ -168,8 +177,9 @@ def _fallback_range(self, utterances, lang, message, fb_range):
 
         sess = SessionManager.get(message)
         # new style bus api
+        available_skills = self._collect_fallback_skills(message, fb_range)
         fallbacks = [(k, v) for k, v in self.registered_fallbacks.items()
-                     if k in self._collect_fallback_skills(message, fb_range)]
+                     if k in available_skills]
         sorted_handlers = sorted(fallbacks, key=operator.itemgetter(1))
         for skill_id, prio in sorted_handlers:
             if skill_id in sess.blacklisted_skills:

requirements/requirements.txt

@@ -12,7 +12,7 @@ ovos-plugin-manager<0.1.0, >=0.0.25
 ovos-config~=0.0,>=0.0.13a8
 ovos-lingua-franca>=0.4.7
 ovos-backend-client~=0.1.0
-ovos-workshop<0.1.0, >=0.0.16a30
+ovos-workshop<0.1.0, >=0.0.16a35
 # provides plugins and classic machine learning framework
 ovos-classifiers<0.1.0, >=0.0.0a53
 

test/end2end/session/test_fallback.py

@@ -301,3 +301,85 @@ def wait_for_n_messages(n):
         self.assertEqual(len(expected_messages), len(messages))
         for idx, m in enumerate(messages):
             self.assertEqual(m.msg_type, expected_messages[idx])
+
+
+class TestFallbackTimeout(TestCase):
+
+    def setUp(self):
+        self.skill_id = "skill-ovos-fallback-unknown.openvoiceos"
+        self.skill_id2 = "ovos-skill-slow-fallback.openvoiceos"
+        self.core = get_minicroft([self.skill_id, self.skill_id2])
+
+    def tearDown(self) -> None:
+        self.core.stop()
+
+    def test_fallback(self):
+        SessionManager.sessions = {}
+        SessionManager.default_session = SessionManager.sessions["default"] = Session("default")
+        SessionManager.default_session.lang = "en-us"
+        SessionManager.default_session.pipeline = [
+            "fallback_medium",
+            "fallback_low"
+        ]
+        messages = []
+
+        def new_msg(msg):
+            nonlocal messages
+            m = Message.deserialize(msg)
+            if m.msg_type in ["ovos.skills.settings_changed"]:
+                return  # skip these, only happen in 1st run
+            messages.append(m)
+            print(len(messages), msg)
+
+        def wait_for_n_messages(n):
+            nonlocal messages
+            t = time.time()
+            while len(messages) < n:
+                sleep(0.1)
+                if time.time() - t > 10:
+                    raise RuntimeError("did not get the number of expected messages under 10 seconds")
+
+        self.core.bus.on("message", new_msg)
+
+        utt = Message("recognizer_loop:utterance",
+                      {"utterances": ["invalid"]},
+                      {"session": SessionManager.default_session.serialize()})
+        self.core.bus.emit(utt)
+
+        # confirm all expected messages are sent
+        expected_messages = [
+            "recognizer_loop:utterance",
+            # FallbackV2
+            "ovos.skills.fallback.ping",
+            "ovos.skills.fallback.pong",
+            "ovos.skills.fallback.pong",
+
+            # slow skill executing
+            f"ovos.skills.fallback.{self.skill_id2}.request",
+            f"ovos.skills.fallback.{self.skill_id2}.start",
+            "ovos.skills.fallback.force_timeout",  # timeout from core
+            f"ovos.skills.fallback.{self.skill_id2}.response",
+            f"ovos.skills.fallback.{self.skill_id2}.killed",  # killable_event decorator response
+
+            # skill executing
+            f"ovos.skills.fallback.{self.skill_id}.request",
+            f"ovos.skills.fallback.{self.skill_id}.start",
+            "enclosure.active_skill",
+            "speak",
+
+            # activated only after skill return True
+            "intent.service.skills.activate",
+            "intent.service.skills.activated",
+            f"{self.skill_id}.activate",
+            "ovos.session.update_default",
+
+            f"ovos.skills.fallback.{self.skill_id}.response",
+            "ovos.utterance.handled",  # handle_utterance returned (intent service)
+            "ovos.session.update_default"
+        ]
+        wait_for_n_messages(len(expected_messages))
+
+        self.assertEqual(len(expected_messages), len(messages))
+
+        for idx, m in enumerate(messages):
+            self.assertEqual(m.msg_type, expected_messages[idx])

test/end2end/skill-ovos-slow-fallback/__init__.py

@@ -0,0 +1,14 @@
+import time
+
+from ovos_workshop.decorators import fallback_handler
+from ovos_workshop.skills.fallback import FallbackSkill
+
+
+class SlowFallbackSkill(FallbackSkill):
+
+    @fallback_handler(priority=20)
+    def handle_fallback(self, message):
+        while True:  # busy skill
+            time.sleep(0.1)
+        self.speak("SLOW")
+        return True

test/end2end/skill-ovos-slow-fallback/setup.py

@@ -0,0 +1,42 @@
+#!/usr/bin/env python3
+from os import walk, path
+
+from setuptools import setup
+
+URL = "https://github.com/OpenVoiceOS/ovos-skill-slow-fallback"
+SKILL_CLAZZ = "SlowFallbackSkill"  # needs to match __init__.py class name
+PYPI_NAME = "ovos-skill-slow-fallback"  # pip install PYPI_NAME
+
+# below derived from github url to ensure standard skill_id
+SKILL_AUTHOR, SKILL_NAME = URL.split(".com/")[-1].split("/")
+SKILL_PKG = SKILL_NAME.lower().replace('-', '_')
+PLUGIN_ENTRY_POINT = f'{SKILL_NAME.lower()}.{SKILL_AUTHOR.lower()}={SKILL_PKG}:{SKILL_CLAZZ}'
+
+
+# skill_id=package_name:SkillClass
+
+
+def find_resource_files():
+    resource_base_dirs = ("locale", "ui", "vocab", "dialog", "regex", "skill")
+    base_dir = path.dirname(__file__)
+    package_data = ["*.json"]
+    for res in resource_base_dirs:
+        if path.isdir(path.join(base_dir, res)):
+            for (directory, _, files) in walk(path.join(base_dir, res)):
+                if files:
+                    package_data.append(
+                        path.join(directory.replace(base_dir, "").lstrip('/'),
+                                  '*'))
+    return package_data
+
+
+setup(
+    name=PYPI_NAME,
+    version="0.0.0",
+    package_dir={SKILL_PKG: ""},
+    package_data={SKILL_PKG: find_resource_files()},
+    packages=[SKILL_PKG],
+    include_package_data=True,
+    keywords='ovos skill plugin',
+    entry_points={'ovos.plugin.skill': PLUGIN_ENTRY_POINT}
+)