diff --git a/.github/workflows/coverage.yml b/.github/workflows/coverage.yml index 2a43a2e23be..79ad2def72f 100644 --- a/.github/workflows/coverage.yml +++ b/.github/workflows/coverage.yml @@ -27,6 +27,7 @@ jobs: pip install ./test/unittests/common_query/ovos_tskill_fakewiki pip install ./test/end2end/skill-ovos-hello-world pip install ./test/end2end/skill-ovos-fallback-unknown + pip install ./test/end2end/skill-ovos-slow-fallback pip install ./test/end2end/skill-converse_test pip install ./test/end2end/skill-ovos-schedule pip install ./test/end2end/skill-new-stop diff --git a/.github/workflows/unit_tests.yml b/.github/workflows/unit_tests.yml index 4fc749fdc1a..fb9cd5eb853 100644 --- a/.github/workflows/unit_tests.yml +++ b/.github/workflows/unit_tests.yml @@ -54,6 +54,7 @@ jobs: pip install ./test/unittests/common_query/ovos_tskill_fakewiki pip install ./test/end2end/skill-ovos-hello-world pip install ./test/end2end/skill-ovos-fallback-unknown + pip install ./test/end2end/skill-ovos-slow-fallback pip install ./test/end2end/skill-converse_test pip install ./test/end2end/skill-ovos-schedule pip install ./test/end2end/skill-new-stop diff --git a/ovos_core/intent_services/fallback_service.py b/ovos_core/intent_services/fallback_service.py index 802ab9fd287..6d8e413a335 100644 --- a/ovos_core/intent_services/fallback_service.py +++ b/ovos_core/intent_services/fallback_service.py @@ -139,13 +139,22 @@ def attempt_fallback(self, utterances, skill_id, lang, message): "utterance": utterances[0], # backwards compat, we send all transcripts now "lang": lang}) result = self.bus.wait_for_response(fb_msg, - f"ovos.skills.fallback.{skill_id}.response") + f"ovos.skills.fallback.{skill_id}.response", + timeout=self.fallback_config.get("max_skill_runtime", 10)) if result and 'error' in result.data: error_msg = result.data['error'] LOG.error(f"{skill_id}: {error_msg}") return False elif result is not None: return result.data.get('result', False) + else: + # abort any ongoing fallback + # if skill crashed or returns False, all good + # if it is just taking a long time, more than 1 fallback would end up answering + self.bus.emit(message.forward("ovos.skills.fallback.force_timeout", + {"skill_id": skill_id})) + LOG.warning(f"{skill_id} took too long to answer, " + f'increasing "max_skill_runtime" in mycroft.conf might help alleviate this issue') return False def _fallback_range(self, utterances, lang, message, fb_range): @@ -168,8 +177,9 @@ def _fallback_range(self, utterances, lang, message, fb_range): sess = SessionManager.get(message) # new style bus api + available_skills = self._collect_fallback_skills(message, fb_range) fallbacks = [(k, v) for k, v in self.registered_fallbacks.items() - if k in self._collect_fallback_skills(message, fb_range)] + if k in available_skills] sorted_handlers = sorted(fallbacks, key=operator.itemgetter(1)) for skill_id, prio in sorted_handlers: if skill_id in sess.blacklisted_skills: diff --git a/requirements/requirements.txt b/requirements/requirements.txt index 9e9dec05898..b73a9f5f2f0 100644 --- a/requirements/requirements.txt +++ b/requirements/requirements.txt @@ -12,7 +12,7 @@ ovos-plugin-manager<0.1.0, >=0.0.25 ovos-config~=0.0,>=0.0.13a8 ovos-lingua-franca>=0.4.7 ovos-backend-client~=0.1.0 -ovos-workshop<0.1.0, >=0.0.16a30 +ovos-workshop<0.1.0, >=0.0.16a35 # provides plugins and classic machine learning framework ovos-classifiers<0.1.0, >=0.0.0a53 diff --git a/test/end2end/session/test_fallback.py b/test/end2end/session/test_fallback.py index e6a8d0ea1f0..edb0030240a 100644 --- a/test/end2end/session/test_fallback.py +++ b/test/end2end/session/test_fallback.py @@ -301,3 +301,85 @@ def wait_for_n_messages(n): self.assertEqual(len(expected_messages), len(messages)) for idx, m in enumerate(messages): self.assertEqual(m.msg_type, expected_messages[idx]) + + +class TestFallbackTimeout(TestCase): + + def setUp(self): + self.skill_id = "skill-ovos-fallback-unknown.openvoiceos" + self.skill_id2 = "ovos-skill-slow-fallback.openvoiceos" + self.core = get_minicroft([self.skill_id, self.skill_id2]) + + def tearDown(self) -> None: + self.core.stop() + + def test_fallback(self): + SessionManager.sessions = {} + SessionManager.default_session = SessionManager.sessions["default"] = Session("default") + SessionManager.default_session.lang = "en-us" + SessionManager.default_session.pipeline = [ + "fallback_medium", + "fallback_low" + ] + messages = [] + + def new_msg(msg): + nonlocal messages + m = Message.deserialize(msg) + if m.msg_type in ["ovos.skills.settings_changed"]: + return # skip these, only happen in 1st run + messages.append(m) + print(len(messages), msg) + + def wait_for_n_messages(n): + nonlocal messages + t = time.time() + while len(messages) < n: + sleep(0.1) + if time.time() - t > 10: + raise RuntimeError("did not get the number of expected messages under 10 seconds") + + self.core.bus.on("message", new_msg) + + utt = Message("recognizer_loop:utterance", + {"utterances": ["invalid"]}, + {"session": SessionManager.default_session.serialize()}) + self.core.bus.emit(utt) + + # confirm all expected messages are sent + expected_messages = [ + "recognizer_loop:utterance", + # FallbackV2 + "ovos.skills.fallback.ping", + "ovos.skills.fallback.pong", + "ovos.skills.fallback.pong", + + # slow skill executing + f"ovos.skills.fallback.{self.skill_id2}.request", + f"ovos.skills.fallback.{self.skill_id2}.start", + "ovos.skills.fallback.force_timeout", # timeout from core + f"ovos.skills.fallback.{self.skill_id2}.response", + f"ovos.skills.fallback.{self.skill_id2}.killed", # killable_event decorator response + + # skill executing + f"ovos.skills.fallback.{self.skill_id}.request", + f"ovos.skills.fallback.{self.skill_id}.start", + "enclosure.active_skill", + "speak", + + # activated only after skill return True + "intent.service.skills.activate", + "intent.service.skills.activated", + f"{self.skill_id}.activate", + "ovos.session.update_default", + + f"ovos.skills.fallback.{self.skill_id}.response", + "ovos.utterance.handled", # handle_utterance returned (intent service) + "ovos.session.update_default" + ] + wait_for_n_messages(len(expected_messages)) + + self.assertEqual(len(expected_messages), len(messages)) + + for idx, m in enumerate(messages): + self.assertEqual(m.msg_type, expected_messages[idx]) diff --git a/test/end2end/skill-ovos-slow-fallback/__init__.py b/test/end2end/skill-ovos-slow-fallback/__init__.py new file mode 100644 index 00000000000..1d4ddc452e6 --- /dev/null +++ b/test/end2end/skill-ovos-slow-fallback/__init__.py @@ -0,0 +1,14 @@ +import time + +from ovos_workshop.decorators import fallback_handler +from ovos_workshop.skills.fallback import FallbackSkill + + +class SlowFallbackSkill(FallbackSkill): + + @fallback_handler(priority=20) + def handle_fallback(self, message): + while True: # busy skill + time.sleep(0.1) + self.speak("SLOW") + return True diff --git a/test/end2end/skill-ovos-slow-fallback/setup.py b/test/end2end/skill-ovos-slow-fallback/setup.py new file mode 100755 index 00000000000..323eecd7a40 --- /dev/null +++ b/test/end2end/skill-ovos-slow-fallback/setup.py @@ -0,0 +1,42 @@ +#!/usr/bin/env python3 +from os import walk, path + +from setuptools import setup + +URL = "https://github.com/OpenVoiceOS/ovos-skill-slow-fallback" +SKILL_CLAZZ = "SlowFallbackSkill" # needs to match __init__.py class name +PYPI_NAME = "ovos-skill-slow-fallback" # pip install PYPI_NAME + +# below derived from github url to ensure standard skill_id +SKILL_AUTHOR, SKILL_NAME = URL.split(".com/")[-1].split("/") +SKILL_PKG = SKILL_NAME.lower().replace('-', '_') +PLUGIN_ENTRY_POINT = f'{SKILL_NAME.lower()}.{SKILL_AUTHOR.lower()}={SKILL_PKG}:{SKILL_CLAZZ}' + + +# skill_id=package_name:SkillClass + + +def find_resource_files(): + resource_base_dirs = ("locale", "ui", "vocab", "dialog", "regex", "skill") + base_dir = path.dirname(__file__) + package_data = ["*.json"] + for res in resource_base_dirs: + if path.isdir(path.join(base_dir, res)): + for (directory, _, files) in walk(path.join(base_dir, res)): + if files: + package_data.append( + path.join(directory.replace(base_dir, "").lstrip('/'), + '*')) + return package_data + + +setup( + name=PYPI_NAME, + version="0.0.0", + package_dir={SKILL_PKG: ""}, + package_data={SKILL_PKG: find_resource_files()}, + packages=[SKILL_PKG], + include_package_data=True, + keywords='ovos skill plugin', + entry_points={'ovos.plugin.skill': PLUGIN_ENTRY_POINT} +)