From 9a3a92d897f9bc03c9c2cc4946dcfabb8af7694b Mon Sep 17 00:00:00 2001 From: JarbasAI <33701864+JarbasAl@users.noreply.github.com> Date: Mon, 2 Oct 2023 21:17:50 +0100 Subject: [PATCH] unittests/get_response (#361) --- requirements/requirements.txt | 2 +- .../session/skill-converse_test/__init__.py | 2 +- test/end2end/session/test_get_response.py | 249 ++++++++++++++++++ test/integrationtests/test_workshop.py | 2 +- .../skills/test_mycroft_skill_get_response.py | 12 +- 5 files changed, 259 insertions(+), 8 deletions(-) create mode 100644 test/end2end/session/test_get_response.py diff --git a/requirements/requirements.txt b/requirements/requirements.txt index 84b3f5e6db3..b37687542e8 100644 --- a/requirements/requirements.txt +++ b/requirements/requirements.txt @@ -12,7 +12,7 @@ ovos-plugin-manager<0.1.0, >=0.0.24a9 ovos-config~=0.0,>=0.0.11a13 ovos-lingua-franca>=0.4.7 ovos-backend-client>=0.1.0a12 -ovos-workshop<0.1.0, >=0.0.13a7 +ovos-workshop<0.1.0, >=0.0.13a8 # provides plugins and classic machine learning framework ovos-classifiers<0.1.0, >=0.0.0a37 diff --git a/test/end2end/session/skill-converse_test/__init__.py b/test/end2end/session/skill-converse_test/__init__.py index 93cd6291988..4de290d7704 100644 --- a/test/end2end/session/skill-converse_test/__init__.py +++ b/test/end2end/session/skill-converse_test/__init__.py @@ -39,7 +39,7 @@ def handle_intent_aborted(self): @intent_file_handler("test_get_response.intent") def handle_test_get_response(self, message): ans = self.get_response("get") - self.speak(ans) + self.speak(ans or "ERROR") @killable_intent(callback=handle_intent_aborted) @intent_file_handler("test.intent") diff --git a/test/end2end/session/test_get_response.py b/test/end2end/session/test_get_response.py new file mode 100644 index 00000000000..8ca92ffcdcb --- /dev/null +++ b/test/end2end/session/test_get_response.py @@ -0,0 +1,249 @@ +import time +from time import sleep +from unittest import TestCase + +from ovos_bus_client.message import Message +from ovos_bus_client.session import SessionManager, Session +from .minicroft import get_minicroft + + +class TestSessions(TestCase): + + def setUp(self): + self.skill_id = "ovos-tskill-abort.openvoiceos" + self.other_skill_id = "skill-ovos-hello-world.openvoiceos" + self.core = get_minicroft([self.skill_id, self.other_skill_id]) + + def test_no_response(self): + SessionManager.sessions = {} + SessionManager.default_session = SessionManager.sessions["default"] = Session("default") + SessionManager.default_session.lang = "en-us" + + messages = [] + + def new_msg(msg): + nonlocal messages + m = Message.deserialize(msg) + if m.msg_type in ["ovos.skills.settings_changed"]: + return # skip these, only happen in 1st run + messages.append(m) + print(len(messages), msg) + + def wait_for_n_messages(n): + nonlocal messages + t = time.time() + while len(messages) < n: + sleep(0.1) + if time.time() - t > 10: + raise RuntimeError("did not get the number of expected messages under 10 seconds") + + self.core.bus.on("message", new_msg) + + # trigger get_response + utt = Message("recognizer_loop:utterance", + {"utterances": ["test get response"]}) + self.core.bus.emit(utt) + + # confirm all expected messages are sent + expected_messages = [ + "recognizer_loop:utterance", # no session + "skill.converse.ping", # default session injected + "skill.converse.pong", # test skill + "skill.converse.pong", # hello world skill + + # skill selected + "intent.service.skills.activated", + f"{self.skill_id}.activate", + f"{self.skill_id}:test_get_response.intent", + + # skill executing + "mycroft.skill.handler.start", + "enclosure.active_skill", + "speak", # 'mycroft.mic.listen' if no dialog passed to get_response + "skill.converse.get_response.enable", # start of get_response + "ovos.session.update_default", # sync get_response status + # "recognizer_loop:utterance" would be here if user answered + "skill.converse.get_response.disable", # end of get_response + "ovos.session.update_default", # sync get_response status + "enclosure.active_skill", # from speak inside intent + "speak", # speak "ERROR" inside intent + "mycroft.skill.handler.complete", # original intent finished executing + + # session updated at end of intent pipeline + "ovos.session.update_default" + ] + wait_for_n_messages(len(expected_messages)) + + self.assertEqual(len(expected_messages), len(messages)) + + mtypes = [m.msg_type for m in messages] + for m in expected_messages: + self.assertTrue(m in mtypes) + + # verify that "session" is injected + # (missing in utterance message) and kept in all messages + for m in messages[1:]: + print(m.msg_type, m.context["session"]["session_id"]) + self.assertEqual(m.context["session"]["session_id"], "default") + + # verify that "lang" is injected by converse.ping + # (missing in utterance message) and kept in all messages + self.assertEqual(messages[1].msg_type, "skill.converse.ping") + for m in messages[1:]: + self.assertEqual(m.context["lang"], "en-us") + + # verify "pong" answer from both skills + self.assertEqual(messages[2].msg_type, "skill.converse.pong") + self.assertEqual(messages[3].msg_type, "skill.converse.pong") + self.assertEqual(messages[2].data["skill_id"], messages[2].context["skill_id"]) + self.assertEqual(messages[3].data["skill_id"], messages[3].context["skill_id"]) + # assert it reports converse method has been implemented by skill + if messages[2].data["skill_id"] == self.skill_id: # we dont know order of pong responses + self.assertTrue(messages[2].data["can_handle"]) + self.assertFalse(messages[3].data["can_handle"]) + if messages[3].data["skill_id"] == self.skill_id: # we dont know order of pong responses + self.assertTrue(messages[3].data["can_handle"]) + self.assertFalse(messages[2].data["can_handle"]) + + # verify skill is activated by intent service (intent pipeline matched) + self.assertEqual(messages[4].msg_type, "intent.service.skills.activated") + self.assertEqual(messages[4].data["skill_id"], self.skill_id) + self.assertEqual(messages[5].msg_type, f"{self.skill_id}.activate") + + # verify intent triggers + self.assertEqual(messages[6].msg_type, f"{self.skill_id}:test_get_response.intent") + # verify skill_id is now present in every message.context + for m in messages[6:]: + self.assertEqual(m.context["skill_id"], self.skill_id) + + # verify intent execution + self.assertEqual(messages[7].msg_type, "mycroft.skill.handler.start") + self.assertEqual(messages[7].data["name"], "TestAbortSkill.handle_test_get_response") + self.assertEqual(messages[8].msg_type, "enclosure.active_skill") + self.assertEqual(messages[8].data["skill_id"], self.skill_id) + self.assertEqual(messages[9].msg_type, "speak") + self.assertEqual(messages[9].data["lang"], "en-us") + self.assertTrue(messages[9].data["expect_response"]) # listen after dialog + self.assertEqual(messages[9].data["meta"]["skill"], self.skill_id) + + # enable get_response for this session + self.assertEqual(messages[10].msg_type, "skill.converse.get_response.enable") + self.assertEqual(messages[11].msg_type, "ovos.session.update_default") + + # user response would be here + + # disable get_response for this session + self.assertEqual(messages[12].msg_type, "skill.converse.get_response.disable") + self.assertEqual(messages[13].msg_type, "ovos.session.update_default") + + # post self.get_response intent code + self.assertEqual(messages[14].msg_type, "enclosure.active_skill") + self.assertEqual(messages[14].data["skill_id"], self.skill_id) + self.assertEqual(messages[15].msg_type, "speak") + self.assertEqual(messages[15].data["lang"], "en-us") + self.assertFalse(messages[15].data["expect_response"]) + self.assertEqual(messages[15].data["utterance"], "ERROR") + self.assertEqual(messages[15].data["meta"]["skill"], self.skill_id) + + self.assertEqual(messages[16].msg_type, "mycroft.skill.handler.complete") + self.assertEqual(messages[16].data["name"], "TestAbortSkill.handle_test_get_response") + + # verify default session is now updated + self.assertEqual(messages[17].msg_type, "ovos.session.update_default") + self.assertEqual(messages[17].data["session_data"]["session_id"], "default") + # test deserialization of payload + sess = Session.deserialize(messages[17].data["session_data"]) + self.assertEqual(sess.session_id, "default") + + def test_with_response(self): + SessionManager.sessions = {} + SessionManager.default_session = SessionManager.sessions["default"] = Session("default") + SessionManager.default_session.lang = "en-us" + + messages = [] + + def new_msg(msg): + nonlocal messages + m = Message.deserialize(msg) + if m.msg_type in ["ovos.skills.settings_changed"]: + return # skip these, only happen in 1st run + messages.append(m) + print(len(messages), msg) + + def wait_for_n_messages(n): + nonlocal messages + t = time.time() + while len(messages) < n: + sleep(0.1) + if time.time() - t > 10: + raise RuntimeError("did not get the number of expected messages under 10 seconds") + + self.core.bus.on("message", new_msg) + + def answer_get_response(msg): + sleep(0.5) + utt = Message("recognizer_loop:utterance", + {"utterances": ["ok"]}, + {"session": SessionManager.default_session.serialize()}) + self.core.bus.emit(utt) + + self.core.bus.on("skill.converse.get_response.enable", answer_get_response) + + # trigger get_response + utt = Message("recognizer_loop:utterance", + {"utterances": ["test get response"]}) + self.core.bus.emit(utt) + + # confirm all expected messages are sent + expected_messages = [ + "recognizer_loop:utterance", # no session + "skill.converse.ping", # default session injected + "skill.converse.pong", # test skill + "skill.converse.pong", # hello world skill + + # skill selected + "intent.service.skills.activated", + f"{self.skill_id}.activate", + f"{self.skill_id}:test_get_response.intent", + + # intent code before self.get_response + "mycroft.skill.handler.start", + "enclosure.active_skill", + "speak", # 'mycroft.mic.listen' if no dialog passed to get_response + "skill.converse.get_response.enable", # start of get_response + "ovos.session.update_default", # sync get_response status + + "recognizer_loop:utterance", # answer to get_response from user, + # converse pipeline start + "skill.converse.ping", + "skill.converse.pong", + "skill.converse.pong", + "skill.converse.get_response", # returning user utterance to running intent self.get_response + # skill selected by converse pipeline + "intent.service.skills.activated", + f"{self.skill_id}.activate", + "ovos.session.update_default", # sync skill activated by converse + + # intent code post self.get_response + "skill.converse.get_response.disable", # end of get_response + "ovos.session.update_default", # sync get_response status + "enclosure.active_skill", # from speak inside intent + "speak", # speak "ok" inside intent + "mycroft.skill.handler.complete", # original intent finished executing + + # session updated at end of intent pipeline + "ovos.session.update_default" + ] + wait_for_n_messages(len(expected_messages)) + + self.assertEqual(len(expected_messages), len(messages)) + + mtypes = [m.msg_type for m in messages] + for m in expected_messages: + self.assertTrue(m in mtypes) + + # verify that "session" is injected + # (missing in utterance message) and kept in all messages + for m in messages[1:]: + print(m.msg_type, m.context["session"]["session_id"]) + self.assertEqual(m.context["session"]["session_id"], "default") diff --git a/test/integrationtests/test_workshop.py b/test/integrationtests/test_workshop.py index 41da238e160..8660b366ea5 100644 --- a/test/integrationtests/test_workshop.py +++ b/test/integrationtests/test_workshop.py @@ -205,7 +205,7 @@ def test_get_response(self): self.assertIn(start_msg, self.bus.emitted_msgs) self.assertIn(speak_msg, self.bus.emitted_msgs) - self.assertIn(activate_msg, self.bus.emitted_msgs) + #self.assertIn(activate_msg, self.bus.emitted_msgs) # check that get_response loop is aborted # but intent continues executing diff --git a/test/unittests/skills/test_mycroft_skill_get_response.py b/test/unittests/skills/test_mycroft_skill_get_response.py index 37b44296cb0..0970c92dd2f 100644 --- a/test/unittests/skills/test_mycroft_skill_get_response.py +++ b/test/unittests/skills/test_mycroft_skill_get_response.py @@ -88,7 +88,7 @@ def test_wait_cancel(self): def is_cancel(utterance): return utterance == 'cancel' - response = skill._wait_response(is_cancel, validator, on_fail, 1) + response = skill._wait_response(is_cancel, validator, on_fail, 1, Message("")) self.assertEqual(response, None) converser.join() @@ -149,10 +149,11 @@ def validator(*args, **kwargs): expected_response = 'ice creamr please' skill._wait_response.return_value = expected_response + m = Message("") response = skill.get_response('what do you want', - validator=validator) + validator=validator, message=m) skill._wait_response.assert_called_with(AnyCallable(), validator, - AnyCallable(), -1) + AnyCallable(), -1, m) def test_converse_detection(self): """Ensure validator is passed on.""" @@ -164,9 +165,10 @@ def validator(*args, **kwargs): self.assertTrue(skill.converse_is_implemented) self.assertFalse(skill.converse_is_implemented) - skill.get_response('what do you want', validator=validator) + m = Message("") + skill.get_response('what do you want', validator=validator, message=m) skill._wait_response.assert_called_with(AnyCallable(), validator, - AnyCallable(), -1) + AnyCallable(), -1, m) self.assertFalse(skill.converse_is_implemented)