From c376922ca147b283fe84684b09c26556820e45ab Mon Sep 17 00:00:00 2001 From: Daniel McKnight Date: Mon, 16 Sep 2024 17:40:42 -0700 Subject: [PATCH 1/3] Add default response to `ask_yesno` to prevent infinite waiting for a response Refactor `test_common_query` for readability Add more output to failure cases --- neon_minerva/tests/test_skill_intents.py | 62 +++++++++++++----------- 1 file changed, 34 insertions(+), 28 deletions(-) diff --git a/neon_minerva/tests/test_skill_intents.py b/neon_minerva/tests/test_skill_intents.py index 67a7ba1..a389d4f 100644 --- a/neon_minerva/tests/test_skill_intents.py +++ b/neon_minerva/tests/test_skill_intents.py @@ -92,6 +92,11 @@ class TestSkillIntentMatching(unittest.TestCase): skill_id=test_skill_id, bus=bus, config_patch=core_config_patch) + @classmethod + def setUpClass(cls): + # Default respond "no" to any yes/no prompts + cls.skill.ask_yesno = Mock(return_value="no") + @classmethod def tearDownClass(cls) -> None: import shutil @@ -168,41 +173,42 @@ def test_common_query(self): for lang in self.common_query.keys(): for utt in self.common_query[lang]: if isinstance(utt, dict): - data = list(utt.values())[0] + cqs_data = list(utt.values())[0] utt = list(utt.keys())[0] else: - data = dict() + cqs_data = dict() utt = utt.lower() message = Message('test_utterance', {"utterance": utt, "lang": lang}) self.common_query_service.handle_question(message) - response = qa_response.call_args[0][0] - callback = qa_response.call_args[0][0] - self.assertIsInstance(response, Message) - self.assertTrue(response.data["phrase"] in utt) - self.assertEqual(response.data["skill_id"], self.skill.skill_id) - self.assertIn("callback_data", response.data.keys()) - self.assertIsInstance(response.data["conf"], float) - self.assertIsInstance(response.data["answer"], str) - - self.assertIsInstance(callback, Message) - self.assertEqual(callback.data['skill_id'], self.skill.skill_id) - self.assertEqual(callback.data['phrase'], - response.data['phrase']) - if not data: + response_msg = qa_response.call_args[0][0] + callback_msg = qa_response.call_args[0][0] + self.assertIsInstance(response_msg, Message) + self.assertTrue(response_msg.data["phrase"] in utt) + self.assertEqual(response_msg.data["skill_id"], self.skill.skill_id) + self.assertIn("callback_data", response_msg.data.keys()) + self.assertIsInstance(response_msg.data["conf"], float) + self.assertIsInstance(response_msg.data["answer"], str) + + self.assertIsInstance(callback_msg, Message) + self.assertEqual(callback_msg.data['skill_id'], self.skill.skill_id) + self.assertEqual(callback_msg.data['phrase'], + response_msg.data['phrase']) + if not cqs_data: continue - if isinstance(data.get('callback'), dict): - self.assertEqual(callback.data['callback_data'], - data['callback']) - elif isinstance(data.get('callback'), list): - self.assertEqual(set(callback.data['callback_data'].keys()), - set(data.get('callback'))) - if data.get('min_confidence'): - self.assertGreaterEqual(response.data['conf'], - data['min_confidence']) - if data.get('max_confidence'): - self.assertLessEqual(response.data['conf'], - data['max_confidence']) + if isinstance(cqs_data.get('callback'), dict): + self.assertEqual(callback_msg.data['callback_data'], + cqs_data['callback']) + elif isinstance(cqs_data.get('callback'), list): + # callback_data has `answer` but callback does not + self.assertEqual(set(callback_msg.data['callback_data'].keys()), + set(cqs_data.get('callback') + ['answer'])) + if cqs_data.get('min_confidence'): + self.assertGreaterEqual(response_msg.data['conf'], + cqs_data['min_confidence'], utt) + if cqs_data.get('max_confidence'): + self.assertLessEqual(response_msg.data['conf'], + cqs_data['max_confidence'], utt) def test_common_play(self): # TODO From 9f20466596dd32e8343e9daad1d011a1d53381cd Mon Sep 17 00:00:00 2001 From: Daniel McKnight Date: Tue, 17 Sep 2024 12:40:47 -0700 Subject: [PATCH 2/3] Add support for a configured "ready_event" to wait for before intent tests --- neon_minerva/tests/test_skill_intents.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/neon_minerva/tests/test_skill_intents.py b/neon_minerva/tests/test_skill_intents.py index a389d4f..4d647ab 100644 --- a/neon_minerva/tests/test_skill_intents.py +++ b/neon_minerva/tests/test_skill_intents.py @@ -30,6 +30,7 @@ from os import getenv from os.path import join, exists +from threading import Event from unittest.mock import Mock from ovos_bus_client import Message @@ -57,10 +58,11 @@ class TestSkillIntentMatching(unittest.TestCase): skill_entrypoint = getenv("TEST_SKILL_ENTRYPOINT") # Populate configuration - languages = list(valid_intents.keys()) - core_config_patch = {"secondary_langs": languages} negative_intents = valid_intents.pop('unmatched intents', dict()) common_query = valid_intents.pop("common query", dict()) + ready_event = valid_intents.pop("ready event") + languages = list(valid_intents.keys()) + core_config_patch = {"secondary_langs": languages} # Define intent parsers for tests if getenv("TEST_PADACIOSO") == "true": @@ -96,6 +98,10 @@ class TestSkillIntentMatching(unittest.TestCase): def setUpClass(cls): # Default respond "no" to any yes/no prompts cls.skill.ask_yesno = Mock(return_value="no") + if cls.ready_event and hasattr(cls.skill, cls.ready_event): + print("Configured ready event to wait for") + event: Event = getattr(cls.skill, cls.ready_event) + event.wait() @classmethod def tearDownClass(cls) -> None: From 927b58227407f21dbc04e97c702bff01726f7e42 Mon Sep 17 00:00:00 2001 From: Daniel McKnight Date: Tue, 17 Sep 2024 12:44:58 -0700 Subject: [PATCH 3/3] Update readme to document `ready event` config --- README.md | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index bc2076c..4c86c36 100644 --- a/README.md +++ b/README.md @@ -106,7 +106,10 @@ The following top-level sections can be added to intent test configuration: to expected: `callback_data` (list keys or dict data), `min_confidence`, and `max_confidence` - `common play`: TBD - +- `ready event`: Optional event name to wait for before starting tests. For example, + a skill may set `_update_event` after doing some long-running process at initialization; + configuring `ready event: _update_event` means intent tests will not start until + the skill has updated and is ready to handle inputs. ## Advanced Usage In addition to convenient CLI methods, this package also provides test cases that may be extended.