diff --git a/.github/workflows/license_tests.yml b/.github/workflows/license_tests.yml new file mode 100644 index 0000000..a284db6 --- /dev/null +++ b/.github/workflows/license_tests.yml @@ -0,0 +1,8 @@ +name: Run License Tests +on: + push: + workflow_dispatch: + +jobs: + license_tests: + uses: neongeckocom/.github/.github/workflows/license_tests.yml@master diff --git a/.github/workflows/propose_release.yml b/.github/workflows/propose_release.yml new file mode 100644 index 0000000..03673b2 --- /dev/null +++ b/.github/workflows/propose_release.yml @@ -0,0 +1,28 @@ +name: Propose Stable Release +on: + workflow_dispatch: + inputs: + release_type: + type: choice + description: Release Type + options: + - patch + - minor + - major +jobs: + update_version: + uses: neongeckocom/.github/.github/workflows/propose_semver_release.yml@master + with: + branch: dev + release_type: ${{ inputs.release_type }} + update_changelog: True + version_file: "neon_minerva/version.py" + pull_changes: + uses: neongeckocom/.github/.github/workflows/pull_master.yml@master + needs: update_version + with: + pr_reviewer: neonreviewers + pr_assignee: ${{ github.actor }} + pr_draft: false + pr_title: ${{ needs.update_version.outputs.version }} + pr_body: ${{ needs.update_version.outputs.changelog }} \ No newline at end of file diff --git a/.github/workflows/publish_release.yml b/.github/workflows/publish_release.yml new file mode 100644 index 0000000..e4b3fa3 --- /dev/null +++ b/.github/workflows/publish_release.yml @@ -0,0 +1,12 @@ +# This workflow will generate a release distribution and upload it to PyPI + +name: Publish Build and GitHub Release +on: + push: + branches: + - master + +jobs: + build_and_publish_pypi_and_release: + uses: neongeckocom/.github/.github/workflows/publish_stable_release.yml@master + secrets: inherit diff --git a/.github/workflows/publish_test_build.yml b/.github/workflows/publish_test_build.yml new file mode 100644 index 0000000..f805d71 --- /dev/null +++ b/.github/workflows/publish_test_build.yml @@ -0,0 +1,17 @@ +# This workflow will generate a distribution and upload it to PyPI + +name: Publish Alpha Build +on: + push: + branches: + - dev + paths-ignore: + - 'neon_minerva/version.py' + +jobs: + build_and_publish_pypi: + uses: neongeckocom/.github/.github/workflows/publish_alpha_release.yml@master + secrets: inherit + with: + version_file: "neon_minerva/version.py" + setup_py: "setup.py" diff --git a/.github/workflows/unit_tests.yml b/.github/workflows/unit_tests.yml new file mode 100644 index 0000000..b3de9ca --- /dev/null +++ b/.github/workflows/unit_tests.yml @@ -0,0 +1,10 @@ +# This workflow will run unit tests + +name: Run Unit Tests +on: + pull_request: + workflow_dispatch: + +jobs: + py_build_tests: + uses: neongeckocom/.github/.github/workflows/python_build_tests.yml@master diff --git a/CHANGELOG.md b/CHANGELOG.md new file mode 100644 index 0000000..d99e8d8 --- /dev/null +++ b/CHANGELOG.md @@ -0,0 +1,17 @@ +# Changelog + +## [0.0.1](https://github.com/NeonGeckoCom/neon-minerva/tree/0.0.1) (2023-10-26) + +[Full Changelog](https://github.com/NeonGeckoCom/neon-minerva/compare/1f81d6670a144b65069a882623ff3ef44ae93582...0.0.1) + +**Merged pull requests:** + +- Update GitHub release automation [\#7](https://github.com/NeonGeckoCom/neon-minerva/pull/7) ([NeonDaniel](https://github.com/NeonDaniel)) +- Update GitHub release automation [\#6](https://github.com/NeonGeckoCom/neon-minerva/pull/6) ([NeonDaniel](https://github.com/NeonDaniel)) +- Loosen padacioso dependency spec for neon-core compat. [\#4](https://github.com/NeonGeckoCom/neon-minerva/pull/4) ([NeonDaniel](https://github.com/NeonDaniel)) +- Add integration tests [\#3](https://github.com/NeonGeckoCom/neon-minerva/pull/3) ([NeonDaniel](https://github.com/NeonDaniel)) +- Initial implementation of portable skill tests [\#1](https://github.com/NeonGeckoCom/neon-minerva/pull/1) ([NeonDaniel](https://github.com/NeonDaniel)) + + + +\* *This Changelog was automatically generated by [github_changelog_generator](https://github.com/github-changelog-generator/github-changelog-generator)* diff --git a/LICENSE.md b/LICENSE.md new file mode 100644 index 0000000..525bb37 --- /dev/null +++ b/LICENSE.md @@ -0,0 +1,21 @@ +# NEON AI (TM) SOFTWARE, Software Development Kit & Application Development System +# All trademark and other rights reserved by their respective owners +# Copyright 2008-2021 Neongecko.com Inc. +# BSD-3 License + +Redistribution and use in source and binary forms, with or without modification, are permitted provided that the +following conditions are met: +1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following +disclaimer. +2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following +disclaimer in the documentation and/or other materials provided with the distribution. +3. Neither the name of the copyright holder nor the names of its contributors may be used to endorse or promote products +derived from this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, +INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, +WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF +THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/README.md b/README.md new file mode 100644 index 0000000..297b728 --- /dev/null +++ b/README.md @@ -0,0 +1,38 @@ +# Neon Minerva +Neon Minerva (Modular INtelligent Evaluation for a Reliable Voice Assistant) +provides tools for testing skills. + +Install the Minerva Python package with: `pip install neon-minerva` +The `minerva` entrypoint is available to interact with a bus via CLI. +Help is available via `minerva --help`. + +## Installation +Since skill intents may use Padatious, the following system packages must be +installed before installing this package: +```shell +sudo apt install swig libfann-dev +``` +To install this package from PyPI, simply run: +```shell +pip install neon-minerva +``` + +## Usage +This package provides a CLI for local testing of skills. Skills installed with +`pip` can be specified by entrypoint, or skills cloned locally can be specified +by root directory. + +### Resource Tests +To test that skill resources are defined for all supported languages, +`minerva test-resources ` +> - is the string entrypoint for the skill to test as specified in `setup.py` OR the path to + the skill's root directory +> - is a relative or absolute path to the resource test file, usually `test_resources.yaml` + +### Intent Tests +To test that skill intents match as expected for all supported languages, +`minerva test-intents ` +> - is the string entrypoint for the skill to test as specified in `setup.py` OR the path to + the skill's root directory +> - is a relative or absolute path to the resource test file, usually `test_intents.yaml` +> - The `--padacioso` flag can be added to test with Padacioso instead of Padatious for relevant intents diff --git a/neon_minerva/__init__.py b/neon_minerva/__init__.py new file mode 100644 index 0000000..d782cbb --- /dev/null +++ b/neon_minerva/__init__.py @@ -0,0 +1,25 @@ +# NEON AI (TM) SOFTWARE, Software Development Kit & Application Development System +# All trademark and other rights reserved by their respective owners +# Copyright 2008-2021 Neongecko.com Inc. +# BSD-3 +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# 1. Redistributions of source code must retain the above copyright notice, +# this list of conditions and the following disclaimer. +# 2. Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. +# 3. Neither the name of the copyright holder nor the names of its +# contributors may be used to endorse or promote products derived from this +# software without specific prior written permission. +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, +# THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR +# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, +# EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, +# OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF +# LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING +# NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +# SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/neon_minerva/cli.py b/neon_minerva/cli.py new file mode 100644 index 0000000..a9bfbc4 --- /dev/null +++ b/neon_minerva/cli.py @@ -0,0 +1,150 @@ +# NEON AI (TM) SOFTWARE, Software Development Kit & Application Development System +# All trademark and other rights reserved by their respective owners +# Copyright 2008-2021 Neongecko.com Inc. +# BSD-3 +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# 1. Redistributions of source code must retain the above copyright notice, +# this list of conditions and the following disclaimer. +# 2. Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. +# 3. Neither the name of the copyright holder nor the names of its +# contributors may be used to endorse or promote products derived from this +# software without specific prior written permission. +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, +# THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR +# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, +# EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, +# OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF +# LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING +# NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +# SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +import os +import click +import yaml + +from os.path import expanduser, relpath, isfile, isdir +from click_default_group import DefaultGroup +from unittest.runner import TextTestRunner +from unittest import makeSuite + +from neon_minerva.version import __version__ + + +def _init_tests(debug: bool = False): + from os.path import join + from os import makedirs + from tempfile import mkdtemp + base_dir = mkdtemp() + config = join(base_dir, "config") + data = join(base_dir, "data") + cache = join(base_dir, "cache") + makedirs(config, exist_ok=True) + makedirs(data, exist_ok=True) + makedirs(cache, exist_ok=True) + os.environ["XDG_CONFIG_HOME"] = config + os.environ["XDG_DATA_HOME"] = data + os.environ["XDG_CACHE_HOME"] = cache + + if debug: + os.environ["OVOS_DEFAULT_LOG_LEVEL"] = "DEBUG" + + +def _get_test_file(test_file: str) -> str: + """ + Parse an input path to locate a test file that may be relative to `~` or the + current working directory. + @param test_file: test file argument + @returns: best guess at the desired file path (may not exist) + """ + test_file = expanduser(test_file) + if not isfile(test_file): + test_file = relpath(test_file) + return test_file + + +def _get_skill_entrypoint(skill_entrypoint: str) -> str: + """ + Parse an input skill entrypoint and resolve either a locally installed skill + path, or an entrypoint for a plugin skill. + @param skill_entrypoint: Plugin entrypoint or path to skill + @returns: absolute file path if exists, else input entrypoint + """ + skill_path = expanduser(skill_entrypoint) + if not isdir(skill_path): + skill_path = relpath(skill_path) + if isdir(skill_path): + return skill_path + return skill_entrypoint + + +@click.group("minerva", cls=DefaultGroup, + no_args_is_help=True, invoke_without_command=True, + help="Minerva: Modular INtelligent Evaluation for a Reliable " + "Voice Assistant.\n\n" + "See also: minerva COMMAND --help") +@click.option("--version", "-v", is_flag=True, required=False, + help="Print the current version") +def neon_minerva_cli(version: bool = False): + if version: + click.echo(f"Minerva version {__version__}") + + +@neon_minerva_cli.command +@click.option('--debug', is_flag=True, default=False, + help="Flag to enable debug logging") +@click.argument("skill_entrypoint") +@click.argument("test_file") +def test_resources(skill_entrypoint, test_file, debug): + _init_tests(debug) + os.environ["TEST_SKILL_ENTRYPOINT"] = _get_skill_entrypoint(skill_entrypoint) + test_file = _get_test_file(test_file) + if not isfile(test_file): + click.echo(f"Could not find test file: {test_file}") + exit(2) + os.environ["RESOURCE_TEST_FILE"] = test_file + from neon_minerva.tests.test_skill_resources import TestSkillResources + TextTestRunner().run(makeSuite(TestSkillResources)) + + +@neon_minerva_cli.command +@click.option('--debug', is_flag=True, default=False, + help="Flag to enable debug logging") +@click.option('--padacioso', is_flag=True, default=False, + help="Flag to enable testing with Padacioso instead of Padatious") +@click.argument("skill_entrypoint") +@click.argument("test_file") +def test_intents(skill_entrypoint, test_file, debug, padacioso): + _init_tests(debug) + os.environ["TEST_PADACIOSO"] = "true" if padacioso else "false" + os.environ["TEST_SKILL_ENTRYPOINT"] = _get_skill_entrypoint(skill_entrypoint) + test_file = _get_test_file(test_file) + if not isfile(test_file): + click.echo(f"Could not find test file: {test_file}") + exit(2) + os.environ["INTENT_TEST_FILE"] = test_file + from neon_minerva.tests.test_skill_intents import TestSkillIntentMatching + TextTestRunner().run(makeSuite(TestSkillIntentMatching)) + + +@neon_minerva_cli.command +@click.option('-l', '--lang', default="en-us", + help="Language of test_file inputs") +@click.option('-a', '--audio', is_flag=True, default=False, + help="Test input as audio") +@click.argument("test_file") +def test_utterances(lang, audio, test_file): + from neon_utils.file_utils import load_commented_file + from neon_minerva.integration.user_utterance import UtteranceTests + + test_file = _get_test_file(test_file) + prompts = load_commented_file(test_file).split('\n') + click.echo(f"Testing {len(prompts)} prompts") + runner = UtteranceTests(prompts, lang=lang, audio=audio) + results = runner.run_test() + click.echo(yaml.safe_dump(results)) diff --git a/neon_minerva/exceptions.py b/neon_minerva/exceptions.py new file mode 100644 index 0000000..a9a305b --- /dev/null +++ b/neon_minerva/exceptions.py @@ -0,0 +1,43 @@ +# NEON AI (TM) SOFTWARE, Software Development Kit & Application Framework +# All trademark and other rights reserved by their respective owners +# Copyright 2008-2022 Neongecko.com Inc. +# Contributors: Daniel McKnight, Guy Daniels, Elon Gasper, Richard Leeds, +# Regina Bloomstine, Casimiro Ferreira, Andrii Pernatii, Kirill Hrymailo +# BSD-3 License +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# 1. Redistributions of source code must retain the above copyright notice, +# this list of conditions and the following disclaimer. +# 2. Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. +# 3. Neither the name of the copyright holder nor the names of its +# contributors may be used to endorse or promote products derived from this +# software without specific prior written permission. +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, +# THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR +# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, +# EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, +# OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF +# LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING +# NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +# SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +class IntentException(Exception): + """ + Base exception for an intent error + """ + +class IntentNotMatched(IntentException): + """ + Exception indicating an intent match was expected but not found + """ + + +class ConfidenceTooLow(IntentException): + """ + Exception indicating an intent match confidence was below specified minimum. + """ \ No newline at end of file diff --git a/neon_minerva/integration/__init__.py b/neon_minerva/integration/__init__.py new file mode 100644 index 0000000..d782cbb --- /dev/null +++ b/neon_minerva/integration/__init__.py @@ -0,0 +1,25 @@ +# NEON AI (TM) SOFTWARE, Software Development Kit & Application Development System +# All trademark and other rights reserved by their respective owners +# Copyright 2008-2021 Neongecko.com Inc. +# BSD-3 +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# 1. Redistributions of source code must retain the above copyright notice, +# this list of conditions and the following disclaimer. +# 2. Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. +# 3. Neither the name of the copyright holder nor the names of its +# contributors may be used to endorse or promote products derived from this +# software without specific prior written permission. +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, +# THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR +# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, +# EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, +# OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF +# LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING +# NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +# SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/neon_minerva/integration/user_utterance.py b/neon_minerva/integration/user_utterance.py new file mode 100644 index 0000000..e4343f6 --- /dev/null +++ b/neon_minerva/integration/user_utterance.py @@ -0,0 +1,210 @@ +# NEON AI (TM) SOFTWARE, Software Development Kit & Application Development System +# All trademark and other rights reserved by their respective owners +# Copyright 2008-2021 Neongecko.com Inc. +# BSD-3 +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# 1. Redistributions of source code must retain the above copyright notice, +# this list of conditions and the following disclaimer. +# 2. Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. +# 3. Neither the name of the copyright holder nor the names of its +# contributors may be used to endorse or promote products derived from this +# software without specific prior written permission. +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, +# THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR +# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, +# EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, +# OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF +# LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING +# NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +# SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +from tempfile import mkstemp +from threading import Event, Lock +from time import time +from typing import List + +from neon_utils.file_utils import encode_file_to_base64_string +from ovos_utils.log import LOG +from ovos_bus_client.client import MessageBusClient +from ovos_bus_client.message import Message +from ovos_plugin_manager.tts import TTS + + +class UtteranceTests: + def __init__(self, prompts: List[str], lang: str = "en-us", + bus_config: dict = None, user_config: dict = None, + audio: bool = False, tts: TTS = None): + if not user_config: + from neon_utils.configuration_utils import get_neon_user_config + user_config = get_neon_user_config().content + user_config['user']['username'] = "minerva" + self._user_config = user_config + bus_config = bus_config or dict() + self.core_bus = MessageBusClient(**bus_config) + self.core_bus.run_in_thread() + self.lang = lang + self.test_audio = audio + self._tts = tts + # TODO: Handle prompt metadata for longer timeouts + self._prompts = prompts + self._stt_timeout = 60 # Time to transcribe + audio parsers + self._intent_timeout = 60 # Time to match AND handle intent + self._speak_timeout = 60 # Time after intent handling for TTS playback + + self._results = list() + self._audio_output_done = Event() + self._prompt_handled = Event() + self._prompt_lock = Lock() + + self._last_message = None + self._audio_output_done.set() + self.register_bus_events() + + def run_test(self) -> dict: + """ + Run tests and return dict timing results + """ + self._results.clear() + for prompt in self._prompts: + self.handle_prompt(prompt) + aggregated_results = {"save_transcript": [], + "text_parsers": [], + "get_tts": [], + "intent_handler": [], + "total": []} + if self.test_audio: + aggregated_results['get_stt'] = [] + for result in self._results: + try: + aggregated_results['save_transcript'].append(result['save_transcript']) + aggregated_results['text_parsers'].append(result['text_parsers']) + aggregated_results['get_tts'].append(result['get_tts']) + aggregated_results['intent_handler'].append(result['speech_start'] - result['handle_utterance']) + aggregated_results['total'].append(result['finished'] - result['transcribed']) + if self.test_audio: + aggregated_results['get_stt'].append(result['get_stt']) + except KeyError: + LOG.error(result) + formatted_results = dict() + for key, values in aggregated_results.items(): + formatted_results[key] = {"average": round(sum(values) / + len(values), 6), + "minimum": round(min(values), 6), + "maximum": round(max(values), 6)} + return formatted_results + + def register_bus_events(self): + """ + Register listeners to track audio and skill module states + """ + self.core_bus.on("recognizer_loop:audio_output_start", + self._audio_started) + self.core_bus.on("recognizer_loop:audio_output_end", + self._audio_stopped) + self.core_bus.on("mycroft.mic.listen", self._mic_listen) + self.core_bus.on("mycroft.skill.handler.complete", + self._handler_complete) + + def _audio_started(self, _): + """ + Handle audio output started + """ + self._audio_output_done.clear() + + def _audio_stopped(self, message): + """ + Handle audio output finished + @param message: Message associated with completed audio playback + """ + LOG.debug("audio finished") + self._last_message = message + self._audio_output_done.set() + + def _mic_listen(self, message): + """ + Handle start listening (for prompts that trigger `get_response`) + @param message: Message associated with completed skill handler + """ + LOG.debug("`get_response` call") + # self._last_message = message + self._prompt_handled.set() + + def _handler_complete(self, _): + """ + Handle skill execution complete (audio output may not be complete) + """ + LOG.debug("Skill Handler Complete") + self._prompt_handled.set() + + def send_prompt(self, prompt: str): + """ + Send a prompt to core for intent handling + """ + context = {"neon_should_respond": True, + "source": ["minerva"], + "destination": ["skills"], + "timing": {"transcribed": time()}, + "username": "minerva", + "user_profiles": [self._user_config]} + if self.test_audio: + if self._tts: + _, file_path = mkstemp() + audio, _ = self._tts.get_tts(prompt, file_path, lang=self.lang) + else: + resp = self.core_bus.wait_for_response( + Message("neon.get_tts", {'text': prompt, + 'speaker': {'language': self.lang, + 'gender': 'female'}}), + timeout=self._stt_timeout) + file_path = resp.data[self.lang]['female'] + resp = self.core_bus.wait_for_response( + Message("neon.audio_input", + {"audio_data": encode_file_to_base64_string(file_path), + "lang": self.lang}, context), + timeout=self._stt_timeout) + LOG.info(resp.data) + if prompt.lower() not in (t.lower() for t + in resp.data['transcripts']): + LOG.warning(f"Invalid transcription for '{prompt}': " + f"{resp.data['transcripts']}") + else: + self.core_bus.emit(Message("recognizer_loop:utterance", + {"utterances": [prompt], + "lang": self.lang}, context)) + + def handle_prompt(self, prompt: str): + """ + Send a prompt (text or audio) and collect timing results. + @param prompt: string prompt to send for intent (and optionally STT) + processing + """ + with self._prompt_lock: + # Ensure event state matches expectation + if not self._audio_output_done.is_set(): + LOG.warning("Audio output not finished when expected!") + self._audio_output_done.wait(self._speak_timeout) + self._audio_output_done.clear() + self._prompt_handled.clear() + self._last_message = None + + # Send prompt + self.send_prompt(prompt) + try: + assert self._prompt_handled.wait(self._intent_timeout) + assert self._audio_output_done.wait(self._speak_timeout) + assert self._last_message is not None + if "speech_start" not in self._last_message.context["timing"]: + LOG.warning(f"Missing speech_start timestamp for {prompt}") + self._last_message.context["timing"]["speech_start"] = \ + self._last_message.context["timing"]["handle_utterance"] + self._results.append({**self._last_message.context["timing"], + **{'finished': time()}}) + except AssertionError as e: + LOG.error(f"{prompt}: {e}") + LOG.debug(f"Handled {prompt}") diff --git a/neon_minerva/intent_services/__init__.py b/neon_minerva/intent_services/__init__.py new file mode 100644 index 0000000..69d7574 --- /dev/null +++ b/neon_minerva/intent_services/__init__.py @@ -0,0 +1,30 @@ +# NEON AI (TM) SOFTWARE, Software Development Kit & Application Development System +# All trademark and other rights reserved by their respective owners +# Copyright 2008-2021 Neongecko.com Inc. +# BSD-3 +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# 1. Redistributions of source code must retain the above copyright notice, +# this list of conditions and the following disclaimer. +# 2. Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. +# 3. Neither the name of the copyright holder nor the names of its +# contributors may be used to endorse or promote products derived from this +# software without specific prior written permission. +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, +# THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR +# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, +# EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, +# OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF +# LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING +# NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +# SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +from collections import namedtuple + +IntentMatch = namedtuple('IntentMatch', + ['intent_service', 'intent_type', + 'intent_data', 'skill_id', 'utterance']) diff --git a/neon_minerva/intent_services/adapt.py b/neon_minerva/intent_services/adapt.py new file mode 100644 index 0000000..0575e9a --- /dev/null +++ b/neon_minerva/intent_services/adapt.py @@ -0,0 +1,88 @@ +# NEON AI (TM) SOFTWARE, Software Development Kit & Application Framework +# All trademark and other rights reserved by their respective owners +# Copyright 2008-2022 Neongecko.com Inc. +# Contributors: Daniel McKnight, Guy Daniels, Elon Gasper, Richard Leeds, +# Regina Bloomstine, Casimiro Ferreira, Andrii Pernatii, Kirill Hrymailo +# BSD-3 License +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# 1. Redistributions of source code must retain the above copyright notice, +# this list of conditions and the following disclaimer. +# 2. Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. +# 3. Neither the name of the copyright holder nor the names of its +# contributors may be used to endorse or promote products derived from this +# software without specific prior written permission. +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, +# THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR +# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, +# EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, +# OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF +# LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING +# NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +# SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +from typing import Optional +from adapt.engine import IntentDeterminationEngine +from ovos_utils.intents.intent_service_interface import open_intent_envelope +from ovos_utils.log import LOG +from ovos_utils.messagebus import FakeBus, get_message_lang + +from neon_minerva.exceptions import IntentNotMatched, ConfidenceTooLow +from neon_minerva.intent_services import IntentMatch + + +class AdaptContainer: + def __init__(self, lang: str, bus: FakeBus): + self.lang = lang.lower() + self.bus = bus + self.adapt = IntentDeterminationEngine() + self.bus.on('register_vocab', self.handle_register_vocab) + self.bus.on('register_intent', self.handle_register_intent) + + def handle_register_vocab(self, message): + entity_value = message.data.get('entity_value') + entity_type = message.data.get('entity_type') + regex_str = message.data.get('regex') + alias_of = message.data.get('alias_of') + lang = get_message_lang(message) + if lang != self.lang: + return + if regex_str: + self.adapt.register_regex_entity(regex_str) + else: + self.adapt.register_entity(entity_value, entity_type, + alias_of=alias_of) + + def handle_register_intent(self, message): + intent = open_intent_envelope(message) + self.adapt.register_intent_parser(intent) + + def test_intent(self, utterance: str) -> Optional[IntentMatch]: + best_intent = None + try: + intents = [i for i in self.adapt.determine_intent( + utterance, 100, + include_tags=True)] + if intents: + best_intent = max(intents, + key=lambda x: x.get('confidence', 0.0)) + except Exception as err: + LOG.exception(err) + + if not best_intent: + raise IntentNotMatched(utterance) + LOG.debug(best_intent) + skill_id = best_intent['intent_type'].split(":")[0] + _norm_id = skill_id.replace('.', '_') + intent_data = {k.replace(_norm_id, '', 1): v for k, v in + best_intent.items() if k.startswith(_norm_id) and + isinstance(v, str)} + LOG.debug(intent_data) + ret = IntentMatch('Adapt', best_intent['intent_type'], intent_data, + skill_id, utterance) + return ret diff --git a/neon_minerva/intent_services/padacioso.py b/neon_minerva/intent_services/padacioso.py new file mode 100644 index 0000000..cb21db3 --- /dev/null +++ b/neon_minerva/intent_services/padacioso.py @@ -0,0 +1,73 @@ +# NEON AI (TM) SOFTWARE, Software Development Kit & Application Framework +# All trademark and other rights reserved by their respective owners +# Copyright 2008-2022 Neongecko.com Inc. +# Contributors: Daniel McKnight, Guy Daniels, Elon Gasper, Richard Leeds, +# Regina Bloomstine, Casimiro Ferreira, Andrii Pernatii, Kirill Hrymailo +# BSD-3 License +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# 1. Redistributions of source code must retain the above copyright notice, +# this list of conditions and the following disclaimer. +# 2. Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. +# 3. Neither the name of the copyright holder nor the names of its +# contributors may be used to endorse or promote products derived from this +# software without specific prior written permission. +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, +# THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR +# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, +# EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, +# OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF +# LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING +# NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +# SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +from padacioso import IntentContainer +from ovos_utils.log import LOG +from ovos_utils.messagebus import FakeBus + + +class PadaciosoContainer: + def __init__(self, lang: str, cache_dir: str, bus: FakeBus): + self.cache_dir = cache_dir + self.lang = lang.lower() + self.bus = bus + self.padatious = IntentContainer(False) + self.bus.on('padatious:register_intent', self.register_intent) + self.bus.on('padatious:register_entity', self.register_entity) + + def register_intent(self, message): + """Messagebus handler for registering intents. + + Args: + message (Message): message triggering action + """ + lang = message.data.get('lang', self.lang) + lang = lang.lower() + if lang == self.lang: + LOG.debug(f"Loading intent: {message.data['name']}") + self.padatious.add_intent(message.data['name'], + message.data["samples"]) + else: + LOG.debug(f"Ignoring {message.data['name']}") + + def register_entity(self, message): + """Messagebus handler for registering entities. + + Args: + message (Message): message triggering action + """ + lang = message.data.get('lang', self.lang) + lang = lang.lower() + if lang == self.lang: + self.padatious.add_entity(message.data['name'], + message.data['samples']) + + def calc_intent(self, utt: str) -> dict: + intent = self.padatious.calc_intent(utt) + LOG.debug(intent) + return intent or dict() diff --git a/neon_minerva/intent_services/padatious.py b/neon_minerva/intent_services/padatious.py new file mode 100644 index 0000000..62f85a4 --- /dev/null +++ b/neon_minerva/intent_services/padatious.py @@ -0,0 +1,102 @@ +# NEON AI (TM) SOFTWARE, Software Development Kit & Application Framework +# All trademark and other rights reserved by their respective owners +# Copyright 2008-2022 Neongecko.com Inc. +# Contributors: Daniel McKnight, Guy Daniels, Elon Gasper, Richard Leeds, +# Regina Bloomstine, Casimiro Ferreira, Andrii Pernatii, Kirill Hrymailo +# BSD-3 License +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# 1. Redistributions of source code must retain the above copyright notice, +# this list of conditions and the following disclaimer. +# 2. Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. +# 3. Neither the name of the copyright holder nor the names of its +# contributors may be used to endorse or promote products derived from this +# software without specific prior written permission. +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, +# THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR +# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, +# EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, +# OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF +# LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING +# NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +# SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +from padatious import IntentContainer +from ovos_utils.log import LOG +from ovos_utils.messagebus import FakeBus + +from neon_minerva.exceptions import IntentNotMatched, ConfidenceTooLow +from neon_minerva.intent_services import IntentMatch + + +class PadatiousContainer: + def __init__(self, lang: str, cache_path: str, bus: FakeBus): + self.cache_dir = cache_path + self.lang = lang.lower() + self.bus = bus + self.padatious = IntentContainer(cache_path) + self.bus.on('padatious:register_intent', self.register_intent) + self.bus.on('padatious:register_entity', self.register_entity) + + def register_intent(self, message): + """Messagebus handler for registering intents. + + Args: + message (Message): message triggering action + """ + lang = message.data.get('lang', self.lang) + lang = lang.lower() + if lang == self.lang: + LOG.debug(f"Loading intent: {message.data['name']}") + self.padatious.load_intent(message.data['name'], + message.data['file_name']) + else: + LOG.debug(f"Ignoring {message.data['name']}") + + def register_entity(self, message): + """Messagebus handler for registering entities. + + Args: + message (Message): message triggering action + """ + lang = message.data.get('lang', self.lang) + lang = lang.lower() + if lang == self.lang: + self.padatious.load_entity(message.data['name'], + message.data['file_name']) + + def calc_intent(self, utt: str) -> dict: + intent = self.padatious.calc_intent(utt) + LOG.debug(intent) + return intent.__dict__ if intent else dict() + + +class TestPadatiousMatcher: + def __init__(self, container: PadatiousContainer, + include_med: bool = True, include_low: bool = False): + LOG.debug("Creating test Padatious Matcher") + if include_low: + self.min_conf = 0.5 + elif include_med: + self.min_conf = 0.8 + else: + self.min_conf = 0.95 + self.padatious = container + + def test_intent(self, utterance: str) -> IntentMatch: + intent = self.padatious.calc_intent(utterance) + if not intent: + raise IntentNotMatched(utterance) + conf = intent.get("conf") or 0.0 + if conf < self.min_conf: + raise ConfidenceTooLow(f"{conf} less than minimum {self.min_conf}") + skill_id = intent.get('name').split(':')[0] + sentence = ' '.join(intent.get('sent')) if intent.get('sent') else utterance + return IntentMatch('Padatious', intent.get('name'), + intent.get('matches') or intent.get('entities'), + skill_id, sentence) diff --git a/neon_minerva/skill.py b/neon_minerva/skill.py new file mode 100644 index 0000000..6106e66 --- /dev/null +++ b/neon_minerva/skill.py @@ -0,0 +1,88 @@ +# NEON AI (TM) SOFTWARE, Software Development Kit & Application Development System +# All trademark and other rights reserved by their respective owners +# Copyright 2008-2021 Neongecko.com Inc. +# BSD-3 +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# 1. Redistributions of source code must retain the above copyright notice, +# this list of conditions and the following disclaimer. +# 2. Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. +# 3. Neither the name of the copyright holder nor the names of its +# contributors may be used to endorse or promote products derived from this +# software without specific prior written permission. +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, +# THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR +# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, +# EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, +# OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF +# LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING +# NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +# SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +import yaml + +from os.path import expanduser, isfile, isdir +from typing import Optional +from ovos_utils.messagebus import FakeBus +from ovos_workshop.skills.base import BaseSkill +from ovos_utils.log import LOG + + +def get_skill_object(skill_entrypoint: str, bus: FakeBus, + skill_id: str, config_patch: Optional[dict] = None) -> BaseSkill: + """ + Get an initialized skill object by entrypoint with the requested skill_id. + @param skill_entrypoint: Skill plugin entrypoint or directory path + @param bus: FakeBus instance to bind to skill for testing + @param skill_id: skill_id to initialize skill with + @returns: Initialized skill object + """ + if config_patch: + from ovos_config.config import update_mycroft_config + update_mycroft_config(config_patch) + if isdir(skill_entrypoint): + LOG.info(f"Loading local skill: {skill_entrypoint}") + from ovos_workshop.skill_launcher import SkillLoader + loader = SkillLoader(bus, skill_entrypoint, skill_id) + if loader.load(): + return loader.instance + from ovos_plugin_manager.skills import find_skill_plugins + plugins = find_skill_plugins() + if skill_entrypoint not in plugins: + raise ValueError(f"Requested skill not found: {skill_entrypoint}") + plugin = plugins[skill_entrypoint] + skill = plugin(bus=bus, skill_id=skill_id) + return skill + + +def load_resource_tests(test_file: str) -> dict: + """ + Load resource tests from a file + @param test_file: Test file to load + @returns: Loaded test spec + """ + test_file = expanduser(test_file) + if not isfile(test_file): + raise FileNotFoundError(test_file) + with open(test_file) as f: + resources = yaml.safe_load(f) + return resources + + +def load_intent_tests(test_file: str) -> dict: + """ + Load intent tests from a file + @param test_file: Test file to load + @returns: Loaded test spec + """ + test_file = expanduser(test_file) + if not isfile(test_file): + raise FileNotFoundError(test_file) + with open(test_file) as f: + intents = yaml.safe_load(f) + return intents diff --git a/neon_minerva/tests/__init__.py b/neon_minerva/tests/__init__.py new file mode 100644 index 0000000..d782cbb --- /dev/null +++ b/neon_minerva/tests/__init__.py @@ -0,0 +1,25 @@ +# NEON AI (TM) SOFTWARE, Software Development Kit & Application Development System +# All trademark and other rights reserved by their respective owners +# Copyright 2008-2021 Neongecko.com Inc. +# BSD-3 +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# 1. Redistributions of source code must retain the above copyright notice, +# this list of conditions and the following disclaimer. +# 2. Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. +# 3. Neither the name of the copyright holder nor the names of its +# contributors may be used to endorse or promote products derived from this +# software without specific prior written permission. +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, +# THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR +# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, +# EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, +# OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF +# LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING +# NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +# SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/neon_minerva/tests/test_skill_intents.py b/neon_minerva/tests/test_skill_intents.py new file mode 100644 index 0000000..71c1a89 --- /dev/null +++ b/neon_minerva/tests/test_skill_intents.py @@ -0,0 +1,145 @@ +# NEON AI (TM) SOFTWARE, Software Development Kit & Application Framework +# All trademark and other rights reserved by their respective owners +# Copyright 2008-2022 Neongecko.com Inc. +# Contributors: Daniel McKnight, Guy Daniels, Elon Gasper, Richard Leeds, +# Regina Bloomstine, Casimiro Ferreira, Andrii Pernatii, Kirill Hrymailo +# BSD-3 License +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# 1. Redistributions of source code must retain the above copyright notice, +# this list of conditions and the following disclaimer. +# 2. Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. +# 3. Neither the name of the copyright holder nor the names of its +# contributors may be used to endorse or promote products derived from this +# software without specific prior written permission. +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, +# THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR +# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, +# EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, +# OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF +# LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING +# NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +# SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +import unittest + +from os import getenv +from os.path import join, exists +from ovos_utils.messagebus import FakeBus +from ovos_utils.log import LOG + +from neon_minerva.exceptions import IntentException +from neon_minerva.skill import get_skill_object, load_intent_tests +from neon_minerva.intent_services.padatious import PadatiousContainer, TestPadatiousMatcher +from neon_minerva.intent_services.adapt import AdaptContainer +from neon_minerva.intent_services.padacioso import PadaciosoContainer +from neon_minerva.intent_services import IntentMatch + + +class TestSkillIntentMatching(unittest.TestCase): + # Static parameters + bus = FakeBus() + bus.run_forever() + test_skill_id = 'test_skill.test' + padatious_cache = join(getenv("XDG_CACHE_HOME"), "padatious") + + # Define skill and resource spec to use in tests + valid_intents = load_intent_tests(getenv("INTENT_TEST_FILE")) + skill_entrypoint = getenv("TEST_SKILL_ENTRYPOINT") + + # Populate configuration + languages = list(valid_intents.keys()) + core_config_patch = {"secondary_langs": languages} + negative_intents = valid_intents.pop('unmatched intents', dict()) + common_query = valid_intents.pop("common query", dict()) + + # Define intent parsers for tests + if getenv("TEST_PADACIOSO") == "true": + container = PadaciosoContainer + else: + container = PadatiousContainer + padatious_services = dict() + adapt_services = dict() + for lang in languages: + padatious_services[lang] = container(lang, join(padatious_cache, lang), + bus) + adapt_services[lang] = AdaptContainer(lang, bus) + + skill = get_skill_object(skill_entrypoint=skill_entrypoint, + skill_id=test_skill_id, bus=bus, + config_patch=core_config_patch) + + @classmethod + def tearDownClass(cls) -> None: + import shutil + for service in cls.padatious_services.values(): + try: + if exists(service.cache_dir): + shutil.rmtree(service.cache_dir) + except Exception as e: + LOG.exception(e) + + def test_intents(self): + for lang in self.valid_intents.keys(): + self.assertIsInstance(lang.split('-')[0], str) + self.assertIsInstance(lang.split('-')[1], str) + for intent, examples in self.valid_intents[lang].items(): + # TODO: Better method to determine parser? + if intent.endswith('.intent'): + parser = TestPadatiousMatcher(self.padatious_services[lang]) + else: + parser = self.adapt_services[lang] + + for utt in examples: + if isinstance(utt, dict): + data = list(utt.values())[0] + utt = list(utt.keys())[0] + else: + data = list() + + match = parser.test_intent(utt) + self.assertIsInstance(match, IntentMatch) + self.assertEqual(match.skill_id, self.test_skill_id) + self.assertEqual(match.intent_type, + f"{self.test_skill_id}:{intent}") + self.assertEqual(match.utterance, utt) + + for datum in data: + if isinstance(datum, dict): + name = list(datum.keys())[0] + value = list(datum.values())[0] + else: + name = datum + value = None + self.assertIn(name, match.intent_data, utt) + if value: + self.assertEqual(match.intent_data[name], value) + + def test_negative_intents(self): + config = self.negative_intents.pop('config', {}) + include_med = config.get('include_med', True) + include_low = config.get('include_low', False) + + for lang in self.negative_intents.keys(): + adapt = self.adapt_services[lang] + padatious = TestPadatiousMatcher(self.padatious_services[lang], + include_med=include_med, + include_low=include_low) + for utt in self.negative_intents[lang]: + with self.assertRaises(IntentException, msg=utt): + adapt.test_intent(utt) + with self.assertRaises(IntentException, msg=utt): + padatious.test_intent(utt) + + def test_common_query(self): + # TODO + pass + + def test_common_play(self): + # TODO + pass diff --git a/neon_minerva/tests/test_skill_resources.py b/neon_minerva/tests/test_skill_resources.py new file mode 100644 index 0000000..afa4a0a --- /dev/null +++ b/neon_minerva/tests/test_skill_resources.py @@ -0,0 +1,144 @@ +# NEON AI (TM) SOFTWARE, Software Development Kit & Application Framework +# All trademark and other rights reserved by their respective owners +# Copyright 2008-2022 Neongecko.com Inc. +# Contributors: Daniel McKnight, Guy Daniels, Elon Gasper, Richard Leeds, +# Regina Bloomstine, Casimiro Ferreira, Andrii Pernatii, Kirill Hrymailo +# BSD-3 License +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# 1. Redistributions of source code must retain the above copyright notice, +# this list of conditions and the following disclaimer. +# 2. Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. +# 3. Neither the name of the copyright holder nor the names of its +# contributors may be used to endorse or promote products derived from this +# software without specific prior written permission. +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, +# THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR +# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, +# EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, +# OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF +# LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING +# NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +# SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +import unittest +import os +import json + +from os import getenv +from ovos_utils.messagebus import FakeBus + +from neon_minerva.skill import get_skill_object, load_resource_tests + + +class TestSkillResources(unittest.TestCase): + # Static parameters + messages = list() + bus = FakeBus() + bus.run_forever() + test_skill_id = 'test_skill.test' + + # Define skill and resource spec to use in tests + resources = load_resource_tests(getenv("RESOURCE_TEST_FILE")) + skill_entrypoint = getenv("TEST_SKILL_ENTRYPOINT") + + # Specify valid languages to test + supported_languages = resources['languages'] + + # Specify skill intents as sets + adapt_intents = set(resources['intents']['adapt']) + padatious_intents = set(resources['intents']['padatious']) + + # regex entities, not necessarily filenames + regex = set(resources['regex']) + # vocab is lowercase .voc file basenames + vocab = set(resources['vocab']) + # dialog is .dialog file basenames (case-sensitive) + dialog = set(resources['dialog']) + + core_config_patch = {"secondary_langs": supported_languages} + + @classmethod + def setUpClass(cls) -> None: + cls.bus.on("message", cls._on_message) + + cls.skill = get_skill_object(skill_entrypoint=cls.skill_entrypoint, + bus=cls.bus, skill_id=cls.test_skill_id, + config_patch=cls.core_config_patch) + + cls.adapt_intents = {f'{cls.test_skill_id}:{intent}' + for intent in cls.adapt_intents} + cls.padatious_intents = {f'{cls.test_skill_id}:{intent}' + for intent in cls.padatious_intents} + + @classmethod + def _on_message(cls, message): + cls.messages.append(json.loads(message)) + + def test_skill_setup(self): + self.assertEqual(self.skill.skill_id, self.test_skill_id) + self.assertEqual(set([self.skill._core_lang] + + self.skill._secondary_langs), + set(self.supported_languages)) + + def test_intent_registration(self): + registered_adapt = list() + registered_padatious = dict() + registered_vocab = dict() + registered_regex = dict() + for msg in self.messages: + if msg["type"] == "register_intent": + registered_adapt.append(msg["data"]["name"]) + elif msg["type"] == "padatious:register_intent": + lang = msg["data"]["lang"] + registered_padatious.setdefault(lang, list()) + registered_padatious[lang].append(msg["data"]["name"]) + elif msg["type"] == "register_vocab": + lang = msg["data"]["lang"] + if msg['data'].get('regex'): + registered_regex.setdefault(lang, dict()) + regex = msg["data"]["regex"].split( + '<', 1)[1].split('>', 1)[0].replace( + self.test_skill_id.replace('.', '_'), '') + registered_regex[lang].setdefault(regex, list()) + registered_regex[lang][regex].append(msg["data"]["regex"]) + else: + registered_vocab.setdefault(lang, dict()) + voc_filename = msg["data"]["entity_type"].replace( + self.test_skill_id.replace('.', '_'), '').lower() + registered_vocab[lang].setdefault(voc_filename, list()) + registered_vocab[lang][voc_filename].append( + msg["data"]["entity_value"]) + self.assertEqual(set(registered_adapt), self.adapt_intents, + registered_adapt) + for lang in self.supported_languages: + if self.padatious_intents: + self.assertEqual(set(registered_padatious[lang]), + self.padatious_intents, + registered_padatious[lang]) + if self.vocab: + self.assertEqual(set(registered_vocab[lang].keys()), + self.vocab, registered_vocab) + if self.regex: + self.assertEqual(set(registered_regex[lang].keys()), + self.regex, registered_regex) + for voc in self.vocab: + # Ensure every vocab file has at least one entry + self.assertGreater(len(registered_vocab[lang][voc]), 0) + for rx in self.regex: + # Ensure every rx file has exactly one entry + self.assertTrue(all((rx in line for line in + registered_regex[lang][rx])), self.regex) + + def test_dialog_files(self): + for lang in self.supported_languages: + for dialog in self.dialog: + file = self.skill.find_resource(f"{dialog}.dialog", "dialog", + lang) + self.assertIsInstance(file, str, f"{dialog} in {self.dialog}") + self.assertTrue(os.path.isfile(file), dialog) diff --git a/neon_minerva/version.py b/neon_minerva/version.py new file mode 100644 index 0000000..ef7228e --- /dev/null +++ b/neon_minerva/version.py @@ -0,0 +1,29 @@ +# NEON AI (TM) SOFTWARE, Software Development Kit & Application Framework +# All trademark and other rights reserved by their respective owners +# Copyright 2008-2022 Neongecko.com Inc. +# Contributors: Daniel McKnight, Guy Daniels, Elon Gasper, Richard Leeds, +# Regina Bloomstine, Casimiro Ferreira, Andrii Pernatii, Kirill Hrymailo +# BSD-3 License +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# 1. Redistributions of source code must retain the above copyright notice, +# this list of conditions and the following disclaimer. +# 2. Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. +# 3. Neither the name of the copyright holder nor the names of its +# contributors may be used to endorse or promote products derived from this +# software without specific prior written permission. +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, +# THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR +# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, +# EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, +# OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF +# LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING +# NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +# SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +__version__ = "0.0.1" diff --git a/requirements/requirements.txt b/requirements/requirements.txt new file mode 100644 index 0000000..55f1e1d --- /dev/null +++ b/requirements/requirements.txt @@ -0,0 +1,8 @@ +click~=8.0 +click-default-group~=1.2 +ovos-utils~=0.0.35 +ovos-workshop~=0.0.12 +fann2==1.0.7 +padatious~=0.4.8 +padacioso~=0.1 +pyyaml~=6.0 \ No newline at end of file diff --git a/setup.py b/setup.py new file mode 100644 index 0000000..52304e5 --- /dev/null +++ b/setup.py @@ -0,0 +1,80 @@ +# NEON AI (TM) SOFTWARE, Software Development Kit & Application Framework +# All trademark and other rights reserved by their respective owners +# Copyright 2008-2022 Neongecko.com Inc. +# Contributors: Daniel McKnight, Guy Daniels, Elon Gasper, Richard Leeds, +# Regina Bloomstine, Casimiro Ferreira, Andrii Pernatii, Kirill Hrymailo +# BSD-3 License +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# 1. Redistributions of source code must retain the above copyright notice, +# this list of conditions and the following disclaimer. +# 2. Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. +# 3. Neither the name of the copyright holder nor the names of its +# contributors may be used to endorse or promote products derived from this +# software without specific prior written permission. +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, +# THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR +# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, +# EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, +# OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF +# LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING +# NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +# SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +import setuptools + +from os import path + + +BASE_PATH = path.abspath(path.dirname(__file__)) + + +def get_requirements(requirements_filename: str): + requirements_file = path.join(BASE_PATH, "requirements", + requirements_filename) + with open(requirements_file, 'r', encoding='utf-8') as r: + requirements = r.readlines() + requirements = [r.strip() for r in requirements if r.strip() and + not r.strip().startswith("#")] + return requirements + + +with open(path.join(BASE_PATH, "README.md"), "r") as f: + long_description = f.read() + +with open(path.join(BASE_PATH, "neon_minerva", "version.py"), + "r", encoding="utf-8") as v: + for line in v.readlines(): + if line.startswith("__version__"): + if '"' in line: + version = line.split('"')[1] + else: + version = line.split("'")[1] + +setuptools.setup( + name="neon-minerva", + version=version, + author='Neongecko', + author_email='developers@neon.ai', + license='BSD-3-Clause', + description="Modular INtelligent Evaluation for a Reliable Voice Assistant", + long_description=long_description, + long_description_content_type="text/markdown", + url="https://github.com/neongeckocom/neon-minerva", + packages=setuptools.find_packages(), + include_package_data=True, + classifiers=[ + "Programming Language :: Python :: 3", + "Operating System :: OS Independent" + ], + python_requires='>=3.6', + install_requires=get_requirements("requirements.txt"), + entry_points={ + 'console_scripts': ['minerva=neon_minerva.cli:neon_minerva_cli'] + } +)