diff --git a/mycroft/skills/intent_services/adapt_service.py b/mycroft/skills/intent_services/adapt_service.py index ebac8cafaa08..b9426d3007c4 100644 --- a/mycroft/skills/intent_services/adapt_service.py +++ b/mycroft/skills/intent_services/adapt_service.py @@ -16,5 +16,279 @@ from adapt.context import ContextManagerFrame from adapt.engine import IntentDeterminationEngine from ovos_utils.intents import AdaptIntent, IntentBuilder, Intent -from ovos_core.intent_services.adapt_service import ContextManager, AdaptService -from ovos_utils.intents import AdaptIntent, IntentBuilder, Intent +from ovos_config.config import Configuration +from ovos_bus_client.message import Message, dig_for_message +from ovos_bus_client.session import IntentContextManager as ContextManager, \ + SessionManager +from ovos_intent_plugin_adapt import AdaptPipelinePlugin +from ovos_utils.log import deprecated, log_deprecation +from ovos_utils.messagebus import get_message_lang, get_mycroft_bus + + +class AdaptService: + """ this class provides methods that were deprecated in 0.0.8 , + it's sole purpose is to log warnings and direct users to use the plugin that replaced this implementation + + Methods here were present in ovos-core 0.0.7 and will be fully removed by ovos-core 0.0.9""" + + def __init__(self, bus=None, config=None): + bus = bus or get_mycroft_bus() # backwards compat, bus was optional + core_config = Configuration() + config = config or core_config.get("context", {}) + self.plugin = AdaptPipelinePlugin(bus, config) + self.plugin.bus.on('intent.service.adapt.get', self.handle_get_adapt) + self.plugin.bus.on('intent.service.adapt.manifest.get', self.handle_adapt_manifest) + self.plugin.bus.on('intent.service.adapt.vocab.manifest.get', self.handle_vocab_manifest) + + # deprecated proxies for plugin methods + def handle_register_vocab(self, message): + """Register adapt vocabulary. + + Args: + message (Message): message containing vocab info + """ + self.plugin._handle_adapt_vocab(message) + + def handle_register_intent(self, message): + """Register adapt intent. + + Args: + message (Message): message containing intent info + """ + self.plugin.handle_register_intent(message) + + def handle_detach_intent(self, message): + """Remover adapt intent. + + Args: + message (Message): message containing intent info + """ + self.plugin.handle_detach_intent(message) + + def handle_detach_skill(self, message): + """Remove all intents registered for a specific skill. + + Args: + message (Message): message containing intent info + """ + self.plugin.handle_detach_skill(message) + + def register_vocab(self, start_concept, end_concept, + alias_of, regex_str, lang): + """Register Vocabulary. DEPRECATED + + This method should not be used, it has been replaced by + register_vocabulary(). + """ + self.register_vocabulary(start_concept, end_concept, alias_of, + regex_str, lang) + + def register_vocabulary(self, entity_value, entity_type, + alias_of, regex_str, lang): + """Register skill vocabulary as adapt entity. + + This will handle both regex registration and registration of normal + keywords. if the "regex_str" argument is set all other arguments will + be ignored. + + Argument: + entity_value: the natural langauge word + entity_type: the type/tag of an entity instance + alias_of: entity this is an alternative for + """ + m = dig_for_message() or Message("") # to dig skill_id in message.context + m = m.forward("register_vocab", {"entity_type": entity_type, + "entity_value": entity_value, + "regex": regex_str, + "alias_of": alias_of, + "lang": lang or self.plugin}) + self.plugin._handle_adapt_vocab(m) + + def register_intent(self, intent): + """Register new intent with adapt engine. + + Args: + intent (IntentParser): IntentParser to register + """ + m = dig_for_message() or Message("") # to dig skill_id in message.context + m = m.forward("register_intent", {"name": intent.name, + "requires": intent.requires, + "optional": intent.optional, + "at_least_one": intent.at_least_one}) + self.plugin.handle_register_keyword_intent(m) + + def detach_skill(self, skill_id): + """Remove all intents for skill. + + Args: + skill_id (str): skill to process + """ + self.plugin.detach_skill(skill_id) + + def detach_intent(self, intent_name): + """Detach a single intent + + Args: + intent_name (str): Identifier for intent to remove. + """ + m = dig_for_message() or Message("") # to dig skill_id in message.context + skill_id = m.data.get("skill_id") or m.context.get("skill_id") + self.plugin.detach_intent(skill_id=skill_id, intent_name=intent_name) + + # deprecated bus handlers + def handle_get_adapt(self, message): + """handler getting the adapt response for an utterance. + + Args: + message (Message): message containing utterance + """ + utterance = message.data["utterance"] + lang = get_message_lang(message) + intent = self.plugin.match_intent([utterance], lang, message) + intent_data = intent.intent_data if intent else None + self.plugin.bus.emit(message.reply("intent.service.adapt.reply", + {"intent": intent_data})) + + def handle_adapt_manifest(self, message): + """Send adapt intent manifest to caller. + + Argument: + message: query message to reply to. + """ + self.plugin.bus.emit(message.reply("intent.service.adapt.manifest", + {"intents": self.registered_intents})) + + def handle_vocab_manifest(self, message): + """Send adapt vocabulary manifest to caller. + + Argument: + message: query message to reply to. + """ + self.plugin.bus.emit(message.reply("intent.service.adapt.vocab.manifest", + {"vocab": self.registered_vocab})) + + # deprecated properties / backwards compat + @property + def registered_vocab(self): + log_deprecation("self.registered_vocab has been deprecated and is unused," + " use the adapt plugin directly instead", "0.0.8") + return [] + + @registered_vocab.setter + def registered_vocab(self, val): + log_deprecation("self.registered_vocab has been deprecated and is unused, " + "use the adapt plugin directly instead", "0.0.8") + + @property + def registered_intents(self): + lang = get_message_lang() + return [parser.__dict__ for parser in self.plugin.engines[lang].intent_parsers] + + @property + def context_keywords(self): + log_deprecation("self.context_keywords has been deprecated and is unused," + " use self.config.get('keywords', []) instead", "0.1.0") + return self.plugin.config.get('keywords', []) + + @context_keywords.setter + def context_keywords(self, val): + log_deprecation("self.context_keywords has been deprecated and is unused, " + "edit mycroft.conf instead, setter will be ignored", "0.1.0") + + @property + def context_max_frames(self): + log_deprecation("self.context_keywords has been deprecated and is unused, " + "use self.config.get('max_frames', 3) instead", "0.1.0") + return self.plugin.config.get('max_frames', 3) + + @context_max_frames.setter + def context_max_frames(self, val): + log_deprecation("self.context_max_frames has been deprecated and is unused, " + "edit mycroft.conf instead, setter will be ignored", "0.1.0") + + @property + def context_timeout(self): + log_deprecation("self.context_timeout has been deprecated and is unused," + " use self.config.get('timeout', 2) instead", "0.1.0") + return self.plugin.config.get('timeout', 2) + + @context_timeout.setter + def context_timeout(self, val): + log_deprecation("self.context_timeout has been deprecated and is unused," + " edit mycroft.conf instead, setter will be ignored", "0.1.0") + + @property + def context_greedy(self): + log_deprecation("self.context_greedy has been deprecated and is unused, " + "use self.config.get('greedy', False) instead", "0.1.0") + return self.plugin.config.get('greedy', False) + + @context_greedy.setter + def context_greedy(self, val): + log_deprecation("self.context_greedy has been deprecated and is unused," + " edit mycroft.conf instead, setter will be ignored", "0.1.0") + + @property + def context_manager(self): + log_deprecation("context_manager has been deprecated, use Session.context instead", "0.1.0") + sess = SessionManager.get() + return sess.context + + @context_manager.setter + def context_manager(self, val): + log_deprecation("context_manager has been deprecated, use Session.context instead", "0.1.0") + + assert isinstance(val, ContextManager) + sess = SessionManager.get() + sess.context = val + + @deprecated("update_context has been deprecated, use Session.context.update_context instead", "0.1.0") + def update_context(self, intent): + """Updates context with keyword from the intent. + + NOTE: This method currently won't handle one_of intent keywords + since it's not using quite the same format as other intent + keywords. This is under investigation in adapt, PR pending. + + Args: + intent: Intent to scan for keywords + """ + sess = SessionManager.get() + ents = [tag['entities'][0] for tag in intent['__tags__'] if 'entities' in tag] + sess.context.update_context(ents) + + +def _entity_skill_id(skill_id): + """Helper converting a skill id to the format used in entities. + + Arguments: + skill_id (str): skill identifier + + Returns: + (str) skill id on the format used by skill entities + """ + skill_id = skill_id[:-1] + skill_id = skill_id.replace('.', '_') + skill_id = skill_id.replace('-', '_') + return skill_id + + +def _is_old_style_keyword_message(message): + """Simple check that the message is not using the updated format. + TODO: Remove in v22.02 + Args: + message (Message): Message object to check + Returns: + (bool) True if this is an old messagem, else False + """ + return ('entity_value' not in message.data and 'start' in message.data) + + +def _update_keyword_message(message): + """Make old style keyword registration message compatible. + Copies old keys in message data to new names. + Args: + message (Message): Message to update + """ + message.data['entity_value'] = message.data['start'] + message.data['entity_type'] = message.data['end'] diff --git a/mycroft/skills/intent_services/padatious_service.py b/mycroft/skills/intent_services/padatious_service.py index b5d1809a4312..aa40705f3c6a 100644 --- a/mycroft/skills/intent_services/padatious_service.py +++ b/mycroft/skills/intent_services/padatious_service.py @@ -13,5 +13,352 @@ # limitations under the License. # """Intent service wrapping padatious.""" -from ovos_core.intent_services.padatious_service import PadatiousMatcher, PadatiousService, PadatiousIntent +from functools import lru_cache +from os import path +from os.path import expanduser, isfile +from threading import Event +from time import time as get_time, sleep +from typing import List, Optional +import padatious +from ovos_config.config import Configuration +from ovos_config.meta import get_xdg_base +from padatious.match_data import MatchData as PadatiousIntent + +from ovos_bus_client.message import Message +from ovos_plugin_manager.templates.pipeline import IntentPipelinePlugin, IntentMatch +from ovos_utils import flatten_list, classproperty +from ovos_utils.log import LOG, deprecated +from ovos_utils.xdg_utils import xdg_data_home + + +class PadatiousMatcher: + """Matcher class to avoid redundancy in padatious intent matching.""" + + @deprecated("PadatiousMatcher class is deprecated, use PadatiousService directly", "0.1.0") + def __init__(self, service): + self.service = service + + def match_high(self, utterances, lang=None, message=None): + """Intent matcher for high confidence. + + Args: + utterances (list of tuples): Utterances to parse, originals paired + with optional normalized version. + """ + return self.service.match_high(utterances, lang, message) + + def match_medium(self, utterances, lang=None, message=None): + """Intent matcher for medium confidence. + + Args: + utterances (list of tuples): Utterances to parse, originals paired + with optional normalized version. + """ + return self.service.match_medium(utterances, lang, message) + + def match_low(self, utterances, lang=None, message=None): + """Intent matcher for low confidence. + + Args: + utterances (list of tuples): Utterances to parse, originals paired + with optional normalized version. + """ + return self.service.match_low(utterances, lang, message) + + +# TODO - scrap the whole thing, import the padatious/padacioso plugins directly +class PadatiousService(IntentPipelinePlugin): + """Service class for padatious intent matching.""" + + def __init__(self, bus, config): + super().__init__(bus, config) + self.padatious_config = config + self.bus = bus + + core_config = Configuration() + lang = core_config.get("lang", "en-us") + langs = core_config.get('secondary_langs') or [] + if lang not in langs: + langs.append(lang) + + self.conf_high = self.padatious_config.get("conf_high") or 0.95 + self.conf_med = self.padatious_config.get("conf_med") or 0.8 + self.conf_low = self.padatious_config.get("conf_low") or 0.5 + + LOG.debug('Using Padatious intent parser.') + intent_cache = self.padatious_config.get( + 'intent_cache') or f"{xdg_data_home()}/{get_xdg_base()}/intent_cache" + self.containers = { + lang: padatious.IntentContainer(path.join(expanduser(intent_cache), lang)) + for lang in langs} + + self.finished_training_event = Event() + self.finished_initial_train = False + + self.train_delay = self.padatious_config.get('train_delay', 4) + self.train_time = get_time() + self.train_delay + + self.registered_intents = [] + self.registered_entities = [] + + # plugin api + @classproperty + def matcher_id(self): + return "padatious" + + def register_bus_events(self): + self.bus.on('padatious:register_intent', self.register_intent) + self.bus.on('padatious:register_entity', self.register_entity) + self.bus.on('detach_intent', self.handle_detach_intent) + self.bus.on('detach_skill', self.handle_detach_skill) + self.bus.on('mycroft.skills.initialized', self.train) + + self.bus.on('intent.service.padatious.get', self.handle_get_padatious) + self.bus.on('intent.service.padatious.manifest.get', self.handle_padatious_manifest) + self.bus.on('intent.service.padatious.entities.manifest.get', self.handle_entity_manifest) + + def _match_level(self, utterances, limit, lang=None): + """Match intent and make sure a certain level of confidence is reached. + + Args: + utterances (list of tuples): Utterances to parse, originals paired + with optional normalized version. + limit (float): required confidence level. + """ + LOG.debug(f'Padatious Matching confidence > {limit}') + # call flatten in case someone is sending the old style list of tuples + utterances = flatten_list(utterances) + lang = lang or self.lang + padatious_intent = self.calc_intent(utterances, lang) + if padatious_intent is not None and padatious_intent.conf > limit: + skill_id = padatious_intent.name.split(':')[0] + return IntentMatch(intent_service=self.matcher_id, + intent_type=padatious_intent.name, + intent_data=padatious_intent.matches, + skill_id=skill_id, + utterance=padatious_intent.sent, + confidence=padatious_intent.conf) + + def match_high(self, utterances, lang=None, message=None): + """Intent matcher for high confidence. + + Args: + utterances (list of tuples): Utterances to parse, originals paired + with optional normalized version. + """ + return self._match_level(utterances, self.conf_high, lang) + + def match_medium(self, utterances, lang=None, message=None): + """Intent matcher for medium confidence. + + Args: + utterances (list of tuples): Utterances to parse, originals paired + with optional normalized version. + """ + return self._match_level(utterances, self.conf_med, lang) + + def match_low(self, utterances, lang=None, message=None): + """Intent matcher for low confidence. + + Args: + utterances (list of tuples): Utterances to parse, originals paired + with optional normalized version. + """ + return self._match_level(utterances, self.conf_low, lang) + + # implementation + def handle_get_padatious(self, message): + """messagebus handler for perfoming padatious parsing. + + Args: + message (Message): message triggering the method + """ + utterance = message.data["utterance"] + norm = message.data.get('norm_utt', utterance) + intent = self.calc_intent(utterance) + if not intent and norm != utterance: + intent = self.calc_intent(norm) + if intent: + intent = intent.__dict__ + self.bus.emit(message.reply("intent.service.padatious.reply", + {"intent": intent})) + + def handle_padatious_manifest(self, message): + """Messagebus handler returning the registered padatious intents. + + Args: + message (Message): message triggering the method + """ + self.bus.emit(message.reply( + "intent.service.padatious.manifest", + {"intents": self.registered_intents})) + + def handle_entity_manifest(self, message): + """Messagebus handler returning the registered padatious entities. + + Args: + message (Message): message triggering the method + """ + self.bus.emit(message.reply( + "intent.service.padatious.entities.manifest", + {"entities": self.registered_entities})) + + def train(self, message=None): + """Perform padatious training. + + Args: + message (Message): optional triggering message + """ + self.finished_training_event.clear() + padatious_single_thread = self.padatious_config.get('single_thread', True) + if message is None: + single_thread = padatious_single_thread + else: + single_thread = message.data.get('single_thread', + padatious_single_thread) + for lang in self.containers: + self.containers[lang].train(single_thread=single_thread) + + LOG.info('Training complete.') + self.finished_training_event.set() + if not self.finished_initial_train: + self.bus.emit(Message('mycroft.skills.trained')) + self.finished_initial_train = True + + def wait_and_train(self): + """Wait for minimum time between training and start training.""" + if not self.finished_initial_train: + return + sleep(self.train_delay) + if self.train_time < 0.0: + return + + if self.train_time <= get_time() + 0.01: + self.train_time = -1.0 + self.train() + + def __detach_intent(self, intent_name): + """ Remove an intent if it has been registered. + + Args: + intent_name (str): intent identifier + """ + if intent_name in self.registered_intents: + self.registered_intents.remove(intent_name) + for lang in self.containers: + self.containers[lang].remove_intent(intent_name) + + def handle_detach_intent(self, message): + """Messagebus handler for detaching padatious intent. + + Args: + message (Message): message triggering action + """ + self.__detach_intent(message.data.get('intent_name')) + + def handle_detach_skill(self, message): + """Messagebus handler for detaching all intents for skill. + + Args: + message (Message): message triggering action + """ + skill_id = message.data['skill_id'] + remove_list = [i for i in self.registered_intents if skill_id in i] + for i in remove_list: + self.__detach_intent(i) + + def _register_object(self, message, object_name, register_func): + """Generic method for registering a padatious object. + + Args: + message (Message): trigger for action + object_name (str): type of entry to register + register_func (callable): function to call for registration + """ + file_name = message.data.get('file_name') + samples = message.data.get("samples") + name = message.data['name'] + + LOG.debug('Registering Padatious ' + object_name + ': ' + name) + + if (not file_name or not isfile(file_name)) and not samples: + LOG.error('Could not find file ' + file_name) + return + + if not samples and isfile(file_name): + with open(file_name) as f: + samples = [l.strip() for l in f.readlines()] + + register_func(name, samples) + + self.train_time = get_time() + self.train_delay + self.wait_and_train() + + def register_intent(self, message): + """Messagebus handler for registering intents. + + Args: + message (Message): message triggering action + """ + lang = message.data.get('lang', self.lang) + lang = lang.lower() + if lang in self.containers: + self.registered_intents.append(message.data['name']) + try: + self._register_object(message, 'intent', + self.containers[lang].add_intent) + except RuntimeError: + name = message.data.get('name', "") + # padacioso fails on reloading a skill, just ignore + if name not in self.containers[lang].intent_samples: + raise + + def register_entity(self, message): + """Messagebus handler for registering entities. + + Args: + message (Message): message triggering action + """ + lang = message.data.get('lang', self.lang) + lang = lang.lower() + if lang in self.containers: + self.registered_entities.append(message.data) + self._register_object(message, 'entity', + self.containers[lang].add_entity) + + def calc_intent(self, utterances: List[str], lang: str = None) -> Optional[PadatiousIntent]: + """ + Get the best intent match for the given list of utterances. Utilizes a + thread pool for overall faster execution. Note that this method is NOT + compatible with Padatious, but is compatible with Padacioso. + @param utterances: list of string utterances to get an intent for + @param lang: language of utterances + @return: + """ + if isinstance(utterances, str): + utterances = [utterances] # backwards compat when arg was a single string + lang = lang or self.lang + lang = lang.lower() + if lang in self.containers: + intent_container = self.containers.get(lang) + intents = [_calc_padatious_intent(utt, intent_container) for utt in utterances] + intents = [i for i in intents if i is not None] + # select best + if intents: + return max(intents, key=lambda k: k.conf) + + +@lru_cache(maxsize=3) # repeat calls under different conf levels wont re-run code +def _calc_padatious_intent(utt, intent_container) -> Optional[PadatiousIntent]: + """ + Try to match an utterance to an intent in an intent_container + @param args: tuple of (utterance, IntentContainer) + @return: matched PadatiousIntent + """ + try: + intent = intent_container.calc_intent(utt) + intent.sent = utt + return intent + except Exception as e: + LOG.error(e) diff --git a/ovos_core/intent_services/adapt_service.py b/ovos_core/intent_services/adapt_service.py index 631efd06f371..e291b1f2268e 100644 --- a/ovos_core/intent_services/adapt_service.py +++ b/ovos_core/intent_services/adapt_service.py @@ -1,296 +1,2 @@ -# Copyright 2020 Mycroft AI Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -"""An intent parsing service using the Adapt parser.""" - -from ovos_config.config import Configuration - -from ovos_bus_client.message import Message, dig_for_message -from ovos_bus_client.session import IntentContextManager as ContextManager, \ - SessionManager -from ovos_intent_plugin_adapt import AdaptPipelinePlugin -from ovos_utils.log import deprecated, log_deprecation -from ovos_utils.messagebus import get_message_lang, get_mycroft_bus - - -# TODO - deprecate whole module, move to mycroft.deprecated -# ovos_core submodule was only introduced in 0.0.8 alpha, no need to keep a reference in this file - -class AdaptService: - """ this class provides methods that were deprecated in 0.0.8 , - it's sole purpose is to log warnings and direct users to use the plugin that replaced this implementation - - Methods here were present in ovos-core 0.0.7 and will be fully removed by ovos-core 0.0.9""" - - def __init__(self, bus=None, config=None): - bus = bus or get_mycroft_bus() # backwards compat, bus was optional - core_config = Configuration() - config = config or core_config.get("context", {}) - self.plugin = AdaptPipelinePlugin(bus, config) - self.plugin.bus.on('intent.service.adapt.get', self.handle_get_adapt) - self.plugin.bus.on('intent.service.adapt.manifest.get', self.handle_adapt_manifest) - self.plugin.bus.on('intent.service.adapt.vocab.manifest.get', self.handle_vocab_manifest) - - # deprecated proxies for plugin methods - def handle_register_vocab(self, message): - """Register adapt vocabulary. - - Args: - message (Message): message containing vocab info - """ - self.plugin._handle_adapt_vocab(message) - - def handle_register_intent(self, message): - """Register adapt intent. - - Args: - message (Message): message containing intent info - """ - self.plugin.handle_register_intent(message) - - def handle_detach_intent(self, message): - """Remover adapt intent. - - Args: - message (Message): message containing intent info - """ - self.plugin.handle_detach_intent(message) - - def handle_detach_skill(self, message): - """Remove all intents registered for a specific skill. - - Args: - message (Message): message containing intent info - """ - self.plugin.handle_detach_skill(message) - - def register_vocab(self, start_concept, end_concept, - alias_of, regex_str, lang): - """Register Vocabulary. DEPRECATED - - This method should not be used, it has been replaced by - register_vocabulary(). - """ - self.register_vocabulary(start_concept, end_concept, alias_of, - regex_str, lang) - - def register_vocabulary(self, entity_value, entity_type, - alias_of, regex_str, lang): - """Register skill vocabulary as adapt entity. - - This will handle both regex registration and registration of normal - keywords. if the "regex_str" argument is set all other arguments will - be ignored. - - Argument: - entity_value: the natural langauge word - entity_type: the type/tag of an entity instance - alias_of: entity this is an alternative for - """ - m = dig_for_message() or Message("") # to dig skill_id in message.context - m = m.forward("register_vocab", {"entity_type": entity_type, - "entity_value": entity_value, - "regex": regex_str, - "alias_of": alias_of, - "lang": lang or self.plugin}) - self.plugin._handle_adapt_vocab(m) - - def register_intent(self, intent): - """Register new intent with adapt engine. - - Args: - intent (IntentParser): IntentParser to register - """ - m = dig_for_message() or Message("") # to dig skill_id in message.context - m = m.forward("register_intent", {"name": intent.name, - "requires": intent.requires, - "optional": intent.optional, - "at_least_one": intent.at_least_one}) - self.plugin.handle_register_keyword_intent(m) - - def detach_skill(self, skill_id): - """Remove all intents for skill. - - Args: - skill_id (str): skill to process - """ - self.plugin.detach_skill(skill_id) - - def detach_intent(self, intent_name): - """Detach a single intent - - Args: - intent_name (str): Identifier for intent to remove. - """ - m = dig_for_message() or Message("") # to dig skill_id in message.context - skill_id = m.data.get("skill_id") or m.context.get("skill_id") - self.plugin.detach_intent(skill_id=skill_id, intent_name=intent_name) - - # deprecated bus handlers - def handle_get_adapt(self, message): - """handler getting the adapt response for an utterance. - - Args: - message (Message): message containing utterance - """ - utterance = message.data["utterance"] - lang = get_message_lang(message) - intent = self.plugin.match_intent([utterance], lang, message) - intent_data = intent.intent_data if intent else None - self.plugin.bus.emit(message.reply("intent.service.adapt.reply", - {"intent": intent_data})) - - def handle_adapt_manifest(self, message): - """Send adapt intent manifest to caller. - - Argument: - message: query message to reply to. - """ - self.plugin.bus.emit(message.reply("intent.service.adapt.manifest", - {"intents": self.registered_intents})) - - def handle_vocab_manifest(self, message): - """Send adapt vocabulary manifest to caller. - - Argument: - message: query message to reply to. - """ - self.plugin.bus.emit(message.reply("intent.service.adapt.vocab.manifest", - {"vocab": self.registered_vocab})) - - # deprecated properties / backwards compat - @property - def registered_vocab(self): - log_deprecation("self.registered_vocab has been deprecated and is unused," - " use the adapt plugin directly instead", "0.0.8") - return [] - - @registered_vocab.setter - def registered_vocab(self, val): - log_deprecation("self.registered_vocab has been deprecated and is unused, " - "use the adapt plugin directly instead", "0.0.8") - - @property - def registered_intents(self): - lang = get_message_lang() - return [parser.__dict__ for parser in self.plugin.engines[lang].intent_parsers] - - @property - def context_keywords(self): - log_deprecation("self.context_keywords has been deprecated and is unused," - " use self.config.get('keywords', []) instead", "0.1.0") - return self.plugin.config.get('keywords', []) - - @context_keywords.setter - def context_keywords(self, val): - log_deprecation("self.context_keywords has been deprecated and is unused, " - "edit mycroft.conf instead, setter will be ignored", "0.1.0") - - @property - def context_max_frames(self): - log_deprecation("self.context_keywords has been deprecated and is unused, " - "use self.config.get('max_frames', 3) instead", "0.1.0") - return self.plugin.config.get('max_frames', 3) - - @context_max_frames.setter - def context_max_frames(self, val): - log_deprecation("self.context_max_frames has been deprecated and is unused, " - "edit mycroft.conf instead, setter will be ignored", "0.1.0") - - @property - def context_timeout(self): - log_deprecation("self.context_timeout has been deprecated and is unused," - " use self.config.get('timeout', 2) instead", "0.1.0") - return self.plugin.config.get('timeout', 2) - - @context_timeout.setter - def context_timeout(self, val): - log_deprecation("self.context_timeout has been deprecated and is unused," - " edit mycroft.conf instead, setter will be ignored", "0.1.0") - - @property - def context_greedy(self): - log_deprecation("self.context_greedy has been deprecated and is unused, " - "use self.config.get('greedy', False) instead", "0.1.0") - return self.plugin.config.get('greedy', False) - - @context_greedy.setter - def context_greedy(self, val): - log_deprecation("self.context_greedy has been deprecated and is unused," - " edit mycroft.conf instead, setter will be ignored", "0.1.0") - - @property - def context_manager(self): - log_deprecation("context_manager has been deprecated, use Session.context instead", "0.1.0") - sess = SessionManager.get() - return sess.context - - @context_manager.setter - def context_manager(self, val): - log_deprecation("context_manager has been deprecated, use Session.context instead", "0.1.0") - - assert isinstance(val, ContextManager) - sess = SessionManager.get() - sess.context = val - - @deprecated("update_context has been deprecated, use Session.context.update_context instead", "0.1.0") - def update_context(self, intent): - """Updates context with keyword from the intent. - - NOTE: This method currently won't handle one_of intent keywords - since it's not using quite the same format as other intent - keywords. This is under investigation in adapt, PR pending. - - Args: - intent: Intent to scan for keywords - """ - sess = SessionManager.get() - ents = [tag['entities'][0] for tag in intent['__tags__'] if 'entities' in tag] - sess.context.update_context(ents) - - -def _entity_skill_id(skill_id): - """Helper converting a skill id to the format used in entities. - - Arguments: - skill_id (str): skill identifier - - Returns: - (str) skill id on the format used by skill entities - """ - skill_id = skill_id[:-1] - skill_id = skill_id.replace('.', '_') - skill_id = skill_id.replace('-', '_') - return skill_id - - -def _is_old_style_keyword_message(message): - """Simple check that the message is not using the updated format. - TODO: Remove in v22.02 - Args: - message (Message): Message object to check - Returns: - (bool) True if this is an old messagem, else False - """ - return ('entity_value' not in message.data and 'start' in message.data) - - -def _update_keyword_message(message): - """Make old style keyword registration message compatible. - Copies old keys in message data to new names. - Args: - message (Message): Message to update - """ - message.data['entity_value'] = message.data['start'] - message.data['entity_type'] = message.data['end'] +# deprecated file, TODO remove in 0.0.8 +from mycroft.skills.intent_services.adapt_service import AdaptService diff --git a/ovos_core/intent_services/padatious_service.py b/ovos_core/intent_services/padatious_service.py index 1ff4f0a994d1..4b4094bc9ce5 100644 --- a/ovos_core/intent_services/padatious_service.py +++ b/ovos_core/intent_services/padatious_service.py @@ -1,369 +1,3 @@ -# Copyright 2020 Mycroft AI Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -"""Intent service wrapping padatious.""" -from functools import lru_cache -from os import path -from os.path import expanduser, isfile -from threading import Event -from time import time as get_time, sleep -from typing import List, Optional +# deprecated file, TODO remove in 0.0.8 +from mycroft.skills.intent_services.padatious_service import PadatiousService, PadatiousMatcher, PadatiousIntent -import padatious -from ovos_config.config import Configuration -from ovos_config.meta import get_xdg_base -from padatious.match_data import MatchData as PadatiousIntent - -from ovos_bus_client.message import Message -from ovos_plugin_manager.templates.pipeline import IntentPipelinePlugin, IntentMatch -from ovos_utils import flatten_list, classproperty -from ovos_utils.log import LOG, deprecated -from ovos_utils.xdg_utils import xdg_data_home - - -# TODO - deprecate whole module, move to mycroft.deprecated -# ovos_core submodule was only introduced in 0.0.8 alpha, no need to keep a reference in this file - - -class PadatiousMatcher: - """Matcher class to avoid redundancy in padatious intent matching.""" - - @deprecated("PadatiousMatcher class is deprecated, use PadatiousService directly", "0.1.0") - def __init__(self, service): - self.service = service - - def match_high(self, utterances, lang=None, message=None): - """Intent matcher for high confidence. - - Args: - utterances (list of tuples): Utterances to parse, originals paired - with optional normalized version. - """ - return self.service.match_high(utterances, lang, message) - - def match_medium(self, utterances, lang=None, message=None): - """Intent matcher for medium confidence. - - Args: - utterances (list of tuples): Utterances to parse, originals paired - with optional normalized version. - """ - return self.service.match_medium(utterances, lang, message) - - def match_low(self, utterances, lang=None, message=None): - """Intent matcher for low confidence. - - Args: - utterances (list of tuples): Utterances to parse, originals paired - with optional normalized version. - """ - return self.service.match_low(utterances, lang, message) - - - -# TODO - scrap the whole thing, import the padatious/padacioso plugins directly -class PadatiousService(IntentPipelinePlugin): - """Service class for padatious intent matching.""" - - def __init__(self, bus, config): - super().__init__(bus, config) - self.padatious_config = config - self.bus = bus - - core_config = Configuration() - lang = core_config.get("lang", "en-us") - langs = core_config.get('secondary_langs') or [] - if lang not in langs: - langs.append(lang) - - self.conf_high = self.padatious_config.get("conf_high") or 0.95 - self.conf_med = self.padatious_config.get("conf_med") or 0.8 - self.conf_low = self.padatious_config.get("conf_low") or 0.5 - - LOG.debug('Using Padatious intent parser.') - intent_cache = self.padatious_config.get( - 'intent_cache') or f"{xdg_data_home()}/{get_xdg_base()}/intent_cache" - self.containers = { - lang: padatious.IntentContainer(path.join(expanduser(intent_cache), lang)) - for lang in langs} - - self.finished_training_event = Event() - self.finished_initial_train = False - - self.train_delay = self.padatious_config.get('train_delay', 4) - self.train_time = get_time() + self.train_delay - - self.registered_intents = [] - self.registered_entities = [] - - # plugin api - @classproperty - def matcher_id(self): - return "padatious" - - def register_bus_events(self): - self.bus.on('padatious:register_intent', self.register_intent) - self.bus.on('padatious:register_entity', self.register_entity) - self.bus.on('detach_intent', self.handle_detach_intent) - self.bus.on('detach_skill', self.handle_detach_skill) - self.bus.on('mycroft.skills.initialized', self.train) - - self.bus.on('intent.service.padatious.get', self.handle_get_padatious) - self.bus.on('intent.service.padatious.manifest.get', self.handle_padatious_manifest) - self.bus.on('intent.service.padatious.entities.manifest.get', self.handle_entity_manifest) - - def _match_level(self, utterances, limit, lang=None): - """Match intent and make sure a certain level of confidence is reached. - - Args: - utterances (list of tuples): Utterances to parse, originals paired - with optional normalized version. - limit (float): required confidence level. - """ - LOG.debug(f'Padatious Matching confidence > {limit}') - # call flatten in case someone is sending the old style list of tuples - utterances = flatten_list(utterances) - lang = lang or self.lang - padatious_intent = self.calc_intent(utterances, lang) - if padatious_intent is not None and padatious_intent.conf > limit: - skill_id = padatious_intent.name.split(':')[0] - return IntentMatch(intent_service=self.matcher_id, - intent_type=padatious_intent.name, - intent_data=padatious_intent.matches, - skill_id=skill_id, - utterance=padatious_intent.sent, - confidence=padatious_intent.conf) - - def match_high(self, utterances, lang=None, message=None): - """Intent matcher for high confidence. - - Args: - utterances (list of tuples): Utterances to parse, originals paired - with optional normalized version. - """ - return self._match_level(utterances, self.conf_high, lang) - - def match_medium(self, utterances, lang=None, message=None): - """Intent matcher for medium confidence. - - Args: - utterances (list of tuples): Utterances to parse, originals paired - with optional normalized version. - """ - return self._match_level(utterances, self.conf_med, lang) - - def match_low(self, utterances, lang=None, message=None): - """Intent matcher for low confidence. - - Args: - utterances (list of tuples): Utterances to parse, originals paired - with optional normalized version. - """ - return self._match_level(utterances, self.conf_low, lang) - - # implementation - def handle_get_padatious(self, message): - """messagebus handler for perfoming padatious parsing. - - Args: - message (Message): message triggering the method - """ - utterance = message.data["utterance"] - norm = message.data.get('norm_utt', utterance) - intent = self.calc_intent(utterance) - if not intent and norm != utterance: - intent = self.calc_intent(norm) - if intent: - intent = intent.__dict__ - self.bus.emit(message.reply("intent.service.padatious.reply", - {"intent": intent})) - - def handle_padatious_manifest(self, message): - """Messagebus handler returning the registered padatious intents. - - Args: - message (Message): message triggering the method - """ - self.bus.emit(message.reply( - "intent.service.padatious.manifest", - {"intents": self.registered_intents})) - - def handle_entity_manifest(self, message): - """Messagebus handler returning the registered padatious entities. - - Args: - message (Message): message triggering the method - """ - self.bus.emit(message.reply( - "intent.service.padatious.entities.manifest", - {"entities": self.registered_entities})) - - def train(self, message=None): - """Perform padatious training. - - Args: - message (Message): optional triggering message - """ - self.finished_training_event.clear() - padatious_single_thread = self.padatious_config.get('single_thread', True) - if message is None: - single_thread = padatious_single_thread - else: - single_thread = message.data.get('single_thread', - padatious_single_thread) - for lang in self.containers: - self.containers[lang].train(single_thread=single_thread) - - LOG.info('Training complete.') - self.finished_training_event.set() - if not self.finished_initial_train: - self.bus.emit(Message('mycroft.skills.trained')) - self.finished_initial_train = True - - def wait_and_train(self): - """Wait for minimum time between training and start training.""" - if not self.finished_initial_train: - return - sleep(self.train_delay) - if self.train_time < 0.0: - return - - if self.train_time <= get_time() + 0.01: - self.train_time = -1.0 - self.train() - - def __detach_intent(self, intent_name): - """ Remove an intent if it has been registered. - - Args: - intent_name (str): intent identifier - """ - if intent_name in self.registered_intents: - self.registered_intents.remove(intent_name) - for lang in self.containers: - self.containers[lang].remove_intent(intent_name) - - def handle_detach_intent(self, message): - """Messagebus handler for detaching padatious intent. - - Args: - message (Message): message triggering action - """ - self.__detach_intent(message.data.get('intent_name')) - - def handle_detach_skill(self, message): - """Messagebus handler for detaching all intents for skill. - - Args: - message (Message): message triggering action - """ - skill_id = message.data['skill_id'] - remove_list = [i for i in self.registered_intents if skill_id in i] - for i in remove_list: - self.__detach_intent(i) - - def _register_object(self, message, object_name, register_func): - """Generic method for registering a padatious object. - - Args: - message (Message): trigger for action - object_name (str): type of entry to register - register_func (callable): function to call for registration - """ - file_name = message.data.get('file_name') - samples = message.data.get("samples") - name = message.data['name'] - - LOG.debug('Registering Padatious ' + object_name + ': ' + name) - - if (not file_name or not isfile(file_name)) and not samples: - LOG.error('Could not find file ' + file_name) - return - - if not samples and isfile(file_name): - with open(file_name) as f: - samples = [l.strip() for l in f.readlines()] - - register_func(name, samples) - - self.train_time = get_time() + self.train_delay - self.wait_and_train() - - def register_intent(self, message): - """Messagebus handler for registering intents. - - Args: - message (Message): message triggering action - """ - lang = message.data.get('lang', self.lang) - lang = lang.lower() - if lang in self.containers: - self.registered_intents.append(message.data['name']) - try: - self._register_object(message, 'intent', - self.containers[lang].add_intent) - except RuntimeError: - name = message.data.get('name', "") - # padacioso fails on reloading a skill, just ignore - if name not in self.containers[lang].intent_samples: - raise - - def register_entity(self, message): - """Messagebus handler for registering entities. - - Args: - message (Message): message triggering action - """ - lang = message.data.get('lang', self.lang) - lang = lang.lower() - if lang in self.containers: - self.registered_entities.append(message.data) - self._register_object(message, 'entity', - self.containers[lang].add_entity) - - def calc_intent(self, utterances: List[str], lang: str = None) -> Optional[PadatiousIntent]: - """ - Get the best intent match for the given list of utterances. Utilizes a - thread pool for overall faster execution. Note that this method is NOT - compatible with Padatious, but is compatible with Padacioso. - @param utterances: list of string utterances to get an intent for - @param lang: language of utterances - @return: - """ - if isinstance(utterances, str): - utterances = [utterances] # backwards compat when arg was a single string - lang = lang or self.lang - lang = lang.lower() - if lang in self.containers: - intent_container = self.containers.get(lang) - intents = [_calc_padatious_intent(utt, intent_container) for utt in utterances] - intents = [i for i in intents if i is not None] - # select best - if intents: - return max(intents, key=lambda k: k.conf) - - -@lru_cache(maxsize=3) # repeat calls under different conf levels wont re-run code -def _calc_padatious_intent(utt, intent_container) -> Optional[PadatiousIntent]: - """ - Try to match an utterance to an intent in an intent_container - @param args: tuple of (utterance, IntentContainer) - @return: matched PadatiousIntent - """ - try: - intent = intent_container.calc_intent(utt) - intent.sent = utt - return intent - except Exception as e: - LOG.error(e)