Merge pull request #9 from OpenVoiceOS/release-0.1.0a1

Release 0.1.0a1

.github/dependabot.yml

@@ -0,0 +1,11 @@
+# To get started with Dependabot version updates, you'll need to specify which
+# package ecosystems to update and where the package manifests are located.
+# Please see the documentation for all configuration options:
+# https://docs.github.com/code-security/dependabot/dependabot-version-updates/configuration-options-for-the-dependabot.yml-file
+
+version: 2
+updates:
+  - package-ecosystem: "pip" # See documentation for possible values
+    directory: "/requirements" # Location of package manifests
+    schedule:
+      interval: "weekly"

.github/workflows/unit_tests.yml

@@ -0,0 +1,66 @@
+name: Run UnitTests
+on:
+  pull_request:
+    branches:
+      - dev
+    paths-ignore:
+      - "ovos_utterance_corrections_transformer/version.py"
+      - "examples/**"
+      - ".github/**"
+      - ".gitignore"
+      - "LICENSE"
+      - "CHANGELOG.md"
+      - "MANIFEST.in"
+      - "README.md"
+      - "scripts/**"
+  push:
+    branches:
+      - master
+    paths-ignore:
+      - "ovos_utterance_corrections_transformer/version.py"
+      - "requirements/**"
+      - "examples/**"
+      - ".github/**"
+      - ".gitignore"
+      - "LICENSE"
+      - "CHANGELOG.md"
+      - "MANIFEST.in"
+      - "README.md"
+      - "scripts/**"
+  workflow_dispatch:
+
+jobs:
+  unit_tests:
+    strategy:
+      max-parallel: 2
+      matrix:
+        python-version: [3.8, 3.9, "3.10", "3.11"]
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - name: Set up python ${{ matrix.python-version }}
+        uses: actions/setup-python@v5
+        with:
+          python-version: ${{ matrix.python-version }}
+          cache: "pip"
+      - name: Install System Dependencies
+        run: |
+          sudo apt-get update
+          sudo apt install python3-dev swig libssl-dev
+          python -m pip install build wheel
+      - name: Install core repo
+        run: |
+          pip install .
+      - name: Install test dependencies
+        run: |
+          pip install -r tests/requirements.txt
+      - name: Run unittests
+        run: |
+          pytest --cov=ovos_utterance_corrections_transformer --cov-report xml tests
+          # NOTE: additional pytest invocations should also add the --cov-append flag
+          #       or they will overwrite previous invocations' coverage reports
+          #       (for an example, see OVOS Skill Manager's workflow)
+      - name: Upload coverage
+        env:
+          CODECOV_TOKEN: ${{secrets.CODECOV_TOKEN}}
+        uses: codecov/codecov-action@v4

CHANGELOG.md

@@ -1,12 +1,12 @@
 # Changelog
 
-## [0.0.2a1](https://github.com/OpenVoiceOS/ovos-utterance-corrections-plugin/tree/0.0.2a1) (2024-09-11)
+## [0.1.0a1](https://github.com/OpenVoiceOS/ovos-utterance-corrections-plugin/tree/0.1.0a1) (2024-12-15)
 
-[Full Changelog](https://github.com/OpenVoiceOS/ovos-utterance-corrections-plugin/compare/0.0.0...0.0.2a1)
+[Full Changelog](https://github.com/OpenVoiceOS/ovos-utterance-corrections-plugin/compare/0.0.2...0.1.0a1)
 
 **Merged pull requests:**
 
-- Update version.py [\#5](https://github.com/OpenVoiceOS/ovos-utterance-corrections-plugin/pull/5) ([JarbasAl](https://github.com/JarbasAl))
+- feat:regex corrections [\#7](https://github.com/OpenVoiceOS/ovos-utterance-corrections-plugin/pull/7) ([JarbasAl](https://github.com/JarbasAl))
 
 
 

README.md

@@ -1,27 +1,106 @@
 # Utterance Corrections plugin
 
-- "secret speech", map some random utterance to something else so you can furtively give orders to your assistant
-- shortcuts, map shorter utterances or slang to utterances you know trigger the correct intent
-- manually correct bad STT transcriptions you experimentally determined to be common for you
+This plugin provides tools to correct or adjust speech-to-text (STT) outputs for better intent matching or improved user experience.
 
-This plugin checks a user defined json for utterance fixes `~/.local/share/mycroft/corrections.json`
+### Key Features:
+1. **"Secret Speech"**: Map random utterances to something else so you can furtively give orders to your assistant.
+2. **Shortcuts**: Map shorter utterances or slang to known utterances that trigger the correct intent.
+3. **Manual STT Fixes**: Correct common STT transcription errors you experimentally determined.
 
-fuzzy matching is used to determine if a utterance matches the transcription
-if >=0.85% similarity then the replacement is returned instead of the original transcription
+---
+
+## 1. Full Utterance Corrections
+
+This plugin checks a user-defined JSON file for **utterance fixes** at `~/.local/share/mycroft/corrections.json`.
+
+**Fuzzy matching** is used to determine if an utterance matches a transcription. If similarity is greater than or equal to **85%**, the replacement is returned instead of the original transcription.
+
+### Example: `corrections.json`
 ```json
 {
     "I hate open source": "I love open source",
     "do the thing": "trigger protocol 404"
 }
 ```
 
-you can also define unconditional replacements at word level `~/.local/share/mycroft/word_corrections.json`
+**Input**:  
+`"I hat open source"`  
+
+**Output**:  
+`"I love open source"`
+
+---
+
+## 2. Word-Level Corrections
+
+You can also define unconditional word-level replacements in `~/.local/share/mycroft/word_corrections.json`.  
+
+This is particularly useful when STT models repeatedly transcribe specific names or words incorrectly.
 
-for example whisper STT often gets artist names wrong, this allows you to correct them
+### Example: `word_corrections.json`
 ```json
 {
     "Jimmy Hendricks": "Jimi Hendrix",
     "Eric Klapptern": "Eric Clapton",
     "Eric Klappton": "Eric Clapton"
 }
-```
+```
+
+**Input**:  
+`"I love Jimmy Hendricks"`  
+
+**Output**:  
+`"I love Jimi Hendrix"`
+
+
+> **use case**: whisper STT often does this mistake in it's transcriptions
+
+---
+
+## 3. Regex-Based Corrections
+
+For more complex corrections, you can use **regular expressions** in `~/.local/share/mycroft/regex_corrections.json`.  
+
+This is useful for fixing consistent patterns in STT errors, such as replacing incorrect trigraphs.
+
+### Example: `regex_corrections.json`
+```json
+{
+    "\\bsh(\\w*)": "sch\\1"
+}
+```
+
+### Explanation:
+- **`\\bsh(\\w*)`**: Matches words starting with `sh` at a word boundary.
+- **`sch\\1`**: Replaces `sh` with `sch` and appends the rest of the word.
+
+### Example Usage:
+**Input**:  
+`"shalter is a switch"`  
+
+**Output**:  
+`"schalter is a switch"`
+
+> **use case**: citrinet german model often does this mistake in it's transcriptions
+
+---
+
+## Configuration Paths
+
+| File                      | Purpose                               |
+|---------------------------|---------------------------------------|
+| `corrections.json`        | Full utterance replacements.          |
+| `word_corrections.json`   | Word-level replacements.              |
+| `regex_corrections.json`  | Regex-based pattern replacements.     |
+
+All correction files are stored under:  
+`~/.local/share/mycroft/`
+
+---
+
+### Usage Scenarios
+- **Improve Intent Matching**: Ensure consistent STT output for accurate intent triggers.
+- **Fix Model-Specific Errors**: Handle recurring transcription mistakes in certain STT engines.
+- **Shortcut Commands**: Simplify complex commands with shorter phrases or slang.
+
+Let us know how you're using this plugin, and feel free to contribute regex examples to this README or new use cases! 🚀

ovos_utterance_corrections_transformer/__init__.py

@@ -1,10 +1,12 @@
+import regex  # Use the regex module instead of re
 from typing import List, Optional
 
 from json_database import JsonStorage
 from ovos_config.meta import get_xdg_base
 from ovos_plugin_manager.templates.transformers import UtteranceTransformer
-from ovos_utils.parse import match_one
+from ovos_utils.parse import match_one, MatchStrategy
 from ovos_utils.xdg_utils import xdg_data_home
+from ovos_utils.log import LOG
 
 
 class UtteranceCorrectionsPlugin(UtteranceTransformer):
@@ -13,20 +15,49 @@ def __init__(self, name="ovos-utterance-corrections", priority=1):
         super().__init__(name, priority)
         self.db = JsonStorage(path=f"{xdg_data_home()}/{get_xdg_base()}/corrections.json")
         self.words_db = JsonStorage(path=f"{xdg_data_home()}/{get_xdg_base()}/word_corrections.json")
+        self.regex_db = JsonStorage(path=f"{xdg_data_home()}/{get_xdg_base()}/regex_corrections.json")
+        self.match_strategy = MatchStrategy.DAMERAU_LEVENSHTEIN_SIMILARITY  # TODO - from config
 
     def transform(self, utterances: List[str], context: Optional[dict] = None) -> (list, dict):
         context = context or {}
 
-        # replace full utterance
+        # Step 1: Replace full utterance
         if utterances and self.db:
-            replacement, conf = match_one(utterances[0], self.db)  # TODO - match strategy from conf
-            if conf >= 0.85:  # TODO make configurable
+            replacement, conf = match_one(
+                utterances[0], self.db, strategy=self.match_strategy
+            )
+            if conf >= self.config.get("thresh", 0.85):
+                LOG.debug(f"Applying utterance replacement: {utterances[0]} -> {replacement}")
                 return [replacement], context
 
-        # replace individual words
+        # Step 2: Apply regex replacements
+        if utterances and self.regex_db:
+            flags = regex.IGNORECASE if self.config.get("ignore_case", True) else 0
+            for idx in range(len(utterances)):
+                for pattern, replacement in self.regex_db.items():
+                    LOG.debug(f"Applying regex pattern: {pattern}")
+                    try:
+                        # Validate pattern length
+                        if len(pattern) > 1000:
+                            LOG.warning(f"Skipping oversized pattern: {pattern}")
+                            continue
+
+                        # Compile pattern with timeout
+                        compiled_pattern = regex.compile(pattern, flags=flags)
+                        utterances[idx] = compiled_pattern.sub(replacement, utterances[idx])
+
+                    except regex.error as e:
+                        LOG.error(f"Invalid regex pattern: {pattern} -> {e}")
+                    except TimeoutError:
+                        LOG.error(f"Regex pattern timed out: {pattern}")
+
+        # Step 3: Replace individual words
         if utterances and self.words_db:
+            flags = regex.IGNORECASE if self.config.get("ignore_case", True) else 0
             for idx in range(len(utterances)):
                 for w, r in self.words_db.items():
-                    utterances[idx] = utterances[idx].replace(w, r)
+                    LOG.debug(f"Applying word replacement: {w} -> {r}")
+                    # Use regex to ensure replacements are surrounded by word boundaries
+                    utterances[idx] = regex.sub(rf"\b{regex.escape(w)}\b", r, utterances[idx], flags=flags)
 
         return utterances, context

ovos_utterance_corrections_transformer/version.py

@@ -1,6 +1,6 @@
 # START_VERSION_BLOCK
 VERSION_MAJOR = 0
-VERSION_MINOR = 0
-VERSION_BUILD = 2
-VERSION_ALPHA = 0
+VERSION_MINOR = 1
+VERSION_BUILD = 0
+VERSION_ALPHA = 1
 # END_VERSION_BLOCK

requirements.txt

@@ -0,0 +1 @@
+ovos-plugin-manager>=0.0.1,<1.0.0

setup.py

@@ -30,11 +30,21 @@ def get_version():
     return version
 
 
+def required(requirements_file):
+    """ Read requirements file and remove comments and empty lines. """
+    with open(os.path.join(BASEDIR, requirements_file), 'r') as f:
+        requirements = f.read().splitlines()
+        if 'MYCROFT_LOOSE_REQUIREMENTS' in os.environ:
+            print('USING LOOSE REQUIREMENTS!')
+            requirements = [r.replace('==', '>=').replace('~=', '>=') for r in requirements]
+        return [pkg for pkg in requirements
+                if pkg.strip() and not pkg.startswith("#")]
+
+
 UTTERANCE_ENTRY_POINT = (
     'ovos-utterance-corrections-plugin=ovos_utterance_corrections_transformer:UtteranceCorrectionsPlugin'
 )
 
-
 setup(
     name='ovos-utterance-corrections-plugin',
     version=get_version(),
@@ -44,6 +54,7 @@ def get_version():
     license='apache-2.0',
     packages=['ovos_utterance_corrections_transformer'],
     include_package_data=True,
+    install_requires=required("requirements.txt"),
     zip_safe=True,
     classifiers=[
         'Development Status :: 3 - Alpha',
@@ -57,4 +68,4 @@ def get_version():
     entry_points={
         'neon.plugin.text': UTTERANCE_ENTRY_POINT
     }
-)
+)

tests/requirements.txt

@@ -0,0 +1,6 @@
+coveralls==1.8.2
+flake8==3.7.9
+pytest==8.2.2
+pytest-cov==2.8.1
+cov-core==1.15.0
+ovos-plugin-manager