From 7e7cab516febee879ab99f6aebd392e68f95eda8 Mon Sep 17 00:00:00 2001 From: spacemanspiff2007 <10754716+spacemanspiff2007@users.noreply.github.com> Date: Thu, 21 Jan 2021 05:48:20 +0100 Subject: [PATCH] 0.20.0 (#201) --- HABApp/__cmd_args__.py | 96 +++++++ HABApp/__init__.py | 3 +- HABApp/__main__.py | 83 +----- HABApp/__version__.py | 2 +- HABApp/config/config_loader.py | 21 +- HABApp/core/event_bus_listener.py | 40 +-- HABApp/core/events/__init__.py | 1 + HABApp/core/events/event_filters.py | 53 ++++ HABApp/core/events/events.py | 17 +- HABApp/core/files/watcher/base_watcher.py | 7 + HABApp/core/files/watcher/folder_watcher.py | 10 +- HABApp/core/items/base_item.py | 12 +- HABApp/core/items/base_item_watch.py | 6 +- HABApp/core/items/item_aggregation.py | 9 +- HABApp/mqtt/__init__.py | 3 +- HABApp/mqtt/events/__init__.py | 1 + HABApp/mqtt/events/mqtt_filters.py | 10 + HABApp/mqtt/mqtt_connection.py | 6 +- .../connection_handler/http_connection.py | 16 +- HABApp/openhab/connection_logic/connection.py | 10 +- .../plugin_things/items_file.py | 9 +- .../plugin_things/plugin_things.py | 12 +- HABApp/openhab/events/__init__.py | 1 + HABApp/openhab/events/channel_events.py | 4 + HABApp/openhab/events/event_filters.py | 10 + HABApp/openhab/events/item_events.py | 36 ++- HABApp/openhab/events/thing_events.py | 16 ++ HABApp/rule/interfaces/http.py | 5 + HABApp/rule/rule.py | 21 +- HABApp/rule/scheduler/base.py | 4 +- HABApp/rule_manager/benchmark/__init__.py | 1 + HABApp/rule_manager/benchmark/bench_base.py | 72 +++++ HABApp/rule_manager/benchmark/bench_file.py | 30 +++ HABApp/rule_manager/benchmark/bench_habapp.py | 109 ++++++++ HABApp/rule_manager/benchmark/bench_mqtt.py | 110 ++++++++ HABApp/rule_manager/benchmark/bench_oh.py | 170 ++++++++++++ HABApp/rule_manager/benchmark/bench_times.py | 76 ++++++ HABApp/rule_manager/rule_file.py | 17 +- HABApp/rule_manager/rule_manager.py | 15 +- HABApp/runtime/__init__.py | 2 +- HABApp/runtime/runtime.py | 12 +- HABApp/runtime/shutdown.py | 64 +++++ HABApp/runtime/shutdown_helper.py | 40 --- _doc/__style_guide__ | 22 ++ _doc/_plugins/sphinx_execute_code.py | 53 +++- _doc/_static/theme_overrides.css | 12 +- _doc/about_habapp.rst | 7 +- _doc/advanced_usage.rst | 7 +- _doc/class_reference.rst | 12 +- _doc/index.rst | 1 + _doc/installation.rst | 2 +- _doc/interface_openhab.rst | 245 +++++++++++------- _doc/rule.rst | 105 ++++++-- _doc/tips.rst | 49 ++++ conf_testing/rules/bench_rule.py | 79 ------ tests/helpers/__init__.py | 4 +- tests/helpers/mock_file.py | 66 +++++ tests/helpers/module_helpers.py | 38 +++ tests/test_core/test_event_bus.py | 26 +- .../test_events/test_core_filters.py | 53 ++++ tests/test_core/test_item_watch.py | 2 +- tests/test_mqtt/test_mqtt_filters.py | 36 +++ .../test_events/test_oh_filters.py | 37 +++ .../test_plugins/test_thing/test_errors.py | 61 +++++ .../test_thing/test_file_writer.py | 13 +- 65 files changed, 1732 insertions(+), 440 deletions(-) create mode 100644 HABApp/__cmd_args__.py create mode 100644 HABApp/core/events/event_filters.py create mode 100644 HABApp/mqtt/events/mqtt_filters.py create mode 100644 HABApp/openhab/events/event_filters.py create mode 100644 HABApp/rule_manager/benchmark/__init__.py create mode 100644 HABApp/rule_manager/benchmark/bench_base.py create mode 100644 HABApp/rule_manager/benchmark/bench_file.py create mode 100644 HABApp/rule_manager/benchmark/bench_habapp.py create mode 100644 HABApp/rule_manager/benchmark/bench_mqtt.py create mode 100644 HABApp/rule_manager/benchmark/bench_oh.py create mode 100644 HABApp/rule_manager/benchmark/bench_times.py create mode 100644 HABApp/runtime/shutdown.py delete mode 100644 HABApp/runtime/shutdown_helper.py create mode 100644 _doc/__style_guide__ create mode 100644 _doc/tips.rst delete mode 100644 conf_testing/rules/bench_rule.py create mode 100644 tests/helpers/mock_file.py create mode 100644 tests/helpers/module_helpers.py create mode 100644 tests/test_core/test_events/test_core_filters.py create mode 100644 tests/test_mqtt/test_mqtt_filters.py create mode 100644 tests/test_openhab/test_events/test_oh_filters.py create mode 100644 tests/test_openhab/test_plugins/test_thing/test_errors.py diff --git a/HABApp/__cmd_args__.py b/HABApp/__cmd_args__.py new file mode 100644 index 00000000..b863b6c3 --- /dev/null +++ b/HABApp/__cmd_args__.py @@ -0,0 +1,96 @@ +import argparse +import os +import sys +import time +import typing +from pathlib import Path + +# Global var if we want to run the benchmark +DO_BENCH = False + + +def parse_args(passed_args=None) -> Path: + global DO_BENCH + + parser = argparse.ArgumentParser(description='Start HABApp') + parser.add_argument( + '-c', + '--config', + help='Path to configuration folder (where the config.yml is located)', + default=None + ) + parser.add_argument( + '-s', + '--sleep', + help='Sleep time in seconds before starting HABApp', + type=int, + default=None + ) + parser.add_argument( + '-b', + '--benchmark', + help='Do a Benchmark based on the current config', + action='store_true' + ) + args = parser.parse_args(passed_args) + + DO_BENCH = args.benchmark + + if args.sleep: + args.sleep = max(0, args.sleep) + print(f'Waiting {args.sleep:d} seconds before starting HABApp ...', end='') + time.sleep(args.sleep) + print(' done!') + + path = args.config + if path is not None: + path = Path(path).resolve() + return find_config_folder(path) + + +def find_config_folder(arg_config_path: typing.Optional[Path]) -> Path: + + if arg_config_path is None: + # Nothing is specified, we try to find the config automatically + check_path = [] + try: + working_dir = Path(os.getcwd()) + check_path.append( working_dir / 'HABApp') + check_path.append( working_dir.with_name('HABApp')) + check_path.append( working_dir.parent.with_name('HABApp')) + except ValueError: + # the ValueError gets raised if the working_dir or its parent is empty (e.g. c:\ or /) + pass + + check_path.append(Path.home() / 'HABApp') # User home + + # if we run in a venv check the venv, too + v_env = os.environ.get('VIRTUAL_ENV', '') + if v_env: + check_path.append(Path(v_env) / 'HABApp') # Virtual env dir + else: + # in case the user specifies the config.yml we automatically switch to the parent folder + if arg_config_path.name.lower() == 'config.yml' and arg_config_path.is_file(): + arg_config_path = arg_config_path.parent + + # Override automatic config detection if something is specified through command line + check_path = [arg_config_path] + + for config_folder in check_path: + config_folder = config_folder.resolve() + if not config_folder.is_dir(): + continue + + config_file = config_folder / 'config.yml' + if config_file.is_file(): + return config_folder + + # we have specified a folder, but the config does not exist so we will create it + if arg_config_path is not None and arg_config_path.is_dir(): + return arg_config_path + + # we have nothing found and nothing specified -> exit + print('Config file "config.yml" not found!') + print('Checked folders:\n - ' + '\n - '.join(str(k) for k in check_path)) + print('Please create file or specify a folder with the "-c" arg switch.') + sys.exit(1) diff --git a/HABApp/__init__.py b/HABApp/__init__.py index 9e9cab89..753e22da 100644 --- a/HABApp/__init__.py +++ b/HABApp/__init__.py @@ -20,5 +20,4 @@ from HABApp.rule import Rule from HABApp.parameters import Parameter, DictParameter -#from HABApp.runtime import Runtime -from HABApp.config import CONFIG \ No newline at end of file +from HABApp.config import CONFIG diff --git a/HABApp/__main__.py b/HABApp/__main__.py index 5c33cbdb..504a52ab 100644 --- a/HABApp/__main__.py +++ b/HABApp/__main__.py @@ -1,99 +1,24 @@ -import argparse import asyncio import logging -import os import signal import sys -import time import traceback import typing -from pathlib import Path import HABApp - - -def find_config_folder(arg_config_path: typing.Optional[Path]) -> Path: - - if arg_config_path is None: - # Nothing is specified, we try to find the config automatically - check_path = [] - try: - working_dir = Path(os.getcwd()) - check_path.append( working_dir / 'HABApp') - check_path.append( working_dir.with_name('HABApp')) - check_path.append( working_dir.parent.with_name('HABApp')) - except ValueError: - # the ValueError gets raised if the working_dir or its parent is empty (e.g. c:\ or /) - pass - - check_path.append(Path.home() / 'HABApp') # User home - - # if we run in a venv check the venv, too - v_env = os.environ.get('VIRTUAL_ENV', '') - if v_env: - check_path.append(Path(v_env) / 'HABApp') # Virtual env dir - else: - # in case the user specifies the config.yml we automatically switch to the parent folder - if arg_config_path.name.lower() == 'config.yml' and arg_config_path.is_file(): - arg_config_path = arg_config_path.parent - - # Override automatic config detection if something is specified through command line - check_path = [arg_config_path] - - for config_folder in check_path: - config_folder = config_folder.resolve() - if not config_folder.is_dir(): - continue - - config_file = config_folder / 'config.yml' - if config_file.is_file(): - return config_folder - - # we have specified a folder, but the config does not exist so we will create it - if arg_config_path is not None and arg_config_path.is_dir(): - return arg_config_path - - # we have nothing found and nothing specified -> exit - print('Config file "config.yml" not found!') - print('Checked folders:\n - ' + '\n - '.join(str(k) for k in check_path)) - print('Please create file or specify a folder with the "-c" arg switch.') - sys.exit(1) - - -def get_command_line_args(args=None): - parser = argparse.ArgumentParser(description='Start HABApp') - parser.add_argument( - '-c', - '--config', - help='Path to configuration folder (where the config.yml is located)', - default=None - ) - parser.add_argument( - '-s', - '--sleep', - help='Sleep time in seconds before starting HABApp', - type=int, - default=None - ) - return parser.parse_args(args) +from HABApp.__cmd_args__ import parse_args def main() -> typing.Union[int, str]: - args = get_command_line_args() - if args.sleep: - args.sleep = max(0, args.sleep) - print(f'Waiting {args.sleep:d} seconds before starting HABApp ...', end='') - time.sleep(args.sleep) - print(' done!') - if args.config is not None: - args.config = Path(args.config).resolve() + # This has do be done before we create HABApp because of the possible sleep time + cfg_folder = parse_args() log = logging.getLogger('HABApp') try: app = HABApp.runtime.Runtime() - app.startup(config_folder=find_config_folder(args.config)) + app.startup(config_folder=cfg_folder) def shutdown_handler(sig, frame): print('Shutting down ...') diff --git a/HABApp/__version__.py b/HABApp/__version__.py index 2d1ae652..1843a7b6 100644 --- a/HABApp/__version__.py +++ b/HABApp/__version__.py @@ -1 +1 @@ -__version__ = '0.19.1' +__version__ = '0.20.0' diff --git a/HABApp/config/config_loader.py b/HABApp/config/config_loader.py index 568d3c9e..9c906295 100644 --- a/HABApp/config/config_loader.py +++ b/HABApp/config/config_loader.py @@ -125,15 +125,24 @@ def load_log(self): except Exception as e: print(f'Error loading logging config: {e}') return None - log.debug('Loaded logging config') # Try rotating the logs on first start if self.first_start: - for handler in logging._handlerList: + for wr in reversed(logging._handlerList[:]): + handler = wr() # weakref -> call it to get object + + # only rotate these types + if not isinstance(handler, (RotatingFileHandler, TimedRotatingFileHandler)): + continue + + # Rotate only if files have content + logfile = Path(handler.baseFilename) + if not logfile.is_file() or logfile.stat().st_size <= 0: + continue + try: - handler = handler() # weakref -> call it to get object - if isinstance(handler, (RotatingFileHandler, TimedRotatingFileHandler)): - handler.doRollover() + handler.acquire() + handler.doRollover() except Exception: lines = traceback.format_exc().splitlines() # cut away AbsolutePathExpected Exception from log output @@ -143,6 +152,8 @@ def load_log(self): start = i for line in lines[start:]: log.error(line) + finally: + handler.release() logging.getLogger('HABApp').info(f'HABApp Version {__version__}') diff --git a/HABApp/core/event_bus_listener.py b/HABApp/core/event_bus_listener.py index 120974d6..f736b2f2 100644 --- a/HABApp/core/event_bus_listener.py +++ b/HABApp/core/event_bus_listener.py @@ -6,13 +6,13 @@ class EventBusListener: def __init__(self, topic, callback, event_type=AllEvents, - prop_name1: Optional[str] = None, prop_value1: Optional[Any] = None, - prop_name2: Optional[str] = None, prop_value2: Optional[Any] = None, + attr_name1: Optional[str] = None, attr_value1: Optional[Any] = None, + attr_name2: Optional[str] = None, attr_value2: Optional[Any] = None, ): assert isinstance(topic, str), type(topic) assert isinstance(callback, WrappedFunction) - assert prop_name1 is None or isinstance(prop_name1, str), prop_name1 - assert prop_name2 is None or isinstance(prop_name2, str), prop_name2 + assert attr_name1 is None or isinstance(attr_name1, str), attr_name1 + assert attr_name2 is None or isinstance(attr_name2, str), attr_name2 self.topic: str = topic self.func: WrappedFunction = callback @@ -20,10 +20,10 @@ def __init__(self, topic, callback, event_type=AllEvents, self.event_filter = event_type # Property filters - self.prop_name1 = prop_name1 - self.prop_value1 = prop_value1 - self.prop_name2 = prop_name2 - self.prop_value2 = prop_value2 + self.attr_name1 = attr_name1 + self.attr_value1 = attr_value1 + self.attr_name2 = attr_name2 + self.attr_value2 = attr_value2 self.__is_all: bool = self.event_filter is AllEvents self.__is_single: bool = not isinstance(self.event_filter, (list, tuple, set)) @@ -38,11 +38,11 @@ def notify_listeners(self, event): if self.__is_single: if isinstance(event, self.event_filter): # If we have property filters wie only trigger when value is set accordingly - if self.prop_name1 is not None: - if getattr(event, self.prop_name1, None) != self.prop_value1: + if self.attr_name1 is not None: + if getattr(event, self.attr_name1, None) != self.attr_value1: return None - if self.prop_name2 is not None: - if getattr(event, self.prop_name2, None) != self.prop_value2: + if self.attr_name2 is not None: + if getattr(event, self.attr_name2, None) != self.attr_value2: return None self.func.run(event) @@ -52,11 +52,11 @@ def notify_listeners(self, event): for cls in self.event_filter: if isinstance(event, cls): # If we have property filters wie only trigger when value is set accordingly - if self.prop_name1 is not None: - if getattr(event, self.prop_name1, None) != self.prop_value1: + if self.attr_name1 is not None: + if getattr(event, self.attr_name1, None) != self.attr_value1: return None - if self.prop_name2 is not None: - if getattr(event, self.prop_name2, None) != self.prop_value2: + if self.attr_name2 is not None: + if getattr(event, self.attr_name2, None) != self.attr_value2: return None self.func.run(event) @@ -73,9 +73,9 @@ def desc(self): _type = _type[8:-2].split('.')[-1] _val = '' - if self.prop_name1 is not None: - _val += f', {self.prop_name1}=={self.prop_value1}' - if self.prop_name2 is not None: - _val += f', {self.prop_name2}=={self.prop_value2}' + if self.attr_name1 is not None: + _val += f', {self.attr_name1}=={self.attr_value1}' + if self.attr_name2 is not None: + _val += f', {self.attr_name2}=={self.attr_value2}' return f'"{self.topic}" (type {_type}{_val})' diff --git a/HABApp/core/events/__init__.py b/HABApp/core/events/__init__.py index e9f8957c..ff9da7c8 100644 --- a/HABApp/core/events/__init__.py +++ b/HABApp/core/events/__init__.py @@ -1,3 +1,4 @@ from .events import ComplexEventValue, ValueUpdateEvent, ValueChangeEvent, \ ItemNoChangeEvent, ItemNoUpdateEvent, AllEvents from . import habapp_events +from .event_filters import EventFilter, ValueChangeEventFilter, ValueUpdateEventFilter diff --git a/HABApp/core/events/event_filters.py b/HABApp/core/events/event_filters.py new file mode 100644 index 00000000..33b31b80 --- /dev/null +++ b/HABApp/core/events/event_filters.py @@ -0,0 +1,53 @@ +from typing import Any + +import HABApp +from HABApp.core.const import MISSING +from . import ValueChangeEvent, ValueUpdateEvent + + +class EventFilter: + def __init__(self, event_type, **kwargs): + assert len(kwargs) < 3, 'EventFilter only allows up to two args that will be used to filter' + + for arg in kwargs: + if arg not in event_type.__annotations__: + raise AttributeError(f'Filter attribute "{arg}" does not exist for "{event_type.__name__}"') + + self.__cls = event_type + self.__filter = kwargs + + def create_event_listener(self, name, cb) -> 'HABApp.core.EventBusListener': + kwargs = {'event_type': self.__cls} + ct = 1 + for k, v in self.__filter.items(): + kwargs[f'attr_name{ct}'] = k + kwargs[f'attr_value{ct}'] = v + ct += 1 + + return HABApp.core.EventBusListener(name, cb, **kwargs) + + def __repr__(self): + name = self.__class__.__name__ + vals = [f'{k}={v}' for k, v in self.__filter.items()] + if name == EventFilter.__name__: + vals.insert(0, f'event_type={self.__cls.__name__}') + return f'{name}({", ".join(vals)})' + + +class ValueUpdateEventFilter(EventFilter): + _EVENT_TYPE = ValueUpdateEvent + + def __init__(self, value): + super().__init__(self._EVENT_TYPE, value=value) + + +class ValueChangeEventFilter(EventFilter): + _EVENT_TYPE = ValueChangeEvent + + def __init__(self, value: Any = MISSING, old_value: Any = MISSING): + args = {} + if value is not MISSING: + args['value'] = value + if old_value is not MISSING: + args['old_value'] = old_value + super().__init__(self._EVENT_TYPE, **args) diff --git a/HABApp/core/events/events.py b/HABApp/core/events/events.py index 7669c476..ef526e37 100644 --- a/HABApp/core/events/events.py +++ b/HABApp/core/events/events.py @@ -1,4 +1,4 @@ -import typing +from typing import Any, Union class AllEvents: @@ -7,10 +7,13 @@ class AllEvents: class ComplexEventValue: def __init__(self, value): - self.value: typing.Any = value + self.value: Any = value class ValueUpdateEvent: + name: str + value: Any + def __init__(self, name=None, value=None): self.name: str = name self.value = value @@ -20,6 +23,10 @@ def __repr__(self): class ValueChangeEvent: + name: str + value: Any + old_value: Any + def __init__(self, name=None, value=None, old_value=None): self.name: str = name self.value = value @@ -30,6 +37,9 @@ def __repr__(self): class ItemNoChangeEvent: + name: str + seconds: Union[int, float] + def __init__(self, name=None, seconds=None): self.name: str = name self.seconds: int = seconds @@ -39,6 +49,9 @@ def __repr__(self): class ItemNoUpdateEvent: + name: str + seconds: Union[int, float] + def __init__(self, name=None, seconds=None): self.name: str = name self.seconds: int = seconds diff --git a/HABApp/core/files/watcher/base_watcher.py b/HABApp/core/files/watcher/base_watcher.py index fd6fd90e..99f9377c 100644 --- a/HABApp/core/files/watcher/base_watcher.py +++ b/HABApp/core/files/watcher/base_watcher.py @@ -1,7 +1,11 @@ +import logging from pathlib import Path from watchdog.events import FileSystemEvent, FileSystemEventHandler +log = logging.getLogger('HABApp.file_events') +log.setLevel(logging.INFO) + class BaseWatcher(FileSystemEventHandler): def __init__(self, folder: Path, file_ending: str, watch_subfolders: bool = False): @@ -14,6 +18,9 @@ def __init__(self, folder: Path, file_ending: str, watch_subfolders: bool = Fals self.watch_subfolders: bool = watch_subfolders def dispatch(self, event: FileSystemEvent): + + log.debug(event) + # we don't process directory events if event.is_directory: return None diff --git a/HABApp/core/files/watcher/folder_watcher.py b/HABApp/core/files/watcher/folder_watcher.py index 20680871..5efb2467 100644 --- a/HABApp/core/files/watcher/folder_watcher.py +++ b/HABApp/core/files/watcher/folder_watcher.py @@ -13,21 +13,19 @@ WATCHES: Dict[str, ObservedWatch] = {} -def start(shutdown_helper): +def start(): global OBSERVER # start only once! assert OBSERVER is None - from HABApp.runtime.shutdown_helper import ShutdownHelper - assert isinstance(shutdown_helper, ShutdownHelper) - OBSERVER = Observer() OBSERVER.start() # register for proper shutdown - shutdown_helper.register_func(OBSERVER.stop) - shutdown_helper.register_func(OBSERVER.join, last=True) + from HABApp.runtime import shutdown + shutdown.register_func(OBSERVER.stop, msg='Stopping folder observer') + shutdown.register_func(OBSERVER.join, last=True, msg='Joining folder observer threads') return None diff --git a/HABApp/core/items/base_item.py b/HABApp/core/items/base_item.py index 0a2302c0..565ce56b 100644 --- a/HABApp/core/items/base_item.py +++ b/HABApp/core/items/base_item.py @@ -1,11 +1,11 @@ import datetime -import typing +from typing import Any, Callable, Union import tzlocal from pytz import utc import HABApp -from .base_item_times import ItemNoChangeWatch, ItemNoUpdateWatch, ChangedTime, UpdatedTime +from .base_item_times import ChangedTime, ItemNoChangeWatch, ItemNoUpdateWatch, UpdatedTime from .tmp_data import add_tmp_data as _add_tmp_data from .tmp_data import restore_tmp_data as _restore_tmp_data @@ -62,7 +62,7 @@ def __repr__(self): ret += f'{", " if ret else ""}{k}: {getattr(self, k)}' return f'<{self.__class__.__name__} {ret:s}>' - def watch_change(self, secs: typing.Union[int, float, datetime.timedelta]) -> ItemNoChangeWatch: + def watch_change(self, secs: Union[int, float, datetime.timedelta]) -> ItemNoChangeWatch: """Generate an event if the item does not change for a certain period of time. Has to be called from inside a rule function. @@ -80,7 +80,7 @@ def watch_change(self, secs: typing.Union[int, float, datetime.timedelta]) -> It HABApp.rule.get_parent_rule().register_cancel_obj(w) return w - def watch_update(self, secs: typing.Union[int, float, datetime.timedelta]) -> ItemNoUpdateWatch: + def watch_update(self, secs: Union[int, float, datetime.timedelta]) -> ItemNoUpdateWatch: """Generate an event if the item does not receive and update for a certain period of time. Has to be called from inside a rule function. @@ -98,8 +98,8 @@ def watch_update(self, secs: typing.Union[int, float, datetime.timedelta]) -> It HABApp.rule.get_parent_rule().register_cancel_obj(w) return w - def listen_event(self, callback: typing.Callable[[typing.Any], typing.Any], - event_type: typing.Union['HABApp.core.events.AllEvents', typing.Any] + def listen_event(self, callback: Callable[[Any], Any], + event_type: Union['HABApp.core.events.AllEvents', 'HABApp.core.events.EventFilter', Any] ) -> 'HABApp.core.EventBusListener': """ Register an event listener which listens to all event that the item receives diff --git a/HABApp/core/items/base_item_watch.py b/HABApp/core/items/base_item_watch.py index 35eded05..c925257f 100644 --- a/HABApp/core/items/base_item_watch.py +++ b/HABApp/core/items/base_item_watch.py @@ -5,7 +5,7 @@ import HABApp from HABApp.core.lib import PendingFuture from ..const import loop -from ..events import ItemNoChangeEvent, ItemNoUpdateEvent +from ..events import ItemNoChangeEvent, ItemNoUpdateEvent, EventFilter log = logging.getLogger('HABApp') @@ -32,9 +32,7 @@ def listen_event(self, callback: typing.Callable[[typing.Any], typing.Any]) -> ' """Listen to (only) the event that is emitted by this watcher""" rule = HABApp.rule.get_parent_rule() cb = HABApp.core.WrappedFunction(callback, name=rule._get_cb_name(callback)) - listener = HABApp.core.EventBusListener( - self.name, cb, self.EVENT, 'seconds', self.fut.secs - ) + listener = EventFilter(self.EVENT, seconds=self.fut.secs).create_event_listener(self.name, cb) return rule._add_event_listener(listener) diff --git a/HABApp/core/items/item_aggregation.py b/HABApp/core/items/item_aggregation.py index c56a1f87..894489e3 100644 --- a/HABApp/core/items/item_aggregation.py +++ b/HABApp/core/items/item_aggregation.py @@ -56,11 +56,14 @@ def aggregation_period(self, period: typing.Union[float, int]) -> 'AggregationIt self.__period = period return self - def aggregation_source(self, source: typing.Union[BaseValueItem, str]) -> 'AggregationItem': + def aggregation_source(self, source: typing.Union[BaseValueItem, str], + only_changes: bool = False) -> 'AggregationItem': """Set the source item which changes will be aggregated - :param item_or_name: name or Item obj + :param source: name or Item obj + :param only_changes: if true only value changes instead of value updates will be added """ + # If we already have one we cancel it if self.__listener is not None: self.__listener.cancel() @@ -69,7 +72,7 @@ def aggregation_source(self, source: typing.Union[BaseValueItem, str]) -> 'Aggre self.__listener = HABApp.core.EventBusListener( topic=source.name if isinstance(source, HABApp.core.items.BaseValueItem) else source, callback=HABApp.core.WrappedFunction(self._add_value, name=f'{self.name}.add_value'), - event_type=HABApp.core.events.ValueChangeEvent + event_type=HABApp.core.events.ValueChangeEvent if only_changes else HABApp.core.events.ValueUpdateEvent ) HABApp.core.EventBus.add_listener(self.__listener) return self diff --git a/HABApp/mqtt/__init__.py b/HABApp/mqtt/__init__.py index 867ab982..2f3d99e1 100644 --- a/HABApp/mqtt/__init__.py +++ b/HABApp/mqtt/__init__.py @@ -1,4 +1,3 @@ from . import events from . import items - -from .interface import subscribe, unsubscribe, publish +from . import interface diff --git a/HABApp/mqtt/events/__init__.py b/HABApp/mqtt/events/__init__.py index 031f0b5e..ca4c4299 100644 --- a/HABApp/mqtt/events/__init__.py +++ b/HABApp/mqtt/events/__init__.py @@ -1 +1,2 @@ from .mqtt_events import MqttValueChangeEvent, MqttValueUpdateEvent +from .mqtt_filters import MqttValueChangeEventFilter, MqttValueUpdateEventFilter diff --git a/HABApp/mqtt/events/mqtt_filters.py b/HABApp/mqtt/events/mqtt_filters.py new file mode 100644 index 00000000..2c1457f3 --- /dev/null +++ b/HABApp/mqtt/events/mqtt_filters.py @@ -0,0 +1,10 @@ +from HABApp.core.events import ValueChangeEventFilter, ValueUpdateEventFilter +from . import MqttValueChangeEvent, MqttValueUpdateEvent + + +class MqttValueUpdateEventFilter(ValueUpdateEventFilter): + _EVENT_TYPE = MqttValueUpdateEvent + + +class MqttValueChangeEventFilter(ValueChangeEventFilter): + _EVENT_TYPE = MqttValueChangeEvent diff --git a/HABApp/mqtt/mqtt_connection.py b/HABApp/mqtt/mqtt_connection.py index 95b353f5..90f09adc 100644 --- a/HABApp/mqtt/mqtt_connection.py +++ b/HABApp/mqtt/mqtt_connection.py @@ -6,7 +6,7 @@ import HABApp from HABApp.core import Items from HABApp.core.wrapper import log_exception -from HABApp.runtime.shutdown_helper import ShutdownHelper +from HABApp.runtime import shutdown from .events import MqttValueChangeEvent, MqttValueUpdateEvent from ..core.const.json import load_json @@ -25,7 +25,7 @@ def __init__(self): STATUS = MqttStatus() -def setup(shutdown_helper: ShutdownHelper): +def setup(): config = HABApp.config.CONFIG.mqtt # config changes @@ -33,7 +33,7 @@ def setup(shutdown_helper: ShutdownHelper): config.connection.subscribe_for_changes(connect) # shutdown - shutdown_helper.register_func(disconnect) + shutdown.register_func(disconnect, msg='Disconnecting MQTT') def connect(): diff --git a/HABApp/openhab/connection_handler/http_connection.py b/HABApp/openhab/connection_handler/http_connection.py index ee6b4efd..10b822a2 100644 --- a/HABApp/openhab/connection_handler/http_connection.py +++ b/HABApp/openhab/connection_handler/http_connection.py @@ -199,8 +199,8 @@ async def check_response(future: aiohttp.client._RequestContextManager, sent_dat return resp -def stop_connection(): - global FUT_UUID, FUT_SSE +async def stop_connection(): + global FUT_UUID, FUT_SSE, HTTP_SESSION if FUT_UUID is not None and not FUT_UUID.done(): FUT_UUID.cancel() FUT_UUID = None @@ -209,17 +209,19 @@ def stop_connection(): FUT_SSE.cancel() FUT_SSE = None - -async def start_connection(): - global HTTP_PREFIX, HTTP_SESSION, FUT_UUID - - stop_connection() + await asyncio.sleep(0) # If we are already connected properly disconnect if HTTP_SESSION is not None: await HTTP_SESSION.close() HTTP_SESSION = None + +async def start_connection(): + global HTTP_PREFIX, HTTP_SESSION, FUT_UUID + + await stop_connection() + host: str = HABApp.CONFIG.openhab.connection.host port: str = HABApp.CONFIG.openhab.connection.port diff --git a/HABApp/openhab/connection_logic/connection.py b/HABApp/openhab/connection_logic/connection.py index c2084efa..9b9069d3 100644 --- a/HABApp/openhab/connection_logic/connection.py +++ b/HABApp/openhab/connection_logic/connection.py @@ -11,8 +11,8 @@ log = http_connection.log -def setup(shutdown): - assert isinstance(shutdown, HABApp.runtime.ShutdownHelper), type(shutdown) +def setup(): + from HABApp.runtime import shutdown # initialize callbacks http_connection.ON_CONNECTED = on_connect @@ -20,10 +20,10 @@ def setup(shutdown): http_connection.ON_SSE_EVENT = on_sse_event # shutdown handler for connection - shutdown.register_func(http_connection.stop_connection) + shutdown.register_func(http_connection.stop_connection, msg='Stopping openHAB connection') # shutdown handler for plugins - shutdown.register_func(on_disconnect) + shutdown.register_func(on_disconnect, msg='Stopping openHAB plugins') # initialize all plugins setup_plugins() @@ -31,7 +31,7 @@ def setup(shutdown): async def start(): - await http_connection.start_connection(), + await http_connection.start_connection() @ignore_exception diff --git a/HABApp/openhab/connection_logic/plugin_things/items_file.py b/HABApp/openhab/connection_logic/plugin_things/items_file.py index 67e6742d..f874512d 100644 --- a/HABApp/openhab/connection_logic/plugin_things/items_file.py +++ b/HABApp/openhab/connection_logic/plugin_things/items_file.py @@ -130,5 +130,12 @@ def create_items_file(path: Path, items_dict: Dict[str, UserItem]): # newline aber each name block lines.append('\n') - with path.open(mode='w', encoding='utf-8') as file: + # If we have multiple parts configs in one file we separate them with newlines + if path.is_file(): + lines.insert(0, '\n') + lines.insert(0, '\n') + lines.insert(0, '\n') + + # Use append, file was deleted when we loaded the config + with path.open(mode='a', encoding='utf-8') as file: file.writelines(lines) diff --git a/HABApp/openhab/connection_logic/plugin_things/plugin_things.py b/HABApp/openhab/connection_logic/plugin_things/plugin_things.py index dcbaad10..7f4eb1d3 100644 --- a/HABApp/openhab/connection_logic/plugin_things/plugin_things.py +++ b/HABApp/openhab/connection_logic/plugin_things/plugin_things.py @@ -16,6 +16,10 @@ from .._plugin import OnConnectPlugin +class DuplicateItemError(Exception): + pass + + class ManualThingConfig(OnConnectPlugin): def __init__(self): @@ -149,7 +153,7 @@ async def update_thing_config(self, path: Path, data=None): for item_cfg in cfg_entry.get_items(thing_context): name = item_cfg.name if name in create_items: - raise ValueError(f'Duplicate item: {name}') + raise DuplicateItemError(f'Duplicate item: {name}') create_items[name] = item_cfg # Channel overview, only if we have something configured @@ -171,7 +175,7 @@ async def update_thing_config(self, path: Path, data=None): name = item_cfg.name if name in create_items: - raise ValueError(f'Duplicate item: {name}') + raise DuplicateItemError(f'Duplicate item: {name}') create_items[name] = item_cfg # newline only if we create logs @@ -180,6 +184,10 @@ async def update_thing_config(self, path: Path, data=None): except InvalidItemNameError as e: HABAppError(log).add_exception(e).dump() continue + except DuplicateItemError as e: + # Duplicates should never happen, the user clearly made a mistake, that's why we exit here + HABAppError(log).add_exception(e).dump() + return None # Create all items for item_cfg in create_items.values(): diff --git a/HABApp/openhab/events/__init__.py b/HABApp/openhab/events/__init__.py index 216e0f5a..64acb856 100644 --- a/HABApp/openhab/events/__init__.py +++ b/HABApp/openhab/events/__init__.py @@ -4,3 +4,4 @@ from .channel_events import ChannelTriggeredEvent from .thing_events import ThingStatusInfoChangedEvent, ThingStatusInfoEvent, \ ThingConfigStatusInfoEvent, ThingFirmwareStatusInfoEvent +from .event_filters import ItemStateChangedEventFilter, ItemStateEventFilter diff --git a/HABApp/openhab/events/channel_events.py b/HABApp/openhab/events/channel_events.py index fcfed52e..d80ccbea 100644 --- a/HABApp/openhab/events/channel_events.py +++ b/HABApp/openhab/events/channel_events.py @@ -7,6 +7,10 @@ class ChannelTriggeredEvent(OpenhabEvent): + name: str + event: str + channel: str + def __init__(self, name: str = '', event: str = '', channel: str = ''): super().__init__() diff --git a/HABApp/openhab/events/event_filters.py b/HABApp/openhab/events/event_filters.py new file mode 100644 index 00000000..fe1b2b8e --- /dev/null +++ b/HABApp/openhab/events/event_filters.py @@ -0,0 +1,10 @@ +from HABApp.core.events import ValueChangeEventFilter, ValueUpdateEventFilter +from . import ItemStateChangedEvent, ItemStateEvent + + +class ItemStateEventFilter(ValueUpdateEventFilter): + _EVENT_TYPE = ItemStateEvent + + +class ItemStateChangedEventFilter(ValueChangeEventFilter): + _EVENT_TYPE = ItemStateChangedEvent diff --git a/HABApp/openhab/events/item_events.py b/HABApp/openhab/events/item_events.py index 05be1107..2fb5f32f 100644 --- a/HABApp/openhab/events/item_events.py +++ b/HABApp/openhab/events/item_events.py @@ -11,6 +11,9 @@ class ItemStateEvent(OpenhabEvent, HABApp.core.events.ValueUpdateEvent): + name: str + value: typing.Any + def __init__(self, name: str = '', value: typing.Any = None): super().__init__() @@ -28,12 +31,16 @@ def __repr__(self): class ItemStateChangedEvent(OpenhabEvent, HABApp.core.events.ValueChangeEvent): + name: str + value: typing.Any + old_value: typing.Any + def __init__(self, name: str = '', value: typing.Any = None, old_value: typing.Any = None): super().__init__() self.name: str = name - self.value = value - self.old_value = old_value + self.value: typing.Any = value + self.old_value: typing.Any = old_value @classmethod def from_dict(cls, topic: str, payload: dict): @@ -49,11 +56,14 @@ def __repr__(self): class ItemCommandEvent(OpenhabEvent): + name: str + value: typing.Any + def __init__(self, name: str = '', value: typing.Any = None): super().__init__() self.name: str = name - self.value = value + self.value: typing.Any = value @classmethod def from_dict(cls, topic: str, payload: dict): @@ -65,6 +75,9 @@ def __repr__(self): class ItemAddedEvent(OpenhabEvent): + name: str + type: str + def __init__(self, name: str = '', type: str = ''): super().__init__() @@ -83,6 +96,9 @@ def __repr__(self): class ItemUpdatedEvent(OpenhabEvent): + name: str + type: str + def __init__(self, name: str = '', type: str = ''): super().__init__() @@ -102,6 +118,8 @@ def __repr__(self): class ItemRemovedEvent(OpenhabEvent): + name: str + def __init__(self, name: str = ''): super().__init__() @@ -117,6 +135,9 @@ def __repr__(self): class ItemStatePredictedEvent(OpenhabEvent): + name: str + value: typing.Any + def __init__(self, name: str = '', value: typing.Any = None): super().__init__() @@ -134,14 +155,19 @@ def __repr__(self): class GroupItemStateChangedEvent(OpenhabEvent): + name: str + item: str + value: typing.Any + old_value: typing.Any + def __init__(self, name: str = '', item: str = '', value: typing.Any = None, old_value: typing.Any = None): super().__init__() self.name: str = name self.item: str = item - self.value = value - self.old_value = old_value + self.value: typing.Any = value + self.old_value: typing.Any = old_value @classmethod def from_dict(cls, topic: str, payload: dict): diff --git a/HABApp/openhab/events/thing_events.py b/HABApp/openhab/events/thing_events.py index eb43e853..e314bbdd 100644 --- a/HABApp/openhab/events/thing_events.py +++ b/HABApp/openhab/events/thing_events.py @@ -9,6 +9,10 @@ class ThingStatusInfoEvent(OpenhabEvent): + name: str + status: str + detail: str + def __init__(self, name: str = '', status: str = '', detail: str = ''): super().__init__() @@ -26,6 +30,12 @@ def __repr__(self): class ThingStatusInfoChangedEvent(OpenhabEvent): + name: str + status: str + detail: str + old_status: str + old_detail: str + def __init__(self, name: str = '', status: str = '', detail: str = '', old_status: str = '', old_detail: str = ''): super().__init__() @@ -52,6 +62,9 @@ def __repr__(self): class ThingConfigStatusInfoEvent(OpenhabEvent): + name: str + messages: typing.List[typing.Dict[str, str]] + def __init__(self, name: str = '', messages: typing.List[typing.Dict[str, str]] = [{}]): super().__init__() @@ -68,6 +81,9 @@ def __repr__(self): class ThingFirmwareStatusInfoEvent(OpenhabEvent): + name: str + status: str + def __init__(self, name: str = '', status: str = ''): super().__init__() self.name: str = name diff --git a/HABApp/rule/interfaces/http.py b/HABApp/rule/interfaces/http.py index ac95840f..20c7a8c5 100644 --- a/HABApp/rule/interfaces/http.py +++ b/HABApp/rule/interfaces/http.py @@ -12,8 +12,13 @@ def __init__(self): self.__client: aiohttp.ClientSession = None async def create_client(self): + assert self.__client is None + self.__client = aiohttp.ClientSession(json_serialize=dump_json, loop=loop) + from HABApp.runtime import shutdown + shutdown.register_func(self.__client.close, msg='Closing generic http connection') + def get(self, url: str, params: Optional[Mapping[str, str]] = None, **kwargs: Any)\ -> aiohttp.client._RequestContextManager: """http get request diff --git a/HABApp/rule/rule.py b/HABApp/rule/rule.py index 77b2f09d..dcd6c89f 100644 --- a/HABApp/rule/rule.py +++ b/HABApp/rule/rule.py @@ -125,21 +125,28 @@ def post_event(self, name, event): def listen_event(self, name: typing.Union[HABApp.core.items.BaseValueItem, str], callback: typing.Callable[[typing.Any], typing.Any], - event_type: typing.Union[AllEvents, typing.Any] = AllEvents + event_type: typing.Union[typing.Type['HABApp.core.events.AllEvents'], + 'HABApp.core.events.EventFilter', typing.Any] = AllEvents ) -> HABApp.core.EventBusListener: """ Register an event listener :param name: item or name to listen to. Use None to listen to all events :param callback: callback that accepts one parameter which will contain the event - :param event_type: Event filter. This is typically :class:`~HABApp.core.ValueUpdateEvent` or - :class:`~HABApp.core.ValueChangeEvent` which will also trigger on changes/update from openhab - or mqtt. + :param event_type: Event filter. This is typically :class:`~HABApp.core.events.ValueUpdateEvent` or + :class:`~HABApp.core.events.ValueChangeEvent` which will also trigger on changes/update from openhab + or mqtt. Additionally it can be an instance of :class:`~HABApp.core.events.EventFilter` which additionally + filters on the values of the event. There are also templates for the most common filters, e.g. + :class:`~HABApp.core.events.ValueUpdateEventFilter` and :class:`~HABApp.core.events.ValueChangeEventFilter` """ cb = HABApp.core.WrappedFunction(callback, name=self._get_cb_name(callback)) - listener = HABApp.core.EventBusListener( - name.name if isinstance(name, HABApp.core.items.BaseValueItem) else name, cb, event_type - ) + name = name.name if isinstance(name, HABApp.core.items.BaseValueItem) else name + + if isinstance(event_type, HABApp.core.events.EventFilter): + listener = event_type.create_event_listener(name, cb) + else: + listener = HABApp.core.EventBusListener(name, cb, event_type) + self.__event_listener.append(listener) HABApp.core.EventBus.add_listener(listener) return listener diff --git a/HABApp/rule/scheduler/base.py b/HABApp/rule/scheduler/base.py index e043b1fe..e71592b0 100644 --- a/HABApp/rule/scheduler/base.py +++ b/HABApp/rule/scheduler/base.py @@ -107,7 +107,7 @@ def offset(self, timedelta_obj: typing.Optional[timedelta]) -> 'ScheduledCallbac def jitter(self, secs: typing.Optional[int]) -> 'ScheduledCallbackBase': """Add a random jitter per call in the intervall [(-1) * secs ... secs] to the next run. - ``None`` will disable jitter. + ``None`` will disable jitter. :param secs: jitter in secs """ @@ -120,7 +120,7 @@ def jitter(self, secs: typing.Optional[int]) -> 'ScheduledCallbackBase': def boundary_func(self, func: typing.Optional[typing.Callable[[datetime], datetime]]): """Add a function which will be called when the datetime changes. Use this to implement custom boundaries. - Use ``None`` to disable the boundary function. + Use ``None`` to disable the boundary function. :param func: Function which returns a datetime obj, arg is a datetime with the next call time """ diff --git a/HABApp/rule_manager/benchmark/__init__.py b/HABApp/rule_manager/benchmark/__init__.py new file mode 100644 index 00000000..c65e859b --- /dev/null +++ b/HABApp/rule_manager/benchmark/__init__.py @@ -0,0 +1 @@ +from .bench_file import BenchFile \ No newline at end of file diff --git a/HABApp/rule_manager/benchmark/bench_base.py b/HABApp/rule_manager/benchmark/bench_base.py new file mode 100644 index 00000000..08501999 --- /dev/null +++ b/HABApp/rule_manager/benchmark/bench_base.py @@ -0,0 +1,72 @@ +from typing import Optional + +import HABApp +from HABApp.core.const.topics import ERRORS + + +class BenchBaseRule(HABApp.Rule): + BENCH_TYPE: str + + def __init__(self): + super().__init__() + + self.err_watcher = None + self.errors = [] + + self.prev_rule: Optional[BenchBaseRule] = None + self.next_rule: Optional[BenchBaseRule] = None + + def link_rule(self, next_rule: 'BenchBaseRule'): + assert self.next_rule is None + assert next_rule.prev_rule is None + + self.next_rule = next_rule + next_rule.prev_rule = self + return next_rule + + def _err_event(self, event): + self.errors.append(event) + + def do_bench_start(self): + self.errors.clear() + self.err_watcher = self.listen_event(ERRORS, self._err_event) + + self.run_in(1, self.do_bench_run) + + def do_bench_run(self): + try: + try: + print('+' + '-' * 78 + '+') + print(f'| {self.BENCH_TYPE:^76s} |') + print('+' + '-' * 78 + '+') + print('') + + self.set_up() + self.run() + finally: + self.tear_down() + finally: + self.run_in(1, self.do_bench_finished) + + def set_up(self): + pass + + def tear_down(self): + pass + + def run(self): + raise NotImplementedError() + + def do_bench_finished(self): + self.err_watcher.cancel() + + if self.errors: + count = len(self.errors) + print(f'{count} error{"" if count == 1 else "s"} during Benchmark in {self.rule_name}!') + for e in self.errors: + print(f' - {type(e.exception)}: {e.exception}') + + if self.next_rule is None: + HABApp.runtime.shutdown.request_shutdown() + else: + self.run_soon(self.next_rule.do_bench_start) diff --git a/HABApp/rule_manager/benchmark/bench_file.py b/HABApp/rule_manager/benchmark/bench_file.py new file mode 100644 index 00000000..10fd1797 --- /dev/null +++ b/HABApp/rule_manager/benchmark/bench_file.py @@ -0,0 +1,30 @@ +from pathlib import Path + +import HABApp +from HABApp.rule_manager import RuleFile +from .bench_habapp import HABAppBenchRule +from .bench_oh import OpenhabBenchRule +from .bench_mqtt import MqttBenchRule + + +class BenchFile(RuleFile): + def __init__(self, rule_manager): + super().__init__(rule_manager, path=Path('BenchmarkFile')) + + def create_rules(self, created_rules: list): + glob = globals() + glob['__HABAPP__RUNTIME__'] = self.rule_manager.runtime + glob['__HABAPP__RULE_FILE__'] = self + glob['__HABAPP__RULES'] = created_rules + + rule_ha = rule = HABAppBenchRule() + if HABApp.CONFIG.mqtt.connection.host: + rule = rule.link_rule(MqttBenchRule()) + if HABApp.CONFIG.openhab.connection.host: + rule = rule.link_rule(OpenhabBenchRule()) + + rule_ha.run_in(5, rule_ha.do_bench_start) + + glob.pop('__HABAPP__RUNTIME__') + glob.pop('__HABAPP__RULE_FILE__') + glob.pop('__HABAPP__RULES') diff --git a/HABApp/rule_manager/benchmark/bench_habapp.py b/HABApp/rule_manager/benchmark/bench_habapp.py new file mode 100644 index 00000000..fc6d433e --- /dev/null +++ b/HABApp/rule_manager/benchmark/bench_habapp.py @@ -0,0 +1,109 @@ +import random +import time +from collections import deque +from threading import Lock + +import HABApp +from HABApp.core.events import ValueUpdateEvent +from .bench_base import BenchBaseRule +from .bench_times import BenchContainer, BenchTime + +LOCK = Lock() + + +class HABAppBenchRule(BenchBaseRule): + BENCH_TYPE = 'HABApp' + + def __init__(self): + super().__init__() + + self.name_list = [f'BenchItem{k}' for k in range(300)] + + self.time_sent = 0.0 + self.bench_started = 0.0 + self.bench_times_container = BenchContainer() + self.bench_times: BenchTime = None + + self.name = '' + self.values = deque() + + def cleanup(self): + for n in self.name_list: + if HABApp.core.Items.item_exists(n): + HABApp.core.Items.pop_item(n) + + def set_up(self): + self.cleanup() + + def tear_down(self): + pass + self.cleanup() + + def run(self): + # These are the benchmarks + self.bench_rtt_time() + + def bench_rtt_time(self): + print('Bench events ', end='') + self.bench_times_container = BenchContainer() + + self.run_rtt('rtt idle') + self.run_rtt('async rtt idle', do_async=True) + + # self.start_load() + # self.run_rtt('rtt load (+10x)') + # self.run_rtt('async rtt load (+10x)', do_async=True) + # self.stop_load() + + print(' done!\n') + time.sleep(0.1) + self.bench_times_container.show() + + def run_rtt(self, test_name, do_async=False): + self.name = self.name_list[0] + self.openhab.create_item('String', self.name, label='MyLabel') + + for i in range(50_000): + self.values.append(random.randint(0, 99999999)) + + listener = self.listen_event( + self.name, self.post_next_event_val if not do_async else self.a_post_next_event_val + ) + + self.bench_times = self.bench_times_container.create(test_name) + + self.time_sent = time.time() + HABApp.core.EventBus.post_event(self.name, self.values[0]) + + self.run_soon(LOCK.acquire) + time.sleep(1) + LOCK.acquire(True, 5) + + listener.cancel() + if LOCK.locked(): + LOCK.release() + + print('.', end='') + + def post_next_event_val(self, value): + if value != self.values[0]: + return None + + self.bench_times.times.append(time.time() - self.time_sent) + + # No items left -> stop benchmark + try: + self.values.popleft() + except IndexError: + LOCK.release() + return None + + if not self.values: + LOCK.release() + return None + + self.time_sent = time.time() + HABApp.core.EventBus.post_event(self.name, self.values[0]) + + async def a_post_next_event_val(self, event: ValueUpdateEvent): + self.post_next_event_val(event) diff --git a/HABApp/rule_manager/benchmark/bench_mqtt.py b/HABApp/rule_manager/benchmark/bench_mqtt.py new file mode 100644 index 00000000..f0aa73d3 --- /dev/null +++ b/HABApp/rule_manager/benchmark/bench_mqtt.py @@ -0,0 +1,110 @@ +import random +import time +from collections import deque +from threading import Lock + +import HABApp +from HABApp.core.events import ValueUpdateEvent +from .bench_base import BenchBaseRule +from .bench_times import BenchContainer, BenchTime +from HABApp.mqtt.interface import publish + +LOCK = Lock() + + +class MqttBenchRule(BenchBaseRule): + BENCH_TYPE = 'MQTT' + + def __init__(self): + super().__init__() + + self.name_list = [f'test/BenchItem{k}' for k in range(300)] + + self.time_sent = 0.0 + self.bench_started = 0.0 + self.bench_times_container = BenchContainer() + self.bench_times: BenchTime = None + + self.name = '' + self.values = deque() + + def cleanup(self): + for n in self.name_list: + if HABApp.core.Items.item_exists(n): + HABApp.core.Items.pop_item(n) + + def set_up(self): + self.cleanup() + + def tear_down(self): + pass + self.cleanup() + + def run(self): + # These are the benchmarks + self.bench_rtt_time() + + def bench_rtt_time(self): + print('Bench events ', end='') + self.bench_times_container = BenchContainer() + + self.run_rtt('rtt idle') + self.run_rtt('async rtt idle', do_async=True) + + # self.start_load() + # self.run_rtt('rtt load (+10x)') + # self.run_rtt('async rtt load (+10x)', do_async=True) + # self.stop_load() + + print(' done!\n') + time.sleep(0.1) + self.bench_times_container.show() + + def run_rtt(self, test_name, do_async=False): + self.name = self.name_list[0] + HABApp.mqtt.items.MqttItem.get_create_item(self.name) + + for i in range(50_000): + self.values.append(random.randint(0, 99999999)) + + listener = self.listen_event( + self.name, self.post_next_event_val if not do_async else self.a_post_next_event_val, ValueUpdateEvent + ) + + self.bench_times = self.bench_times_container.create(test_name) + + self.time_sent = time.time() + publish(self.name, self.values[0]) + + self.run_soon(LOCK.acquire) + time.sleep(1) + LOCK.acquire(True, 5) + + listener.cancel() + if LOCK.locked(): + LOCK.release() + + print('.', end='') + + def post_next_event_val(self, event): + if event.value != self.values[0]: + return None + + self.bench_times.times.append(time.time() - self.time_sent) + + # No items left -> stop benchmark + try: + self.values.popleft() + except IndexError: + LOCK.release() + return None + + if not self.values: + LOCK.release() + return None + + self.time_sent = time.time() + publish(self.name, self.values[0]) + + async def a_post_next_event_val(self, event: ValueUpdateEvent): + self.post_next_event_val(event) diff --git a/HABApp/rule_manager/benchmark/bench_oh.py b/HABApp/rule_manager/benchmark/bench_oh.py new file mode 100644 index 00000000..ca276270 --- /dev/null +++ b/HABApp/rule_manager/benchmark/bench_oh.py @@ -0,0 +1,170 @@ +import random +import time +from collections import deque +from threading import Lock + +import HABApp +from HABApp.core.events import ValueUpdateEvent +from .bench_base import BenchBaseRule +from .bench_times import BenchContainer, BenchTime + +LOCK = Lock() + + +class OpenhabBenchRule(BenchBaseRule): + BENCH_TYPE = 'openHAB' + + def __init__(self): + super().__init__() + + self.name_list = [f'BenchItem{k}' for k in range(300)] + + self.time_sent = 0.0 + self.bench_started = 0.0 + self.bench_times_container = BenchContainer() + self.bench_times: BenchTime = None + + self.item_values = deque() + self.item_name = '' + + self.load_listener = [] + + def cleanup(self): + self.stop_load() + + all_items = set(HABApp.core.Items.get_all_item_names()) + to_rem = set(self.name_list) & all_items + + if not to_rem: + return None + + print('Cleanup ... ', end='') + for name in to_rem: + self.oh.remove_item(name) + print('complete') + + def set_up(self): + self.cleanup() + + def tear_down(self): + self.cleanup() + + def run(self): + # These are the benchmarks + self.bench_item_create() + self.bench_rtt_time() + + def bench_item_create(self): + print('Bench item operations ', end='') + + times = BenchContainer() + + b = times.create('create item') + for k in self.name_list: + start = time.time() + self.openhab.create_item('Number', k, label='MyLabel') + b.times.append(time.time() - start) + + time.sleep(0.2) + + print('.', end='') + b = times.create('update item') + for k in self.name_list: + start = time.time() + self.openhab.create_item('Number', k, label='New Label') + b.times.append(time.time() - start) + + time.sleep(0.2) + + print('.', end='') + b = times.create('delete item') + for k in self.name_list: + start = time.time() + self.openhab.remove_item(k) + b.times.append(time.time() - start) + + print('. done!\n') + times.show() + + def bench_rtt_time(self): + print('Bench item state update ', end='') + self.bench_times_container = BenchContainer() + + self.run_rtt('rtt idle') + self.run_rtt('async rtt idle', do_async=True) + + self.start_load() + self.run_rtt('rtt load (+10x)') + self.run_rtt('async rtt load (+10x)', do_async=True) + self.stop_load() + + print(' done!\n') + self.bench_times_container.show() + + def start_load(self): + + for i in range(10, 20): + def load_cb(event, item=self.name_list[i]): + self.openhab.post_update(item, str(random.randint(0, 99999999))) + + self.openhab.create_item('String', self.name_list[i], label='MyLabel') + listener = self.listen_event(self.name_list[i], load_cb, ValueUpdateEvent) + + self.load_listener.append(listener) + self.openhab.post_update(self.name_list[i], str(random.randint(0, 99999999))) + + def stop_load(self): + for list in self.load_listener: + list.cancel() + self.load_listener.clear() + + def run_rtt(self, test_name, do_async=False): + self.item_name = self.name_list[0] + self.openhab.create_item('String', self.item_name, label='MyLabel') + + for i in range(2000): + self.item_values.append(str(random.randint(0, 99999999))) + + listener = self.listen_event( + self.item_name, self.proceed_item_val if not do_async else self.a_proceed_item_val, ValueUpdateEvent + ) + + self.bench_times = self.bench_times_container.create(test_name) + + self.bench_started = time.time() + self.time_sent = time.time() + self.openhab.post_update(self.item_name, self.item_values[0]) + + self.run_soon(LOCK.acquire) + time.sleep(1) + LOCK.acquire(True, 6) + + listener.cancel() + if LOCK.locked(): + LOCK.release() + + print('.', end='') + + def proceed_item_val(self, event: ValueUpdateEvent): + if event.value != self.item_values[0]: + return None + + self.bench_times.times.append(time.time() - self.time_sent) + + # Time up -> stop benchmark + if time.time() - self.bench_started > 5: + LOCK.release() + return None + + # No items left -> stop benchmark + try: + self.item_values.popleft() + except IndexError: + LOCK.release() + return None + + self.time_sent = time.time() + self.openhab.post_update(self.item_name, self.item_values[0]) + + async def a_proceed_item_val(self, event: ValueUpdateEvent): + self.proceed_item_val(event) diff --git a/HABApp/rule_manager/benchmark/bench_times.py b/HABApp/rule_manager/benchmark/bench_times.py new file mode 100644 index 00000000..f11cc7e4 --- /dev/null +++ b/HABApp/rule_manager/benchmark/bench_times.py @@ -0,0 +1,76 @@ +from statistics import mean, median +from typing import Union + + +def format_duration(duration: Union[None, str, float]) -> str: + if duration is None: + return ' ' * 6 + if isinstance(duration, str): + return f'{duration:^6s}' + + if duration < 0.0001: + # 99.9ns + return f'{duration * 1000 * 1000:4.1f}us' + elif duration < 0.01: + # 9.99ms + return f'{duration * 1000:4.2f}ms' + elif duration < 0.1: + # 99.9ms + return f'{duration * 1000:4.1f}ms' + elif duration < 10: + # 1.234s + return f'{duration:4.3f}s' + else: + # 19.2s + return f'{duration:4.1f}s' + + +class BenchContainer: + def __init__(self): + self.times = [] + + def create(self, name: str) -> 'BenchTime': + c = BenchTime(name) + self.times.append(c) + return c + + def show(self): + indent = max(map(lambda x: len(x.name), self.times), default=0) + BenchTime.show_table(indent) + for b in self.times: + b.show(indent) + print('') + + +class BenchTime: + + @classmethod + def show_table(cls, indent_name=0): + print(f'{"":{indent_name}s} | {format_duration("dur")} | {"per sec":7s} | 'f'{format_duration("median")} | ' + f'{format_duration("min")} | {format_duration("max")} | {format_duration("mean")}') + + def __init__(self, name: str, factor: int = 1): + self.name = name + self.times = [] + self.factor = factor + + def show(self, indent_name=0): + total = sum(self.times) + count = len(self.times) + _mean = mean(self.times) if self.times else 0 + _medi = median(self.times) if self.times else 0 + + per_sec = ((count * self.factor) / total) if total else 0 + unit = '' + if per_sec > 1000: + per_sec /= 1000 + unit = 'k' + if per_sec > 1000: + per_sec /= 1000 + unit = 'm' + + print(f'{self.name:>{indent_name}s} | {format_duration(total)} | ' + f'{per_sec:{7 - len(unit)}.{3 - len(unit)}f}{unit} | ' + f'{format_duration(_medi)} | {format_duration(min(self.times, default=0))} | ' + f'{format_duration(max(self.times, default=0))} | ' + f'{format_duration(_mean)}') diff --git a/HABApp/rule_manager/rule_file.py b/HABApp/rule_manager/rule_file.py index 094cba11..c61f9336 100644 --- a/HABApp/rule_manager/rule_file.py +++ b/HABApp/rule_manager/rule_file.py @@ -59,6 +59,15 @@ def __process_tc(self, tb: list): tb.insert(0, f"Could not load {self.path}!") return [line.replace('', self.path.name) for line in tb] + def create_rules(self, created_rules: list): + # It seems like python 3.8 doesn't allow path like objects any more: + # https://github.com/spacemanspiff2007/HABApp/issues/111 + runpy.run_path(str(self.path), run_name=str(self.path), init_globals={ + '__HABAPP__RUNTIME__': self.rule_manager.runtime, + '__HABAPP__RULE_FILE__': self, + '__HABAPP__RULES': created_rules, + }) + def load(self) -> bool: created_rules = [] @@ -67,13 +76,7 @@ def load(self) -> bool: ign.proc_tb = self.__process_tc with ign: - # It seems like python 3.8 doesn't allow path like objects any more: - # https://github.com/spacemanspiff2007/HABApp/issues/111 - runpy.run_path(str(self.path), run_name=str(self.path), init_globals={ - '__HABAPP__RUNTIME__': self.rule_manager.runtime, - '__HABAPP__RULE_FILE__': self, - '__HABAPP__RULES': created_rules, - }) + self.create_rules(created_rules) if ign.raised_exception: # unload all rule instances which might have already been created otherwise they might diff --git a/HABApp/rule_manager/rule_manager.py b/HABApp/rule_manager/rule_manager.py index aa14928f..fff77a5b 100644 --- a/HABApp/rule_manager/rule_manager.py +++ b/HABApp/rule_manager/rule_manager.py @@ -15,6 +15,7 @@ from HABApp.core.logger import log_warning from HABApp.core.wrapper import log_exception from .rule_file import RuleFile +import HABApp.__cmd_args__ as cmd_args log = logging.getLogger('HABApp.Rules') @@ -51,6 +52,16 @@ def __init__(self, parent): def setup(self): + if cmd_args.DO_BENCH: + from HABApp.rule_manager.benchmark import BenchFile + self.files['bench'] = file = BenchFile(self) + if not file.load(): + log.error('Failed to load Benchmark!') + HABApp.runtime.shutdown.request_shutdown() + return None + file.check_all_rules() + return + # Add event bus listener HABApp.core.files.add_event_bus_listener('rule', self.request_file_load, self.request_file_unload, log) @@ -72,7 +83,7 @@ def load_rules_on_startup(self): break # stop waiting if we want to shut down - if self.runtime.shutdown.requested: + if HABApp.runtime.shutdown.requested: return None time.sleep(2.2) else: @@ -85,7 +96,7 @@ def load_rules_on_startup(self): @log_exception async def process_scheduled_events(self): - while not self.runtime.shutdown.requested: + while not HABApp.runtime.shutdown.requested: now = datetime.datetime.now(tz=utc) diff --git a/HABApp/runtime/__init__.py b/HABApp/runtime/__init__.py index 7c602c51..f55b1c69 100644 --- a/HABApp/runtime/__init__.py +++ b/HABApp/runtime/__init__.py @@ -1,2 +1,2 @@ -from .shutdown_helper import ShutdownHelper +from . import shutdown from .runtime import Runtime \ No newline at end of file diff --git a/HABApp/runtime/runtime.py b/HABApp/runtime/runtime.py index b80b5057..0ea9ce39 100644 --- a/HABApp/runtime/runtime.py +++ b/HABApp/runtime/runtime.py @@ -8,14 +8,12 @@ import HABApp.rule_manager import HABApp.util from HABApp.openhab import connection_logic as openhab_connection -from .shutdown_helper import ShutdownHelper +from HABApp.runtime import shutdown class Runtime: def __init__(self): - self.shutdown = ShutdownHelper() - self.config: HABApp.config.Config = None self.async_http: HABApp.rule.interfaces.AsyncHttpConnection = HABApp.rule.interfaces.AsyncHttpConnection() @@ -28,21 +26,21 @@ def __init__(self): # Async Workers & shutdown callback HABApp.core.WrappedFunction._EVENT_LOOP = HABApp.core.const.loop - self.shutdown.register_func(HABApp.core.WrappedFunction._WORKERS.shutdown) + shutdown.register_func(HABApp.core.WrappedFunction._WORKERS.shutdown, msg='Stopping workers') def startup(self, config_folder: Path): # Start Folder watcher! - HABApp.core.files.watcher.start(self.shutdown) + HABApp.core.files.watcher.start() self.config_loader = HABApp.config.HABAppConfigLoader(config_folder) # MQTT - HABApp.mqtt.mqtt_connection.setup(self.shutdown) + HABApp.mqtt.mqtt_connection.setup() HABApp.mqtt.mqtt_connection.connect() # openhab - openhab_connection.setup(self.shutdown) + openhab_connection.setup() # Parameter Files HABApp.parameters.parameter_files.setup_param_files() diff --git a/HABApp/runtime/shutdown.py b/HABApp/runtime/shutdown.py new file mode 100644 index 00000000..424c02b4 --- /dev/null +++ b/HABApp/runtime/shutdown.py @@ -0,0 +1,64 @@ +import itertools +import logging +import logging.handlers +import traceback +import typing +from asyncio import iscoroutinefunction, run_coroutine_threadsafe, sleep +from dataclasses import dataclass +from types import FunctionType, MethodType +from typing import Callable, Coroutine, Union + +from HABApp.core.const import loop + + +@dataclass(frozen=True) +class ShutdownInfo: + func: Union[Callable[[], typing.Any], Coroutine] + msg: str + last: bool + + +_FUNCS: typing.List[ShutdownInfo] = [] + +requested: bool = False + + +def register_func(func, last=False, msg: str = ''): + assert isinstance(func, (FunctionType, MethodType)) or iscoroutinefunction(func), print(type(func)) + assert last is True or last is False, last + assert isinstance(msg, str) + + _FUNCS.append(ShutdownInfo(func, f'{func.__module__}.{func.__name__}' if not msg else msg, last)) + + +def request_shutdown(): + run_coroutine_threadsafe(_shutdown(), loop) + + +async def _shutdown(): + global requested + "Request execution of all functions" + + log = logging.getLogger('HABApp.Shutdown') + log.debug('Requested shutdown') + + requested = True + + for obj in itertools.chain(filter(lambda x: not x.last, _FUNCS), + filter(lambda x: x.last, _FUNCS)): + try: + log.debug(f'{obj.msg}') + if iscoroutinefunction(obj.func): + await obj.func() + else: + obj.func() + log.debug('-> done!') + await sleep(0.02) + except Exception as ex: + log.error(ex) + tb = traceback.format_exc().splitlines() + for line in tb: + log.error(line) + + log.debug('Shutdown complete') + return None diff --git a/HABApp/runtime/shutdown_helper.py b/HABApp/runtime/shutdown_helper.py deleted file mode 100644 index d2ad6b6c..00000000 --- a/HABApp/runtime/shutdown_helper.py +++ /dev/null @@ -1,40 +0,0 @@ -import itertools -import logging -import traceback - - -class ShutdownHelper: - def __init__(self, name=''): - self.__cb_funcs = [] - self.__cb_last = [] - - self.requested = False - - def register_func(self, func, last=False): - assert callable(func) - if last: - self.__cb_last.append(func) - else: - self.__cb_funcs.append(func) - - def request_shutdown(self): - "Request execution of all functions" - - log = logging.getLogger('HABApp.Shutdown') - log.debug('Requested shutdown') - - self.requested = True - - for func in itertools.chain(self.__cb_funcs, self.__cb_last): - try: - log.debug(f'Calling {func.__name__}') - func() - log.debug(f'{func.__name__} done!') - except Exception as ex: - log.error(ex) - tb = traceback.format_exc().splitlines() - for line in tb: - log.error(line) - - log.debug('Shutdown complete') - return None diff --git a/_doc/__style_guide__ b/_doc/__style_guide__ new file mode 100644 index 00000000..622fd8fa --- /dev/null +++ b/_doc/__style_guide__ @@ -0,0 +1,22 @@ +************************************** +Structural Elements +************************************** + +Document Section +====================================== + +Document Subsection +-------------------------------------- + +Document Subsubsection +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +Document Paragraph +"""""""""""""""""""""""""""""""""""""" + + +************************************** +Structural Elements 2 +************************************** + +... \ No newline at end of file diff --git a/_doc/_plugins/sphinx_execute_code.py b/_doc/_plugins/sphinx_execute_code.py index 6956f07e..fff2fab4 100644 --- a/_doc/_plugins/sphinx_execute_code.py +++ b/_doc/_plugins/sphinx_execute_code.py @@ -18,13 +18,16 @@ print 'Execute this python code' """ import functools +import re import subprocess import sys import traceback from pathlib import Path +from typing import Optional from docutils import nodes from docutils.parsers.rst import Directive, directives +from sphinx.errors import ExtensionError from sphinx.util import logging log = logging.getLogger(__name__) @@ -36,12 +39,29 @@ def PrintException( func): def f(*args, **kwargs): try: return func(*args, **kwargs) + except ExtensionError: + raise except Exception as e: - print("{}\n{}".format( e, traceback.format_exc())) + print("\n{}\n{}".format( e, traceback.format_exc())) raise return f +re_line = re.compile(r'line (\d+),') + + +class CodeException(Exception): + def __init__(self, ret: int, stderr: str): + self.ret = ret + self.err = stderr + + self.line: Optional[int] = None + + # Find the last line where the error happened + for m in re_line.finditer(self.err): + self.line = int(m.group(1)) + + class ExecuteCode(Directive): """ Sphinx class for execute_code directive """ @@ -90,6 +110,7 @@ def run(self): executed_code += line + '\n' shown_code = shown_code.strip() + executed_code = executed_code.strip() # Show the example code if 'hide_code' not in self.options: @@ -105,7 +126,27 @@ def run(self): if 'header_output' in self.options: output.append(nodes.caption(text=self.options['header_output'])) - code_results = execute_code( executed_code, ignore_stderr='ignore_stderr' in self.options) + try: + code_results = execute_code(executed_code, ignore_stderr='ignore_stderr' in self.options) + except CodeException as e: + # Newline so we don't have the build message mixed up with logs + print('\n') + + code_lines = executed_code.splitlines() + + # If we don't get the line we print everything + if e.line is None: + e.line = len(code_lines) + + for i in range(max(0, e.line - 8), e.line - 1): + log.error(f' {code_lines[i]}') + log.error(f' {code_lines[e.line - 1]} <--') + + log.error('') + for line in e.err.splitlines(): + log.error(line) + + raise ExtensionError('Could not execute code!') from None if 'ignore_stderr' not in self.options: for out in code_results.split('\n'): @@ -125,14 +166,14 @@ def run(self): WORKING_DIR = None -@PrintException def execute_code(code, ignore_stderr) -> str: run = subprocess.run([sys.executable, '-c', code], capture_output=True, cwd=WORKING_DIR) if run.returncode != 0: - print(f'stdout: {run.stdout.decode()}') - print(f'stderr: {run.stderr.decode()}') - raise ValueError() + # print('') + # print(f'stdout: {run.stdout.decode()}') + # print(f'stderr: {run.stderr.decode()}') + raise CodeException(run.returncode, run.stderr.decode()) from None if ignore_stderr: return run.stdout.decode().strip() diff --git a/_doc/_static/theme_overrides.css b/_doc/_static/theme_overrides.css index 46a1bab0..67bec8f1 100644 --- a/_doc/_static/theme_overrides.css +++ b/_doc/_static/theme_overrides.css @@ -2,7 +2,7 @@ @media screen and (min-width: 767px) { .wy-nav-content { - max-width: 1000px !important; + max-width: 1100px !important; } .wy-table-responsive table td { @@ -14,4 +14,12 @@ .wy-table-responsive { overflow: visible !important; } -} \ No newline at end of file +} + + +h1 { font-size: 200% } +h2 { font-size: 180% } +h3 { font-size: 160% } +h4 { font-size: 140% } +h5 { font-size: 120% } +h6 { font-size: 100% } diff --git a/_doc/about_habapp.rst b/_doc/about_habapp.rst index a419b21d..0c5325c4 100644 --- a/_doc/about_habapp.rst +++ b/_doc/about_habapp.rst @@ -21,19 +21,20 @@ HABApp folder structure Integration with openHAB ------------------------------ -HABApp connects to the openhab event stream and automatically updates the local items when an item in openhab changes. +HABApp connects to the openhab event stream and automatically updates the local openhab items when an item in openhab changes. These item values are cached, so accessing and working with items in rules is very fast. The events from openhab are also mirrored to the internal event bus which means that triggering on these events is also possible. -When HABApp connects to openhab for the first time it will load all items/things from the openhab instance. +When HABApp connects to openhab for the first time it will load all items/things from the openhab instance and create local items. +The name of the local openhab items is equal to the name in openhab. Posting updates, sending commands or any other openhab interface call will issue a corresponding REST-API call to change openhab. Integration with MQTT ------------------------------ HABApp subscribes to the defined mqtt topics. For every MQTT message with the ``retain`` flag HABApp will automatically -create an :class:`~HABApp.mqtt.items.MqttItem` so these values can be accessed later. +create an :class:`~HABApp.mqtt.items.MqttItem` so these values can be accessed later. The name of the created item is the the mqtt topic. All other messages will **not** automatically create an item but still create an event on the event bus. MqttItems created by rules will automatically be updated with the latest value once a message is received. diff --git a/_doc/advanced_usage.rst b/_doc/advanced_usage.rst index 4e6c3545..12727797 100644 --- a/_doc/advanced_usage.rst +++ b/_doc/advanced_usage.rst @@ -30,7 +30,7 @@ An example would be dynamically reloading files or an own notifier in case there * - HABApp.Errors - All errors in functions and rules of HABApp create an according event. Use this topic to create an own notifier in case of errors (e.g. Pushover). - - :class:`~HABApp.core.events.file_events.HABAppError` or ``str`` + - :class:`~HABApp.core.events.habapp_events.HABAppError` or ``str`` @@ -107,7 +107,7 @@ And since it is just like a normal item triggering on changes etc. is possible, :hide_output: from HABApp.core.items import AggregationItem - my_agg = AggregationItem('MyAggregationItem') + my_agg = AggregationItem.get_create_item('MyAggregationItem') # Connect the source item with the aggregation item my_agg.aggregation_source('MyInputItem') @@ -118,6 +118,7 @@ And since it is just like a normal item triggering on changes etc. is possible, # Use max as an aggregation function my_agg.aggregation_func = max + The value of ``my_agg`` in the example will now always be the maximum of ``MyInputItem`` in the last two hours. It will automatically update and always reflect the latest changes of ``MyInputItem``. @@ -127,7 +128,7 @@ It will automatically update and always reflect the latest changes of ``MyInputI Invoking OpenHAB actions ------------------------ -The openhab REST interface does not expose _actions: https://www.openhab.org/docs/configuration/actions.html, +The openhab REST interface does not expose `actions `_, and thus there is no way to trigger them from HABApp. If it is not possible to create and OpenHAB item that directly triggers the action there is a way to work around it with additional items within openhab. An additional OpenHAB (note not HABapp) rule listens to changes on those items and invokes the appropriate diff --git a/_doc/class_reference.rst b/_doc/class_reference.rst index d166d7cd..9c38fdcb 100644 --- a/_doc/class_reference.rst +++ b/_doc/class_reference.rst @@ -1,12 +1,16 @@ - +************************************** Class reference -================================== +************************************** Reference for returned classes from some functions. These are not intended to be created by the user. +Watches +====================================== + + ItemNoUpdateWatch ---------------------------------- +"""""""""""""""""""""""""""""""""""""" .. autoclass:: HABApp.core.items.base_item_watch.ItemNoUpdateWatch :members: @@ -14,7 +18,7 @@ ItemNoUpdateWatch :member-order: groupwise ItemNoChangeWatch ---------------------------------- +"""""""""""""""""""""""""""""""""""""" .. autoclass:: HABApp.core.items.base_item_watch.ItemNoChangeWatch :members: diff --git a/_doc/index.rst b/_doc/index.rst index d6a5753b..0b92cb60 100644 --- a/_doc/index.rst +++ b/_doc/index.rst @@ -20,6 +20,7 @@ Welcome to the HABApp documentation! asyncio util rule_examples + tips class_reference diff --git a/_doc/installation.rst b/_doc/installation.rst index 25a74bfb..541e4e74 100644 --- a/_doc/installation.rst +++ b/_doc/installation.rst @@ -190,7 +190,7 @@ HABApp arguments # hide import HABApp.__main__ - HABApp.__main__.get_command_line_args(['-h']) + HABApp.__cmd_args__.parse_args(['-h']) # hide diff --git a/_doc/interface_openhab.rst b/_doc/interface_openhab.rst index caaf0ad6..eb11190d 100644 --- a/_doc/interface_openhab.rst +++ b/_doc/interface_openhab.rst @@ -1,10 +1,12 @@ .. _ref_openhab: +************************************** openHAB -================================== +************************************** + Interaction with a openHAB ------------------------------- +====================================== All interaction with the openHAB is done through the ``self.oh`` or ``self.openhab`` object in the rule or through an `````OpenhabItem```. @@ -13,7 +15,7 @@ or through an `````OpenhabItem```. Function parameters ------------------------------- +-------------------------------------- .. automodule:: HABApp.openhab.interface :members: :imported-members: @@ -22,10 +24,10 @@ Function parameters .. _OPENHAB_ITEM_TYPES: Openhab item types ------------------------------- +====================================== Description and example -~~~~~~~~~~~~~~~~~~~~~~~~ +-------------------------------------- Openhab items are mapped to special classes and provide convenience functions. @@ -51,7 +53,7 @@ Example: NumberItem -~~~~~~~~~~~~~~~~~~~~~~~~ +-------------------------------------- .. inheritance-diagram:: HABApp.openhab.items.NumberItem :parts: 1 @@ -62,7 +64,7 @@ NumberItem ContactItem -~~~~~~~~~~~~~~~~~~~~~~~~ +-------------------------------------- .. inheritance-diagram:: HABApp.openhab.items.ContactItem :parts: 1 @@ -73,7 +75,7 @@ ContactItem SwitchItem -~~~~~~~~~~~~~~~~~~~~~~~~ +-------------------------------------- .. inheritance-diagram:: HABApp.openhab.items.SwitchItem :parts: 1 @@ -84,7 +86,7 @@ SwitchItem DimmerItem -~~~~~~~~~~~~~~~~~~~~~~~~ +-------------------------------------- .. inheritance-diagram:: HABApp.openhab.items.DimmerItem :parts: 1 @@ -95,7 +97,7 @@ DimmerItem RollershutterItem -~~~~~~~~~~~~~~~~~~~~~~~~ +-------------------------------------- .. inheritance-diagram:: HABApp.openhab.items.RollershutterItem :parts: 1 @@ -106,7 +108,7 @@ RollershutterItem ColorItem -~~~~~~~~~~~~~~~~~~~~~~~~ +-------------------------------------- .. inheritance-diagram:: HABApp.openhab.items.ColorItem :parts: 1 @@ -117,7 +119,7 @@ ColorItem StringItem -~~~~~~~~~~~~~~~~~~~~~~~~ +-------------------------------------- .. inheritance-diagram:: HABApp.openhab.items.StringItem :parts: 1 @@ -128,7 +130,7 @@ StringItem LocationItem -~~~~~~~~~~~~~~~~~~~~~~~~ +-------------------------------------- .. inheritance-diagram:: HABApp.openhab.items.LocationItem :parts: 1 @@ -139,7 +141,7 @@ LocationItem PlayerItem -~~~~~~~~~~~~~~~~~~~~~~~~ +-------------------------------------- .. inheritance-diagram:: HABApp.openhab.items.PlayerItem :parts: 1 @@ -150,7 +152,7 @@ PlayerItem GroupItem -~~~~~~~~~~~~~~~~~~~~~~~~ +-------------------------------------- .. inheritance-diagram:: HABApp.openhab.items.GroupItem :parts: 1 @@ -161,7 +163,7 @@ GroupItem ImageItem -~~~~~~~~~~~~~~~~~~~~~~~~ +-------------------------------------- .. inheritance-diagram:: HABApp.openhab.items.ImageItem :parts: 1 @@ -172,7 +174,7 @@ ImageItem Thing -~~~~~~~~~~~~~~~~~~~~~~~~ +-------------------------------------- .. inheritance-diagram:: HABApp.openhab.items.Thing :parts: 1 @@ -183,21 +185,16 @@ Thing Textual thing configuration ------------------------------- +====================================== Description -~~~~~~~~~~~~~~~~~~~~~~~~ +-------------------------------------- HABApp offers a special mechanism to textually define thing configuration parameters and linked items for things which have been added through the gui. This combines the best of both worlds: auto discovery, easy and fast sharing of parameters and items across things. -.. WARNING:: - The value of the configuration parameters will not be checked and will be written as specified. - It is recommended to use HABmin or PaperUI to generate the initial configuration and use this mechanism to spread - it to things of the same type. - Configuration is done in the ``thing_your_name.yml`` file in the ``config`` folder (see :doc:`configuration`). Every file that starts with ``thing_`` has the ``.yml`` ending will be loaded. @@ -205,27 +202,31 @@ The Parameters and items will be checked/set when HABApp connects to openHAB or whenever the corresponding file gets changed. Principle of operation -~~~~~~~~~~~~~~~~~~~~~~~~ +-------------------------------------- + All existing things from openHAB can be filtered by different criteria. -For each one of these things it is then possible to +For each one of these remaining things it is then possible to * Set thing parameters -* Create items with wildcards taken from the thing -* | Apply filters to the channels of the thing. - | For each matching channel it is possible to link items with wildcards taken from the thing and the matching channel +* Create items with values taken from the thing fields +* | Apply filters to the channels of the thing + | For each matching channel it is possible to create and link items with values taken from the thing and the matching channel values + There is also a test mode which prints out all required information and does not make any changes. A valid ``.items`` file will automatically be created next to the ``.yml`` file containing all created items. It can be used to get a quick overview what items (would) have been created or copied into the items folder. + + File Structure -~~~~~~~~~~~~~~~~~~~~ +-------------------------------------- Configuration is done through a .yml file. -Example -""""""""""""""""""""""""""""" +Example +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ The following example will show how to set the Z-Wave Parameters 4, 5, 6 and 8 for a ``Philio PST02A`` Z-Wave sensor and how to automatically link items to it. @@ -282,8 +283,8 @@ The entries ``thing config``, ``create items`` and ``channels`` are optional and icon: battery -Multiple thing definitions in one file -"""""""""""""""""""""""""""""""""""""""" +Multiple and filter definitions in one file +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ It is possible to add multiple thing processors into one file. To achieve this the root entry is now a list. @@ -306,8 +307,68 @@ Filters can also be lists e.g. if the have to be applied multiple times to the s ... -Adding Metadata to items -""""""""""""""""""""""""""""" +Thing configuration +-------------------------------------- + +With the ``thing config`` block it is possible to set a configuration for each matching thing. +If the parameters are already correct, they will not be set again. + +.. WARNING:: + The value of the configuration parameters will not be checked and will be written as specified. + It is recommended to use HABmin or PaperUI to generate the initial configuration and use this mechanism to spread + it to things of the same type. + + +Example +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +.. code-block:: yaml + + thing config: + 4: 99 # Light Threshold + 5: 8 # Operation Mode + 6: 4 # MultiSensor Function Switch + 7: 20 # Customer Function + +References to other parameters +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +It is possible to use references to mathematically build parameters from other parameters. +Typically this would be fade duration and refresh interval. +References to other parameter values can be created with ``$``. +Example: + +.. code-block:: yaml + + thing config: + 5: 8 + 6: '$5 / 2' # Use value from parameter 5 and divide it by two. + 7: 'int($5 / 2)' # it is possible to use normal python data conversions + + +Item configuration +-------------------------------------- + +Items can be configured under ``create items -> []`` and ``channels -> [] -> link items -> []``. + +Structure +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +Mandatory values are ``type`` and ``name``, all other values are optional. + +.. code-block:: yaml + + type: Number + name: my_name + label: my_label + icon: my_icon + groups: ['group1', 'group2'] + tags: ['tag1', 'tag1'] + + +.. _ref_textual_thing_config_metadata: + +Metadata +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ It is possible to add metadata to the created items through the optional ``metadata`` entry in the item config. There are two forms how metadata can be set. The implicit form for simple key-value pairs (e.g. ``autoupdate``) or @@ -335,42 +396,68 @@ The config is equivalent to the following item configuration:: +Fields +-------------------------------------- -References in thing config -~~~~~~~~~~~~~~~~~~~~~~~~~~ -It is possible to use references to mathematically build parameters from other parameters. -Typically this would be fade duration and refresh interval. -References to other parameter values can be created with ``$``. -Example: + +Filtering things/channels +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +The filter value can be applied to any available field from the Thing/Channel. +The filter value is a regex that has to fully match the value. + +Syntax: .. code-block:: yaml - thing config: - 5: 8 - 6: '$5 / 2' # Use value from parameter 5 and divide it by two. - 7: 'int($5 / 2)' # it is possible to use normal python datatypes + filter: + FIELD_NAME: REGULAR_EXPRESSION + +e.g. +.. code-block:: yaml + + filter: + thing_uid: zwave:device:controller:node35 -Wildcards for items and channels -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -Wildcards are available for item configuration and can be applied to all fields except for 'type' and 'metadata'. +If multiple filters are specified all have to match to select the Thing or Channel. + +.. code-block:: yaml + + # Multiple filters on different columns + filter: + thing_type: zwave:fibaro.+ + thing_uid: zwave:device:controller:node35 + + # Multiple filters on the same columns (rarely needed) + filter: + - thing_type: zwave:fibaro.+ + - thing_type: zwave:fibaro_fgrgbw_00_000 + + +Field values as inputs for items and channels +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +Filed values are available for item configuration and can be applied to all fields except for ``type`` and ``metadata``. Syntax -""""""""""""" -Wildcards are framed with ``{}`` so the containing string has to be put in annotation marks. +"""""""""""""""""""""""""""""""""""""" +Macros that select field values are framed with ``{}`` so the containing string has to be put in annotation marks. There are three modes of operation with wildcards: -1. | Just insert the value from the wildcard: - | ``{wildcard}`` -2. | Insert a part of the value from the wildcard. A regular expression is used to extract the part and +1. | Just insert the value from the field: + | ``{field}`` +2. | Insert a part of the value from the field. A regular expression is used to extract the part and therefore has to contain a capturing group. - | ``{wildcard, regex(with_group)}`` -3. | Do a regex, replace on the value from the wildcard and use the result - | ``{wildcard, regex, replace}`` + | ``{field, regex(with_group)}`` +3. | Do a regex replace on the value from the field and use the result + | ``{field, regex, replace}`` -Available wildcards -""""""""""""""""""""" -The following wildcards are available for things: +Available fields +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +.. tip:: + Test mode will show a table with all available fields and their value + +The following fields are available for things: * ``thing_uid`` * ``thing_type`` @@ -378,22 +465,20 @@ The following wildcards are available for things: * ``thing_label`` * ``bridge_uid`` -Additional available wildcards for channels: +Additional available fields for channels: * ``channel_uid`` * ``channel_type`` * ``channel_label`` * ``channel_kind`` -.. tip:: - Test mode will show a table with all available wildcards and their value Example -~~~~~~~~~~ +-------------------------------------- Log output -""""""""""""""""""""" +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ This will show the output for the example from `File Structure`_ .. code-block:: text @@ -498,7 +583,7 @@ This will show the output for the example from `File Structure`_ Created items file -""""""""""""""""""""" +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ .. code-block:: text @@ -519,44 +604,24 @@ Created items file Number FrontDoor_Temperature "FrontDoor Temperature [%d %%]" {channel = "zwave:device:controller:node5:sensor_temperature"} -Entry sharing -"""""""""""""""" - -If the values should be reused `yml features anchors `_ -with ``&`` which then can be referenced with ``*``. This allows to reuse the defined structures: - -.. code-block:: yaml - - my_key_value_pairs: &my_kv # <-- this creates the anchor node with the name my_kv - 4: 99 # Light Threshold - 5: 8 # Operation Mode - 7: 20 # Customer Function - - value_1: *my_kv # <-- '*my_kv' references the anchor node my_kv - value_2: *my_kv - value_3: - <<: *my_kv # <-- '<<: *my_kv' references and inserts the content (!) of the anchor node my_kv - 4: 80 # and then overwrites parameter 4 - - Example openHAB rules ---------------------- +====================================== Example 1 -~~~~~~~~~~~~~~~~~~~~~~~~ +-------------------------------------- .. literalinclude:: ../conf/rules/openhab_rule.py Check status of things -~~~~~~~~~~~~~~~~~~~~~~~~ +-------------------------------------- This rule prints the status of all ``Things`` and shows how to subscribe to events of the ``Thing`` status .. literalinclude:: ../conf/rules/openhab_things.py Check status if thing is constant -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +-------------------------------------- Sometimes ``Things`` recover automatically from small outages. This rule only triggers then the ``Thing`` is constant for 60 seconds. diff --git a/_doc/rule.rst b/_doc/rule.rst index 2eb06a5b..b3cfba78 100644 --- a/_doc/rule.rst +++ b/_doc/rule.rst @@ -71,36 +71,95 @@ It is possible to check the item value by comparing it Events ------------------------------ -.. list-table:: - :widths: auto - :header-rows: 1 +It is possible to listen to events through the :meth:`~HABApp.Rule.listen_event` function. +The passed function will be called as soon as an event occurs and the event will pe passed as an argument +into the function. - * - Function - - Description +There is the possibility to reduce the function calls to a certain event type with an additional parameter +(typically :class:`~HABApp.core.ValueUpdateEvent` or :class:`~HABApp.core.ValueChangeEvent`). - * - :meth:`~HABApp.Rule.listen_event` - - Add a function which will be called as soon as an event occurs. - The event will be passed as an argument into the function. - There is the possibility to specify the event class which will reduce the function calls accordingly - (typically :class:`~HABApp.core.ValueUpdateEvent` or :class:`~HABApp.core.ValueChangeEvent`). +.. execute_code:: + :hide_output: + :header_code: Example -Example:: + # hide + import time, HABApp + from tests import SimpleRuleRunner + runner = SimpleRuleRunner() + runner.set_up() + HABApp.core.Items.create_item('MyItem', HABApp.core.items.Item) + # hide + from HABApp import Rule + from HABApp.core.events import ValueChangeEvent, ValueUpdateEvent + from HABApp.core.items import Item + + class MyRule(Rule): + def __init__(self): + super().__init__() + self.listen_event('MyOpenhabItem', self.on_change, ValueChangeEvent) + self.listen_event('My/MQTT/Topic', self.on_update, ValueUpdateEvent) + + # If you already have an item you can and should use the more convenient method of the item + # to listen to the item events + my_item = Item.get_item('MyItem') + my_item.listen_event(self.on_change, ValueUpdateEvent) + + def on_change(self, event: ValueChangeEvent): + assert isinstance(event, ValueChangeEvent), type(event) + + def on_update(self, event: ValueUpdateEvent): + assert isinstance(event, ValueUpdateEvent), type(event) + + MyRule() + +Additionally there is the possibility to filter not only on the event type but on the event values, too. +This can be achieved by passing an **instance** of EventFilter as event type. +There are convenience Filters (e.g. :class:`~HABApp.core.events.ValueUpdateEventFilter` and +:class:`~HABApp.core.events.ValueChangeEventFilter`) for the most used event types that provide type hints. + + + +.. autoclass:: HABApp.core.events.EventFilter + :members: + +.. autoclass:: HABApp.core.events.ValueUpdateEventFilter + :members: + +.. autoclass:: HABApp.core.events.ValueChangeEventFilter + :members: + + +.. execute_code:: + :hide_output: + :header_code: Example + + # hide + import time, HABApp + from tests import SimpleRuleRunner + runner = SimpleRuleRunner() + runner.set_up() + HABApp.core.Items.create_item('MyItem', HABApp.core.items.Item) + # hide + from HABApp import Rule + from HABApp.core.events import EventFilter, ValueUpdateEventFilter, ValueUpdateEvent + from HABApp.core.items import Item + + class MyRule(Rule): + def __init__(self): + super().__init__() + my_item = Item.get_item('MyItem') + + # This will only to ValueUpdateEvents where the value==my_value + my_item.listen_event(self.on_val_my_value, ValueUpdateEventFilter(value='my_value')) - def __init__(self): - super().__init__() - self.listen_event('MyOpenhabItem', self.on_change, ValueChangeEvent) - self.listen_event('My/MQTT/Topic', self.on_update, ValueUpdateEvent) + # This is the same as above but with the generic filter + my_item.listen_event(self.on_val_my_value, EventFilter(ValueUpdateEvent, value='my_value')) - # If you already have an item you can use the more convenient method of the item - # to listen to the item events - my_item = Item.get_item('MyItem') - my_item.listen_event(self.on_change, ValueUpdateEvent) + def on_val_my_value(self, event: ValueUpdateEvent): + assert isinstance(event, ValueUpdateEvent), type(event) - def on_change(self, event): - assert isinstance(event, ValueChangeEvent), type(event) + MyRule() - def on_update(self, event): - assert isinstance(event, ValueUpdateEvent), type(event) Scheduler ------------------------------ diff --git a/_doc/tips.rst b/_doc/tips.rst new file mode 100644 index 00000000..19c5fcb2 --- /dev/null +++ b/_doc/tips.rst @@ -0,0 +1,49 @@ +************************************** +Tips & Tricks +************************************** + + +yml files +====================================== + +Entry sharing +-------------------------------------- + +If the values should be reused `yml features anchors `_ +with ``&`` which then can be referenced with ``*``. This allows to reuse the defined structures: + +.. code-block:: yaml + + my_key_value_pairs: &my_kv # <-- this creates the anchor node with the name my_kv + 4: 99 # Light Threshold + 5: 8 # Operation Mode + 7: 20 # Customer Function + + value_1: *my_kv # <-- '*my_kv' references the anchor node my_kv + value_2: *my_kv + + value_3: + <<: *my_kv # <-- '<<: *my_kv' references and inserts the content (!) of the anchor node my_kv + 4: 80 # and then overwrites parameter 4 + + + +openHAB +====================================== + +autoupdate +-------------------------------------- + +If external devices are capable of reporting their state (e.g. Z-Wave) it is always advised to use disable ``autoupdate`` for these items. +This prevents openhab from guessing the item state based on the command and forces it to use the actual reported value. +If in doubt if the device supports reporting their state watch the state after sending a command with ``autoupdate`` off. +If the state changes ``autoupdate`` can remain off. + + +In the ``*.items`` file ``autoupdate`` can be disabled by adding the following statement in the metadata field. + +``` +Number MyItem { channel = "zwave:my_zwave_link", autoupdate="false" } +``` + +It's also possible with textual thing configuration to add it as :ref:`_ref_textual_thing_config_metadata`. diff --git a/conf_testing/rules/bench_rule.py b/conf_testing/rules/bench_rule.py deleted file mode 100644 index f2961a29..00000000 --- a/conf_testing/rules/bench_rule.py +++ /dev/null @@ -1,79 +0,0 @@ -import asyncio -import datetime -import random -import statistics -import time - -import HABApp -from HABApp.core.events import ValueChangeEvent, ValueUpdateEvent -from HABApp.openhab.items import NumberItem - -WAIT_PREPARE = 5 * 60 -RUN_EVERY = 2 * 60 - - -class OpenhabBenchRule(HABApp.Rule): - - def __init__(self): - super().__init__() - - self.item_list = [f'BenchItem{k}' for k in range(300)] - - self.__b_start = 0 - self.__b_val = random.randint(0, 9999) - - self.run_in(WAIT_PREPARE, self.prepare_bench) - - self.pings = [] - self.ping_item = NumberItem.get_item('Ping') - self.ping_item.listen_event(self.ping_received, ValueUpdateEvent) - - def prepare_bench(self): - print('') - print('Benchmark item creation') - - start = time.time() - for k in self.item_list: - self.openhab.create_item('String', k) - - self.run_every(None, datetime.timedelta(seconds=RUN_EVERY), self.bench_start) - self.listen_event(self.item_list[-1], self.bench_stop, ValueChangeEvent) - - dur = time.time() - start - print(f'Updated {len(self.item_list)} item definitions in {dur:.3f}s' - f' --> {len(self.item_list)/dur:5.3f} updates per sec') - - def ping_received(self, event: ValueUpdateEvent): - self.pings.append(event.value) - - def bench_start(self): - print('') - print('Benchmark value update creation') - self.__b_val = str(random.randint(0, 99999999)) - self.__b_start = time.time() - for k in self.item_list: - self.openhab.post_update(k, self.__b_val) - - async def bench_stop(self, event): - if event.value != self.__b_val: - return None - - ts_start = time.time() - while True: - for k in self.item_list: - if HABApp.openhab.items.OpenhabItem.get_item(k).value != self.__b_val: - break - else: - break - - await asyncio.sleep(0.1) - - print(f'Wait: {time.time() - ts_start:.3f}') - dur = time.time() - self.__b_start - print(f'Benchmark duration: {dur:.3f}s --> {len(self.item_list)/dur:5.3f} updates per sec') - print(f'Pings ({len(self.pings)}): min: {min(self.pings):.1f} max: {max(self.pings):.1f} ' - f'median: {statistics.median(self.pings):.1f}') - self.pings.clear() - - -OpenhabBenchRule() diff --git a/tests/helpers/__init__.py b/tests/helpers/__init__.py index ac7af5d8..3f604fa5 100644 --- a/tests/helpers/__init__.py +++ b/tests/helpers/__init__.py @@ -1,4 +1,6 @@ from .sync_worker import sync_worker from .parent_rule import parent_rule from .parameters import params -from .event_bus import event_bus, TmpEventBus \ No newline at end of file +from .event_bus import event_bus, TmpEventBus +from .mock_file import MockFile +from .module_helpers import get_module_classes, check_class_annotations diff --git a/tests/helpers/mock_file.py b/tests/helpers/mock_file.py new file mode 100644 index 00000000..898c784a --- /dev/null +++ b/tests/helpers/mock_file.py @@ -0,0 +1,66 @@ +from io import StringIO +from pathlib import Path, PurePath +from typing import Any, Callable, TextIO, Union +from warnings import warn + + +class MyStringIO(StringIO): + + def __init__(self, close_cb: Callable[[str], Any], *args, **kwargs) -> None: + super().__init__(*args, **kwargs) + + self.close_cb = close_cb + + def close(self) -> None: + self.close_cb(self.getvalue()) + super().close() + + +class MockFile: + + def __init__(self, path: str, data: str = ''): + super().__init__() + + self.path = Path(path) + self.data: str = data + self.warn_on_delete = True + + def __set_data(self, val: str): + self.data = val + + def is_file(self) -> bool: + return True + + def open(self, *args, **kwargs) -> TextIO: + return MyStringIO(self.__set_data, self.data) + + def rename(self, target: Union[str, PurePath]): + if self.warn_on_delete: + warn(f'Not supported for {self.__class__.__name__}!', UserWarning, stacklevel=2) + return None + + def replace(self, target: Union[str, PurePath]) -> None: + if self.warn_on_delete: + warn(f'Not supported for {self.__class__.__name__}!', UserWarning, stacklevel=2) + return None + + def rmdir(self) -> None: + if self.warn_on_delete: + warn(f'Not supported for {self.__class__.__name__}!', UserWarning, stacklevel=2) + return None + + def unlink(self) -> None: + if self.warn_on_delete: + warn(f'Not supported for {self.__class__.__name__}!', UserWarning, stacklevel=2) + return None + + # ----------------------------------------------------------------------------------- + # Funcs that are just passed through + def with_suffix(self, suffix): + c = self.__class__(self.path.with_suffix(suffix)) + c.warn_on_delete = self.warn_on_delete + return c + + @property + def name(self): + return self.path.name diff --git a/tests/helpers/module_helpers.py b/tests/helpers/module_helpers.py new file mode 100644 index 00000000..6d794695 --- /dev/null +++ b/tests/helpers/module_helpers.py @@ -0,0 +1,38 @@ +import importlib +import inspect +import sys +from typing import Iterable, Optional + + +def get_module_classes(module_name: str, exclude: Optional[Iterable[str]] = None, skip_imports=True): + if exclude is None: + exclude = set() + + importlib.import_module(module_name) + return dict(inspect.getmembers( + sys.modules[module_name], + lambda x: inspect.isclass(x) and (skip_imports or x.__module__ == module_name) and x.__name__ not in exclude + )) + + +def check_class_annotations(module_name: str, exclude: Optional[Iterable[str]] = None, skip_imports=True): + """Ensure that the annotations match with the actual variables""" + + classes = get_module_classes(module_name, exclude=exclude) + for name, cls in classes.items(): + c = cls() + args = dict(filter( + lambda x: not x[0].startswith('__'), + dict(inspect.getmembers(c, lambda x: not inspect.ismethod(x))).items()) + ) + + # Check that all vars are in __annotations__ + for arg_name in args: + assert arg_name in c.__annotations__, f'"{arg_name}" is missing in annotations!"\n' \ + f'members : {", ".join(sorted(args))}\n' \ + f'annotations: {", ".join(sorted(c.__annotations__))}' + + for arg_name in c.__annotations__: + assert arg_name in args, f'"{arg_name}" is missing in args!"\n' \ + f'members : {", ".join(sorted(args))}\n' \ + f'annotations: {", ".join(sorted(c.__annotations__))}' diff --git a/tests/test_core/test_event_bus.py b/tests/test_core/test_event_bus.py index 9dea9ed0..3f2054da 100644 --- a/tests/test_core/test_event_bus.py +++ b/tests/test_core/test_event_bus.py @@ -24,14 +24,14 @@ def test_repr(clean_event_bus: EventBus, sync_worker): listener = EventBusListener('test_name', f) assert listener.desc() == '"test_name" (type AllEvents)' - listener = EventBusListener('test_name', f, prop_name1='test1', prop_value1='value1') + listener = EventBusListener('test_name', f, attr_name1='test1', attr_value1='value1') assert listener.desc() == '"test_name" (type AllEvents, test1==value1)' - listener = EventBusListener('test_name', f, prop_name2='test2', prop_value2='value2') + listener = EventBusListener('test_name', f, attr_name2='test2', attr_value2='value2') assert listener.desc() == '"test_name" (type AllEvents, test2==value2)' - listener = EventBusListener('test_name', f, prop_name1='test1', prop_value1='value1', - prop_name2='test2', prop_value2='value2') + listener = EventBusListener('test_name', f, attr_name1='test1', attr_value1='value1', + attr_name2='test2', attr_value2='value2') assert listener.desc() == '"test_name" (type AllEvents, test1==value1, test2==value2)' @@ -95,8 +95,8 @@ def test_complex_event_unpack(clean_event_bus: EventBus, sync_worker): assert vars(arg3) == vars(ValueChangeEvent(item.name, 'ValNew', 'ValOld')) -def test_event_filter_single(clean_event_bus: EventBus, sync_worker): - events_all, events_filtered1, events_filtered2 = [], [], [] +def test_event_filter(clean_event_bus: EventBus, sync_worker): + events_all, events_filtered1, events_filtered2 , events_filtered3 = [], [], [], [] def append_all(event): events_all.append(event) @@ -107,9 +107,13 @@ def append_filter1(event): def append_filter2(event): events_filtered2.append(event) + def append_filter3(event): + events_filtered3.append(event) + name = 'test_filter' func1 = wrappedfunction.WrappedFunction(append_filter1) func2 = wrappedfunction.WrappedFunction(append_filter2) + func3 = wrappedfunction.WrappedFunction(append_filter3) # listener to all events EventBus.add_listener( @@ -120,22 +124,30 @@ def append_filter2(event): EventBus.add_listener(listener) listener = EventBusListener(name, func2, ValueUpdateEvent, None, None, 'value', 1) EventBus.add_listener(listener) + listener = EventBusListener(name, func3, ValueChangeEvent, 'old_value', None, 'value', 1) + EventBus.add_listener(listener) event0 = ValueUpdateEvent(name, None) event1 = ValueUpdateEvent(name, 'test_value') event2 = ValueUpdateEvent(name, 1) + event3 = ValueChangeEvent(name, 1, None) EventBus.post_event(name, event0) EventBus.post_event(name, event1) EventBus.post_event(name, event2) + EventBus.post_event(name, event3) - assert len(events_all) == 3 + assert len(events_all) == 4 assert vars(events_all[0]) == vars(event0) assert vars(events_all[1]) == vars(event1) assert vars(events_all[2]) == vars(event2) + assert vars(events_all[3]) == vars(event3) assert len(events_filtered1) == 1 assert vars(events_filtered1[0]) == vars(event1) assert len(events_filtered2) == 1 assert vars(events_filtered2[0]) == vars(event2) + + assert len(events_filtered3) == 1 + assert vars(events_filtered3[0]) == vars(event3) diff --git a/tests/test_core/test_events/test_core_filters.py b/tests/test_core/test_events/test_core_filters.py new file mode 100644 index 00000000..7fd530a6 --- /dev/null +++ b/tests/test_core/test_events/test_core_filters.py @@ -0,0 +1,53 @@ +import pytest + +from HABApp.core.event_bus_listener import WrappedFunction +from HABApp.core.events import EventFilter, ValueChangeEvent, ValueChangeEventFilter, ValueUpdateEvent, \ + ValueUpdateEventFilter +from tests.helpers import check_class_annotations + + +def test_class_annotations(): + """EventFilter relies on the class annotations so we test that every event has those""" + + check_class_annotations('HABApp.core.events.events', exclude=['ComplexEventValue', 'AllEvents']) + + +def test_repr(): + f = EventFilter(ValueUpdateEvent, value=1) + assert str(f) == 'EventFilter(event_type=ValueUpdateEvent, value=1)' + + f = ValueUpdateEventFilter(value='asd') + assert str(f) == 'ValueUpdateEventFilter(value=asd)' + + f = ValueChangeEventFilter(value=1.5) + assert str(f) == 'ValueChangeEventFilter(value=1.5)' + + f = ValueChangeEventFilter(old_value=3) + assert str(f) == 'ValueChangeEventFilter(old_value=3)' + + f = ValueChangeEventFilter(value=1.5, old_value=3) + assert str(f) == 'ValueChangeEventFilter(value=1.5, old_value=3)' + + +def test_exception_missing(): + with pytest.raises(AttributeError) as e: + EventFilter(ValueUpdateEvent, asdf=1) + + assert str(e.value) == 'Filter attribute "asdf" does not exist for "ValueUpdateEvent"' + + +def test_create_listener(): + + f = EventFilter(ValueUpdateEvent, value=1) + e = f.create_event_listener('asdf', WrappedFunction(lambda x: x)) + + assert e.event_filter is ValueUpdateEvent + assert e.attr_name1 == 'value' + assert e.attr_value1 == 1 + + f = ValueChangeEventFilter(old_value='asdf') + e = f.create_event_listener('asdf', WrappedFunction(lambda x: x)) + + assert e.event_filter is ValueChangeEvent + assert e.attr_name1 == 'old_value' + assert e.attr_value1 == 'asdf' diff --git a/tests/test_core/test_item_watch.py b/tests/test_core/test_item_watch.py index 91782814..98f7e491 100644 --- a/tests/test_core/test_item_watch.py +++ b/tests/test_core/test_item_watch.py @@ -25,7 +25,7 @@ async def test_multiple_add(parent_rule: DummyRule): @pytest.mark.asyncio -async def test_watch_update(parent_rule: DummyRule, event_bus: TmpEventBus, sync_worker): +async def test_watch_update(parent_rule: DummyRule, event_bus: TmpEventBus, sync_worker, caplog): for meth in ('watch_update', 'watch_change'): diff --git a/tests/test_mqtt/test_mqtt_filters.py b/tests/test_mqtt/test_mqtt_filters.py new file mode 100644 index 00000000..9ad93dba --- /dev/null +++ b/tests/test_mqtt/test_mqtt_filters.py @@ -0,0 +1,36 @@ +from HABApp.core.wrappedfunction import WrappedFunction +from HABApp.mqtt.events import MqttValueChangeEvent, MqttValueChangeEventFilter, MqttValueUpdateEvent, \ + MqttValueUpdateEventFilter +from tests.helpers import check_class_annotations + + +def test_class_annotations(): + """EventFilter relies on the class annotations so we test that every event has those""" + + exclude = ['MqttValueChangeEventFilter', 'MqttValueUpdateEventFilter'] + check_class_annotations('HABApp.mqtt.events', exclude=exclude, skip_imports=False) + + +def test_create_listener(): + f = MqttValueUpdateEventFilter(value=1) + e = f.create_event_listener('asdf', WrappedFunction(lambda x: x)) + + assert e.event_filter is MqttValueUpdateEvent + assert e.attr_name1 == 'value' + assert e.attr_value1 == 1 + + f = MqttValueChangeEventFilter(old_value='asdf') + e = f.create_event_listener('asdf', WrappedFunction(lambda x: x)) + + assert e.event_filter is MqttValueChangeEvent + assert e.attr_name1 == 'old_value' + assert e.attr_value1 == 'asdf' + + f = MqttValueChangeEventFilter(old_value='asdf', value=1) + e = f.create_event_listener('asdf', WrappedFunction(lambda x: x)) + + assert e.event_filter is MqttValueChangeEvent + assert e.attr_name1 == 'value' + assert e.attr_value1 == 1 + assert e.attr_name2 == 'old_value' + assert e.attr_value2 == 'asdf' diff --git a/tests/test_openhab/test_events/test_oh_filters.py b/tests/test_openhab/test_events/test_oh_filters.py new file mode 100644 index 00000000..d66feaee --- /dev/null +++ b/tests/test_openhab/test_events/test_oh_filters.py @@ -0,0 +1,37 @@ +from HABApp.core.wrappedfunction import WrappedFunction +from HABApp.openhab.events import ItemStateChangedEvent, ItemStateChangedEventFilter, ItemStateEvent, \ + ItemStateEventFilter +from tests.helpers import check_class_annotations + + +def test_class_annotations(): + """EventFilter relies on the class annotations so we test that every event has those""" + + exclude = ['OpenhabEvent', 'ItemStateChangedEventFilter', 'ItemStateEventFilter'] + check_class_annotations('HABApp.openhab.events', exclude=exclude, skip_imports=False) + + +def test_create_listener(): + + f = ItemStateEventFilter(value=1) + e = f.create_event_listener('asdf', WrappedFunction(lambda x: x)) + + assert e.event_filter is ItemStateEvent + assert e.attr_name1 == 'value' + assert e.attr_value1 == 1 + + f = ItemStateChangedEventFilter(old_value='asdf') + e = f.create_event_listener('asdf', WrappedFunction(lambda x: x)) + + assert e.event_filter is ItemStateChangedEvent + assert e.attr_name1 == 'old_value' + assert e.attr_value1 == 'asdf' + + f = ItemStateChangedEventFilter(old_value='asdf', value=1) + e = f.create_event_listener('asdf', WrappedFunction(lambda x: x)) + + assert e.event_filter is ItemStateChangedEvent + assert e.attr_name1 == 'value' + assert e.attr_value1 == 1 + assert e.attr_name2 == 'old_value' + assert e.attr_value2 == 'asdf' diff --git a/tests/test_openhab/test_plugins/test_thing/test_errors.py b/tests/test_openhab/test_plugins/test_thing/test_errors.py new file mode 100644 index 00000000..cdea0b99 --- /dev/null +++ b/tests/test_openhab/test_plugins/test_thing/test_errors.py @@ -0,0 +1,61 @@ +import pytest + +from HABApp.openhab.connection_logic.plugin_things.plugin_things import ManualThingConfig +from tests.helpers import MockFile + + +@pytest.mark.asyncio +async def test_errors(caplog): + + cfg = ManualThingConfig() + + data = [{"statusInfo": {"status": "ONLINE", "statusDetail": "NONE"}, "editable": True, + "label": "Astronomische Sonnendaten", + "configuration": {"interval": 120, "geolocation": "26.399750112407446,34.980468750000014"}, + "properties": {}, "UID": "astro:sun:1d5f16df", "thingTypeUID": "astro:sun", "channels": [ + {"linkedItems": [], "uid": "astro:sun:1d5f16df:rise#start", "id": "rise#start", + "channelTypeUID": "astro:start", "itemType": "DateTime", "kind": "STATE", "label": "Startzeit", + "description": "Die Startzeit des Ereignisses", "defaultTags": [], "properties": {}, + "configuration": {"offset": 0}}, ]}] + + text = """ + test: False + + filter: + thing_type: astro:sun + + channels: + - filter: + channel_type: astro:start + link items: + - type: Number + name: Name1 + - type: Number + name: Name1 + """ + file = MockFile('/thing_test.yml', data=text) + file.warn_on_delete = False + + await cfg.update_thing_config(file, data) + + assert caplog.records[0].message == 'Duplicate item: Name1' + + text = """ +test: False + +filter: + thing_type: astro:sun + +channels: + - filter: + channel_type: astro:start + link items: + - type: Number + name: â ß { ) + """ + file = MockFile('/thing_test.yml', data=text) + file.warn_on_delete = False + + await cfg.update_thing_config(file, data) + + assert caplog.records[1].message == '"â_ß_{_)" is not a valid name for an item!' diff --git a/tests/test_openhab/test_plugins/test_thing/test_file_writer.py b/tests/test_openhab/test_plugins/test_thing/test_file_writer.py index 0756b820..0cc66563 100644 --- a/tests/test_openhab/test_plugins/test_thing/test_file_writer.py +++ b/tests/test_openhab/test_plugins/test_thing/test_file_writer.py @@ -7,6 +7,7 @@ class MyStringIO(io.StringIO): def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) self.text = None + self.exists = False def open(self, *args, **kwargs): return self @@ -14,6 +15,9 @@ def open(self, *args, **kwargs): def close(self, *args, **kwargs): self.text = self.getvalue() super().close(*args, **kwargs) + + def is_file(self): + return self.exists def test_creation(tmp_path_factory): @@ -35,7 +39,7 @@ def test_creation(tmp_path_factory): t = MyStringIO() create_items_file(t, {k.name: k for k in objs}) - print('\n' + '-' * 120 + '\n' + t.text + '-' * 120) + # print('\n' + '-' * 120 + '\n' + t.text + '-' * 120) expected = """String Test_zwave_o_1 {channel = "zwave:link:device" } String Test_zwave_o_2 {channel = "zwave:link:device1", auto_update="False"} @@ -49,3 +53,10 @@ def test_creation(tmp_path_factory): """ assert expected == t.text + + # When the file already exists we append with newlines + t = MyStringIO() + t.exists = True + create_items_file(t, {k.name: k for k in objs}) + + assert '\n\n\n' + expected == t.text