add support for diagnostics platform

custom_components/meross_lan/__init__.py

@@ -115,6 +115,7 @@ async def async_unload_entry(hass: HomeAssistant, entry: ConfigEntry) -> bool:
                 device.unsub_updatecoordinator_listener()
                 device.unsub_updatecoordinator_listener = None
             api.devices.pop(device_id)
+            device.shutdown()
 
         #when removing the last configentry do a complete cleanup
         if (not api.devices) and (len(hass.config_entries.async_entries(DOMAIN)) == 1):

custom_components/meross_lan/config_flow.py

@@ -27,7 +27,7 @@
     CONF_PAYLOAD,
     CONF_PROTOCOL, CONF_PROTOCOL_OPTIONS,
     CONF_POLLING_PERIOD, CONF_POLLING_PERIOD_DEFAULT,
-    CONF_TRACE, CONF_TRACE_TIMEOUT,
+    CONF_TRACE, CONF_TRACE_TIMEOUT, CONF_TRACE_TIMEOUT_DEFAULT,
 )
 
 # helper conf keys not persisted to config
@@ -320,10 +320,11 @@ def __init__(self, config_entry):
             self.device_id = data.get(CONF_DEVICE_ID)
             self._host = data.get(CONF_HOST) # null for devices discovered over mqtt
             self._key = data.get(CONF_KEY)
+            self._cloud_key = data.get(CONF_CLOUD_KEY) # null for non cloud keys
             self._protocol = data.get(CONF_PROTOCOL)
             self._polling_period = data.get(CONF_POLLING_PERIOD)
             self._trace = data.get(CONF_TRACE, 0) > time()
-            self._cloud_key = data.get(CONF_CLOUD_KEY) # null for non cloud keys
+            self._trace_timeout = data.get(CONF_TRACE_TIMEOUT)
 
 
     async def async_step_init(self, user_input=None):
@@ -369,6 +370,7 @@ async def async_step_device(self, user_input=None):
             self._protocol = user_input.get(CONF_PROTOCOL)
             self._polling_period = user_input.get(CONF_POLLING_PERIOD)
             self._trace = user_input.get(CONF_TRACE)
+            self._trace_timeout = user_input.get(CONF_TRACE_TIMEOUT, CONF_TRACE_TIMEOUT_DEFAULT)
             try:
                 if self._host is not None:
                     _keymode = user_input[CONF_KEYMODE]
@@ -389,7 +391,8 @@ async def async_step_device(self, user_input=None):
                 data[CONF_KEY] = self._key
                 data[CONF_PROTOCOL] = self._protocol
                 data[CONF_POLLING_PERIOD] = self._polling_period
-                data[CONF_TRACE] = time() + CONF_TRACE_TIMEOUT if self._trace else 0
+                data[CONF_TRACE] = (time() + self._trace_timeout) if self._trace else 0
+                data[CONF_TRACE_TIMEOUT] = self._trace_timeout
                 self.hass.config_entries.async_update_entry(self._config_entry, data=data)
                 if device is not None:
                     try:
@@ -456,6 +459,13 @@ async def async_step_device(self, user_input=None):
                 description={ DESCR: self._trace}
                 )
             ] = bool
+        config_schema[
+            vol.Optional(
+                CONF_TRACE_TIMEOUT,
+                default=CONF_TRACE_TIMEOUT_DEFAULT,
+                description={ DESCR: self._trace_timeout}
+                )
+            ] = cv.positive_int
 
 
         return self.async_show_form(

custom_components/meross_lan/const.py

@@ -31,7 +31,8 @@
 CONF_POLLING_PERIOD_DEFAULT = 30
 
 CONF_TRACE = 'trace' # create a file with device info and communication tracing (only CONF_TRACE_TIMEOUT seconds then shut off)
-CONF_TRACE_TIMEOUT = 600 # when starting a trace stop it and close the file after .. secs
+CONF_TRACE_TIMEOUT = 'trace_timeout'
+CONF_TRACE_TIMEOUT_DEFAULT = 600 # when starting a trace stop it and close the file after .. secs
 CONF_TRACE_MAXSIZE = 65536 # or when MAXSIZE exceeded
 CONF_TRACE_DIRECTORY = 'traces' # folder where to store traces
 CONF_TRACE_FILENAME = '{}-{}.csv' # filename format: device_type-device_id.csv

custom_components/meross_lan/diagnostics.py

@@ -0,0 +1,64 @@
+from homeassistant.config_entries import ConfigEntry
+from homeassistant.components.diagnostics import REDACTED
+
+from .helpers import obfuscate
+
+from .const import (
+    DOMAIN,
+    CONF_DEVICE_ID, CONF_PAYLOAD,
+    CONF_HOST, CONF_KEY, CONF_CLOUD_KEY,
+    CONF_PROTOCOL, CONF_POLLING_PERIOD,
+    CONF_TRACE, CONF_TRACE_TIMEOUT,
+)
+
+async def async_get_config_entry_diagnostics(
+    hass, entry: ConfigEntry
+) -> dict[str, object]:
+    """Return diagnostics for a config entry."""
+    return await _async_get_diagnostics(hass, entry)
+
+
+async def async_get_device_diagnostics(
+    hass, entry: ConfigEntry, device
+) -> dict[str, object]:
+    """Return diagnostics for a device entry."""
+    return await _async_get_diagnostics(hass, entry)
+
+
+async def _async_get_diagnostics(hass, entry: ConfigEntry):
+
+    device_id = entry.data.get(CONF_DEVICE_ID)
+    if device_id is None:# MQTT hub entry
+        return {
+            CONF_KEY: REDACTED if entry.data.get(CONF_KEY) else None,
+            "disabled_by": entry.disabled_by,
+            "disabled_polling": entry.pref_disable_polling,
+        }
+
+    device = None
+    deviceclass = None
+    api = hass.data.get(DOMAIN)
+    if api is not None:# all of the meross_lan entries disabled?
+        device = api.devices.get(device_id)
+        deviceclass = type(device).__name__
+
+    trace_timeout = entry.data.get(CONF_TRACE_TIMEOUT)
+    payload = dict(entry.data.get(CONF_PAYLOAD)) #copy to avoid obfuscation of entry.data
+    obfuscate(payload)
+
+    data = {
+        CONF_HOST: REDACTED if entry.data.get(CONF_HOST) else None,
+        CONF_KEY: REDACTED if entry.data.get(CONF_KEY) else None,
+        CONF_CLOUD_KEY: REDACTED if entry.data.get(CONF_CLOUD_KEY) else None,
+        CONF_PROTOCOL: entry.data.get(CONF_PROTOCOL),
+        CONF_POLLING_PERIOD: entry.data.get(CONF_POLLING_PERIOD),
+        CONF_TRACE_TIMEOUT: trace_timeout,
+        CONF_DEVICE_ID: REDACTED,
+        CONF_PAYLOAD: payload,
+        "deviceclass": deviceclass,
+        "disabled_by": entry.disabled_by,
+        "disabled_polling": entry.pref_disable_polling,
+        CONF_TRACE: await device.get_dignostics_trace(trace_timeout) if device is not None else None
+    }
+
+    return data

custom_components/meross_lan/helpers.py

@@ -4,6 +4,8 @@
 import logging
 from time import time
 
+from .merossclient import const as mc
+
 LOGGER = logging.getLogger(__name__[:-8]) #get base custom_component name for logging
 _trap_dict = dict()
 
@@ -24,6 +26,48 @@ def LOGGER_trap(level, timeout, msg, *args):
     _trap_dict[trap_key] = epoch
 
 
+"""
+    obfuscation:
+    call obfuscate on a paylod (dict) to remove well-known sensitive
+    keys (list in OBFUSCATE_KEYS). The returned dictionary contains a
+    copy of original values and need to be used a gain when calling
+    deobfuscate on the previously obfuscated payload
+"""
+OBFUSCATE_KEYS = (
+    mc.KEY_UUID, mc.KEY_MACADDRESS, mc.KEY_WIFIMAC, mc.KEY_INNERIP,
+    mc.KEY_SERVER, mc.KEY_PORT, mc.KEY_USERID, mc.KEY_TOKEN
+)
+
+
+def obfuscate(payload: dict) -> dict:
+    """
+    payload: input-output gets modified by blanking sensistive keys
+    returns: a dict with the original mapped obfuscated keys
+    parses the input payload and 'hides' (obfuscates) some sensitive keys.
+    returns the mapping of the obfuscated keys in 'obfuscated' so to re-set them in _deobfuscate
+    this function is recursive
+    """
+    obfuscated = dict()
+    for key, value in payload.items():
+        if isinstance(value, dict):
+            o = obfuscate(value)
+            if o:
+                obfuscated[key] = o
+        elif key in OBFUSCATE_KEYS:
+            obfuscated[key] = value
+            payload[key] = '#' * len(str(value))
+
+    return obfuscated
+
+
+def deobfuscate(payload: dict, obfuscated: dict):
+    for key, value in obfuscated.items():
+        if isinstance(value, dict):
+            deobfuscate(payload[key], value)
+        else:
+            payload[key] = value
+
+
 """
 MQTT helpers
 """

custom_components/meross_lan/meross_device.py

@@ -28,13 +28,17 @@
     get_replykey, build_default_payload_get,
 )
 from .meross_entity import MerossFakeEntity
-from .helpers import LOGGER, LOGGER_trap, mqtt_is_connected
+from .helpers import (
+    LOGGER, LOGGER_trap,
+    obfuscate, deobfuscate,
+    mqtt_is_connected,
+)
 from .const import (
     DOMAIN, DND_ID,
     CONF_DEVICE_ID, CONF_KEY, CONF_PAYLOAD, CONF_HOST, CONF_TIMESTAMP,
     CONF_POLLING_PERIOD, CONF_POLLING_PERIOD_DEFAULT, CONF_POLLING_PERIOD_MIN,
     CONF_PROTOCOL, CONF_OPTION_AUTO, CONF_OPTION_HTTP, CONF_OPTION_MQTT,
-    CONF_TRACE, CONF_TRACE_DIRECTORY, CONF_TRACE_FILENAME, CONF_TRACE_MAXSIZE,
+    CONF_TRACE, CONF_TRACE_DIRECTORY, CONF_TRACE_FILENAME, CONF_TRACE_MAXSIZE, CONF_TRACE_TIMEOUT_DEFAULT,
     PARAM_HEARTBEAT_PERIOD, PARAM_TIMEZONE_CHECK_PERIOD, PARAM_TIMESTAMP_TOLERANCE,
 )
 
@@ -73,42 +77,9 @@
     mc.NS_APPLIANCE_CONTROL_PHYSICALLOCK # disconnects
 )
 
-TRACE_KEYS_OBFUSCATE = (
-    mc.KEY_UUID, mc.KEY_MACADDRESS, mc.KEY_WIFIMAC, mc.KEY_INNERIP,
-    mc.KEY_SERVER, mc.KEY_PORT, mc.KEY_USERID, mc.KEY_TOKEN
-)
-
 TRACE_DIRECTION_RX = 'RX'
 TRACE_DIRECTION_TX = 'TX'
 
-def _obfuscate(payload: dict) -> dict:
-    """
-    payload: input-output gets modified by blanking sensistive keys
-    returns: a dict with the original mapped obfuscated keys
-    parses the input payload and 'hides' (obfuscates) some sensitive keys.
-    returns the mapping of the obfuscated keys in 'obfuscated' so to re-set them in _deobfuscate
-    this function is recursive
-    """
-    obfuscated = dict()
-    for key, value in payload.items():
-        if isinstance(value, dict):
-            o = _obfuscate(value)
-            if o:
-                obfuscated[key] = o
-        elif key in TRACE_KEYS_OBFUSCATE:
-            obfuscated[key] = value
-            payload[key] = '#' * len(str(value))
-
-    return obfuscated
-
-def _deobfuscate(payload: dict, obfuscated: dict):
-    for key, value in obfuscated.items():
-        if isinstance(value, dict):
-            _deobfuscate(payload[key], value)
-        else:
-            payload[key] = value
-
-
 TIMEZONES_SET = None
 
 class Protocol(Enum):
@@ -155,6 +126,8 @@ def __init__(
         self.lastmqtt = 0 # means we recently received an mqtt message
         self.hasmqtt = False # hasmqtt means it is somehow available to communicate over mqtt
         self._trace_file: TextIOWrapper = None
+        self._trace_future: asyncio.Future = None
+        self._trace_data: list = None
         self._trace_endtime = 0
         self._trace_ability_iter = None
         """
@@ -240,6 +213,15 @@ def __del__(self):
         return
 
 
+    def shutdown(self):
+        """
+            called when the config entry is unloaded
+            we'll try to clear everything here
+        """
+        if self._trace_file:
+            self._trace_close()
+
+
     @property
     def host(self) -> str:
         return self._host or self.descriptor.innerIp
@@ -623,20 +605,9 @@ async def entry_update_listener(self, hass: HomeAssistant, config_entry: ConfigE
         """
         if self._trace_file is not None:
             self._trace_close()
-        _trace_endtime = config_entry.data.get(CONF_TRACE, 0)
-        if _trace_endtime > time():
-            try:
-                tracedir = hass.config.path('custom_components', DOMAIN, CONF_TRACE_DIRECTORY)
-                os.makedirs(tracedir, exist_ok=True)
-                self._trace_file = open(os.path.join(tracedir, CONF_TRACE_FILENAME.format(self.descriptor.type, int(_trace_endtime))), 'w')
-                self._trace_endtime = _trace_endtime
-                self._trace(self.descriptor.all, mc.NS_APPLIANCE_SYSTEM_ALL, mc.METHOD_GETACK)
-                self._trace(self.descriptor.ability, mc.NS_APPLIANCE_SYSTEM_ABILITY, mc.METHOD_GETACK)
-                self._trace_ability_iter = iter(self.descriptor.ability)
-                self._trace_ability()
-            except Exception as e:
-                LOGGER.warning("MerossDevice(%s) error while creating trace file (%s)", self.device_id, str(e))
-
+        endtime = config_entry.data.get(CONF_TRACE, 0)
+        if endtime > time():
+            self._trace_open(endtime)
         #await hass.config_entries.async_reload(config_entry.entry_id)
 
 
@@ -912,13 +883,54 @@ def _set_config_entry(self, data: dict) -> None:
             self.polling_period = CONF_POLLING_PERIOD_MIN
 
 
+    def get_dignostics_trace(self, trace_timeout) -> asyncio.Future:
+        """
+        invoked by the diagnostics callback:
+        here we set the device to start tracing the classical way (in file)
+        but we also fill in a dict which will set back as the result of the
+        Future we're returning to dignostics
+        """
+        if self._trace_future is not None:
+            # avoid re-entry..keep going the running trace
+            return self._trace_future
+        if self._trace_file is not None:
+            self._trace_close()
+        loop = asyncio.get_running_loop()
+        self._trace_future = loop.create_future()
+        self._trace_data = list()
+        self._trace_data.append(['time','rxtx','protocol','method','namespace','data'])
+        self._trace_open(time() + (trace_timeout or CONF_TRACE_TIMEOUT_DEFAULT))
+        return self._trace_future
+
+
+    def _trace_open(self, endtime):
+        try:
+            tracedir = self.api.hass.config.path('custom_components', DOMAIN, CONF_TRACE_DIRECTORY)
+            os.makedirs(tracedir, exist_ok=True)
+            self._trace_file = open(os.path.join(tracedir, CONF_TRACE_FILENAME.format(self.descriptor.type, int(endtime))), 'w')
+            self._trace_endtime = endtime
+            self._trace(self.descriptor.all, mc.NS_APPLIANCE_SYSTEM_ALL, mc.METHOD_GETACK)
+            self._trace(self.descriptor.ability, mc.NS_APPLIANCE_SYSTEM_ABILITY, mc.METHOD_GETACK)
+            self._trace_ability_iter = iter(self.descriptor.ability)
+            self._trace_ability()
+        except Exception as e:
+            LOGGER.warning("MerossDevice(%s) error while creating trace file (%s)", self.device_id, str(e))
+            if self._trace_file is not None:
+                self._trace_close()
+
+
     def _trace_close(self):
         try:
             self._trace_file.close()
         except Exception as e:
             LOGGER.warning("MerossDevice(%s) error while closing trace file (%s)", self.device_id, str(e))
         self._trace_file = None
         self._trace_ability_iter = None
+        if self._trace_future is not None:
+            self._trace_future.set_result(self._trace_data)
+            self._trace_future = None
+        self._trace_data = None
+
 
     @callback
     def _trace_ability(self, *args):
@@ -950,22 +962,25 @@ def _trace(
         ):
         if self._trace_file is not None:
             now = time()
-            if now > self._trace_endtime:
+            if (now > self._trace_endtime) or \
+                (self._trace_file.tell() > CONF_TRACE_MAXSIZE):
                 self._trace_close()
                 return
 
             if isinstance(data, dict):
-                obfuscated = _obfuscate(data)
+                obfuscated = obfuscate(data)
 
             try:
-                self._trace_file.write(strftime('%Y/%m/%d - %H:%M:%S\t') \
-                    + rxtx + '\t' + protocol + '\t' + method + '\t' + namespace + '\t' \
-                    + (json_dumps(data) if isinstance(data, dict) else data) + '\r\n')
-                if self._trace_file.tell() > CONF_TRACE_MAXSIZE:
-                    self._trace_close()
+                texttime = strftime('%Y/%m/%d - %H:%M:%S')
+                textdata = json_dumps(data) if isinstance(data, dict) else data
+                columns = [texttime, rxtx, protocol, method, namespace, textdata]
+                self._trace_file.write('\t'.join(columns) + '\r\n')
+                if self._trace_data is not None:
+                    columns[5] = data # better have json for dignostic trace
+                    self._trace_data.append(columns)
             except Exception as e:
                 LOGGER.warning("MerossDevice(%s) error while writing to trace file (%s)", self.device_id, str(e))
                 self._trace_close()
 
             if isinstance(data, dict):
-                _deobfuscate(data, obfuscated)
+                deobfuscate(data, obfuscated)