From 4d3b978398818f4fe7a2094cb54f83c20a57ef18 Mon Sep 17 00:00:00 2001 From: Paarth Shah Date: Sat, 2 Sep 2023 06:02:55 -0700 Subject: [PATCH 01/12] Change matrix component to use matrix-nio instead of matrix_client (#72797) --- .coveragerc | 3 +- .strict-typing | 1 + CODEOWNERS | 2 + homeassistant/components/matrix/__init__.py | 537 ++++++++++-------- homeassistant/components/matrix/manifest.json | 4 +- homeassistant/components/matrix/notify.py | 13 +- mypy.ini | 10 + requirements_all.txt | 3 +- requirements_test.txt | 1 + requirements_test_all.txt | 4 + tests/components/matrix/__init__.py | 1 + tests/components/matrix/conftest.py | 248 ++++++++ tests/components/matrix/test_join_rooms.py | 22 + tests/components/matrix/test_login.py | 118 ++++ tests/components/matrix/test_matrix_bot.py | 88 +++ tests/components/matrix/test_send_message.py | 71 +++ 16 files changed, 881 insertions(+), 245 deletions(-) create mode 100644 tests/components/matrix/__init__.py create mode 100644 tests/components/matrix/conftest.py create mode 100644 tests/components/matrix/test_join_rooms.py create mode 100644 tests/components/matrix/test_login.py create mode 100644 tests/components/matrix/test_matrix_bot.py create mode 100644 tests/components/matrix/test_send_message.py diff --git a/.coveragerc b/.coveragerc index bf3dd5f4a00cf1..d5a491a330f3ac 100644 --- a/.coveragerc +++ b/.coveragerc @@ -705,7 +705,8 @@ omit = homeassistant/components/mailgun/notify.py homeassistant/components/map/* homeassistant/components/mastodon/notify.py - homeassistant/components/matrix/* + homeassistant/components/matrix/__init__.py + homeassistant/components/matrix/notify.py homeassistant/components/matter/__init__.py homeassistant/components/meater/__init__.py homeassistant/components/meater/sensor.py diff --git a/.strict-typing b/.strict-typing index e8bca0a1abd181..3059c42f33fab6 100644 --- a/.strict-typing +++ b/.strict-typing @@ -213,6 +213,7 @@ homeassistant.components.lookin.* homeassistant.components.luftdaten.* homeassistant.components.mailbox.* homeassistant.components.mastodon.* +homeassistant.components.matrix.* homeassistant.components.matter.* homeassistant.components.media_extractor.* homeassistant.components.media_player.* diff --git a/CODEOWNERS b/CODEOWNERS index 65a36205518470..bf6fdaf9fc5713 100644 --- a/CODEOWNERS +++ b/CODEOWNERS @@ -723,6 +723,8 @@ build.json @home-assistant/supervisor /homeassistant/components/lyric/ @timmo001 /tests/components/lyric/ @timmo001 /homeassistant/components/mastodon/ @fabaff +/homeassistant/components/matrix/ @PaarthShah +/tests/components/matrix/ @PaarthShah /homeassistant/components/matter/ @home-assistant/matter /tests/components/matter/ @home-assistant/matter /homeassistant/components/mazda/ @bdr99 diff --git a/homeassistant/components/matrix/__init__.py b/homeassistant/components/matrix/__init__.py index febafc367f1be2..cf7bcce7b3c7e6 100644 --- a/homeassistant/components/matrix/__init__.py +++ b/homeassistant/components/matrix/__init__.py @@ -1,10 +1,28 @@ """The Matrix bot component.""" -from functools import partial +from __future__ import annotations + +import asyncio import logging import mimetypes import os - -from matrix_client.client import MatrixClient, MatrixRequestError +import re +from typing import NewType, TypedDict + +import aiofiles.os +from nio import AsyncClient, Event, MatrixRoom +from nio.events.room_events import RoomMessageText +from nio.responses import ( + ErrorResponse, + JoinError, + JoinResponse, + LoginError, + Response, + UploadError, + UploadResponse, + WhoamiError, + WhoamiResponse, +) +from PIL import Image import voluptuous as vol from homeassistant.components.notify import ATTR_DATA, ATTR_MESSAGE, ATTR_TARGET @@ -16,8 +34,8 @@ EVENT_HOMEASSISTANT_START, EVENT_HOMEASSISTANT_STOP, ) -from homeassistant.core import HomeAssistant, ServiceCall -from homeassistant.exceptions import HomeAssistantError +from homeassistant.core import Event as HassEvent, HomeAssistant, ServiceCall +from homeassistant.exceptions import ConfigEntryAuthFailed, HomeAssistantError import homeassistant.helpers.config_validation as cv from homeassistant.helpers.json import save_json from homeassistant.helpers.typing import ConfigType @@ -35,23 +53,37 @@ CONF_WORD = "word" CONF_EXPRESSION = "expression" +EVENT_MATRIX_COMMAND = "matrix_command" + DEFAULT_CONTENT_TYPE = "application/octet-stream" MESSAGE_FORMATS = [FORMAT_HTML, FORMAT_TEXT] DEFAULT_MESSAGE_FORMAT = FORMAT_TEXT -EVENT_MATRIX_COMMAND = "matrix_command" - ATTR_FORMAT = "format" # optional message format ATTR_IMAGES = "images" # optional images +WordCommand = NewType("WordCommand", str) +ExpressionCommand = NewType("ExpressionCommand", re.Pattern) +RoomID = NewType("RoomID", str) + + +class ConfigCommand(TypedDict, total=False): + """Corresponds to a single COMMAND_SCHEMA.""" + + name: str # CONF_NAME + rooms: list[RoomID] | None # CONF_ROOMS + word: WordCommand | None # CONF_WORD + expression: ExpressionCommand | None # CONF_EXPRESSION + + COMMAND_SCHEMA = vol.All( vol.Schema( { vol.Exclusive(CONF_WORD, "trigger"): cv.string, vol.Exclusive(CONF_EXPRESSION, "trigger"): cv.is_regex, vol.Required(CONF_NAME): cv.string, - vol.Optional(CONF_ROOMS, default=[]): vol.All(cv.ensure_list, [cv.string]), + vol.Optional(CONF_ROOMS): vol.All(cv.ensure_list, [cv.string]), } ), cv.has_at_least_one_key(CONF_WORD, CONF_EXPRESSION), @@ -75,7 +107,6 @@ extra=vol.ALLOW_EXTRA, ) - SERVICE_SCHEMA_SEND_MESSAGE = vol.Schema( { vol.Required(ATTR_MESSAGE): cv.string, @@ -90,30 +121,26 @@ ) -def setup(hass: HomeAssistant, config: ConfigType) -> bool: +async def async_setup(hass: HomeAssistant, config: ConfigType) -> bool: """Set up the Matrix bot component.""" config = config[DOMAIN] - try: - bot = MatrixBot( - hass, - os.path.join(hass.config.path(), SESSION_FILE), - config[CONF_HOMESERVER], - config[CONF_VERIFY_SSL], - config[CONF_USERNAME], - config[CONF_PASSWORD], - config[CONF_ROOMS], - config[CONF_COMMANDS], - ) - hass.data[DOMAIN] = bot - except MatrixRequestError as exception: - _LOGGER.error("Matrix failed to log in: %s", str(exception)) - return False + matrix_bot = MatrixBot( + hass, + os.path.join(hass.config.path(), SESSION_FILE), + config[CONF_HOMESERVER], + config[CONF_VERIFY_SSL], + config[CONF_USERNAME], + config[CONF_PASSWORD], + config[CONF_ROOMS], + config[CONF_COMMANDS], + ) + hass.data[DOMAIN] = matrix_bot - hass.services.register( + hass.services.async_register( DOMAIN, SERVICE_SEND_MESSAGE, - bot.handle_send_message, + matrix_bot.handle_send_message, schema=SERVICE_SCHEMA_SEND_MESSAGE, ) @@ -123,164 +150,141 @@ def setup(hass: HomeAssistant, config: ConfigType) -> bool: class MatrixBot: """The Matrix Bot.""" + _client: AsyncClient + def __init__( self, - hass, - config_file, - homeserver, - verify_ssl, - username, - password, - listening_rooms, - commands, - ): + hass: HomeAssistant, + config_file: str, + homeserver: str, + verify_ssl: bool, + username: str, + password: str, + listening_rooms: list[RoomID], + commands: list[ConfigCommand], + ) -> None: """Set up the client.""" self.hass = hass self._session_filepath = config_file - self._auth_tokens = self._get_auth_tokens() + self._access_tokens: JsonObjectType = {} self._homeserver = homeserver self._verify_tls = verify_ssl self._mx_id = username self._password = password - self._listening_rooms = listening_rooms - - # We have to fetch the aliases for every room to make sure we don't - # join it twice by accident. However, fetching aliases is costly, - # so we only do it once per room. - self._aliases_fetched_for = set() - - # Word commands are stored dict-of-dict: First dict indexes by room ID - # / alias, second dict indexes by the word - self._word_commands = {} + self._client = AsyncClient( + homeserver=self._homeserver, user=self._mx_id, ssl=self._verify_tls + ) - # Regular expression commands are stored as a list of commands per - # room, i.e., a dict-of-list - self._expression_commands = {} + self._listening_rooms = listening_rooms - for command in commands: - if not command.get(CONF_ROOMS): - command[CONF_ROOMS] = listening_rooms - - if command.get(CONF_WORD): - for room_id in command[CONF_ROOMS]: - if room_id not in self._word_commands: - self._word_commands[room_id] = {} - self._word_commands[room_id][command[CONF_WORD]] = command - else: - for room_id in command[CONF_ROOMS]: - if room_id not in self._expression_commands: - self._expression_commands[room_id] = [] - self._expression_commands[room_id].append(command) + self._word_commands: dict[RoomID, dict[WordCommand, ConfigCommand]] = {} + self._expression_commands: dict[RoomID, list[ConfigCommand]] = {} + self._load_commands(commands) - # Log in. This raises a MatrixRequestError if login is unsuccessful - self._client = self._login() + async def stop_client(event: HassEvent) -> None: + """Run once when Home Assistant stops.""" + if self._client is not None: + await self._client.close() - def handle_matrix_exception(exception): - """Handle exceptions raised inside the Matrix SDK.""" - _LOGGER.error("Matrix exception:\n %s", str(exception)) + self.hass.bus.async_listen_once(EVENT_HOMEASSISTANT_STOP, stop_client) - self._client.start_listener_thread(exception_handler=handle_matrix_exception) + async def handle_startup(event: HassEvent) -> None: + """Run once when Home Assistant finished startup.""" + self._access_tokens = await self._get_auth_tokens() + await self._login() + await self._join_rooms() + # Sync once so that we don't respond to past events. + await self._client.sync(timeout=30_000) - def stop_client(_): - """Run once when Home Assistant stops.""" - self._client.stop_listener_thread() + self._client.add_event_callback(self._handle_room_message, RoomMessageText) - self.hass.bus.listen_once(EVENT_HOMEASSISTANT_STOP, stop_client) + await self._client.sync_forever( + timeout=30_000, + loop_sleep_time=1_000, + ) # milliseconds. - # Joining rooms potentially does a lot of I/O, so we defer it - def handle_startup(_): - """Run once when Home Assistant finished startup.""" - self._join_rooms() + self.hass.bus.async_listen_once(EVENT_HOMEASSISTANT_START, handle_startup) - self.hass.bus.listen_once(EVENT_HOMEASSISTANT_START, handle_startup) + def _load_commands(self, commands: list[ConfigCommand]) -> None: + for command in commands: + # Set the command for all listening_rooms, unless otherwise specified. + command.setdefault(CONF_ROOMS, self._listening_rooms) # type: ignore[misc] + + # COMMAND_SCHEMA guarantees that exactly one of CONF_WORD and CONF_expression are set. + if (word_command := command.get(CONF_WORD)) is not None: + for room_id in command[CONF_ROOMS]: # type: ignore[literal-required] + self._word_commands.setdefault(room_id, {}) + self._word_commands[room_id][word_command] = command # type: ignore[index] + else: + for room_id in command[CONF_ROOMS]: # type: ignore[literal-required] + self._expression_commands.setdefault(room_id, []) + self._expression_commands[room_id].append(command) - def _handle_room_message(self, room_id, room, event): + async def _handle_room_message(self, room: MatrixRoom, message: Event) -> None: """Handle a message sent to a Matrix room.""" - if event["content"]["msgtype"] != "m.text": + # Corresponds to message type 'm.text' and NOT other RoomMessage subtypes, like 'm.notice' and 'm.emote'. + if not isinstance(message, RoomMessageText): return - - if event["sender"] == self._mx_id: + # Don't respond to our own messages. + if message.sender == self._mx_id: return + _LOGGER.debug("Handling message: %s", message.body) - _LOGGER.debug("Handling message: %s", event["content"]["body"]) + room_id = RoomID(room.room_id) - if event["content"]["body"][0] == "!": - # Could trigger a single-word command - pieces = event["content"]["body"].split(" ") - cmd = pieces[0][1:] + if message.body.startswith("!"): + # Could trigger a single-word command. + pieces = message.body.split() + word = WordCommand(pieces[0].lstrip("!")) - command = self._word_commands.get(room_id, {}).get(cmd) - if command: - event_data = { + if command := self._word_commands.get(room_id, {}).get(word): + message_data = { "command": command[CONF_NAME], - "sender": event["sender"], + "sender": message.sender, "room": room_id, "args": pieces[1:], } - self.hass.bus.fire(EVENT_MATRIX_COMMAND, event_data) + self.hass.bus.async_fire(EVENT_MATRIX_COMMAND, message_data) - # After single-word commands, check all regex commands in the room + # After single-word commands, check all regex commands in the room. for command in self._expression_commands.get(room_id, []): - match = command[CONF_EXPRESSION].match(event["content"]["body"]) + match: re.Match = command[CONF_EXPRESSION].match(message.body) # type: ignore[literal-required] if not match: continue - event_data = { + message_data = { "command": command[CONF_NAME], - "sender": event["sender"], + "sender": message.sender, "room": room_id, "args": match.groupdict(), } - self.hass.bus.fire(EVENT_MATRIX_COMMAND, event_data) - - def _join_or_get_room(self, room_id_or_alias): - """Join a room or get it, if we are already in the room. - - We can't just always call join_room(), since that seems to crash - the client if we're already in the room. - """ - rooms = self._client.get_rooms() - if room_id_or_alias in rooms: - _LOGGER.debug("Already in room %s", room_id_or_alias) - return rooms[room_id_or_alias] - - for room in rooms.values(): - if room.room_id not in self._aliases_fetched_for: - room.update_aliases() - self._aliases_fetched_for.add(room.room_id) - - if ( - room_id_or_alias in room.aliases - or room_id_or_alias == room.canonical_alias - ): - _LOGGER.debug( - "Already in room %s (known as %s)", room.room_id, room_id_or_alias - ) - return room - - room = self._client.join_room(room_id_or_alias) - _LOGGER.info("Joined room %s (known as %s)", room.room_id, room_id_or_alias) - return room + self.hass.bus.async_fire(EVENT_MATRIX_COMMAND, message_data) + + async def _join_room(self, room_id_or_alias: str) -> None: + """Join a room or do nothing if already joined.""" + join_response = await self._client.join(room_id_or_alias) + + if isinstance(join_response, JoinResponse): + _LOGGER.debug("Joined or already in room '%s'", room_id_or_alias) + elif isinstance(join_response, JoinError): + _LOGGER.error( + "Could not join room '%s': %s", + room_id_or_alias, + join_response, + ) - def _join_rooms(self): + async def _join_rooms(self) -> None: """Join the Matrix rooms that we listen for commands in.""" - for room_id in self._listening_rooms: - try: - room = self._join_or_get_room(room_id) - room.add_listener( - partial(self._handle_room_message, room_id), "m.room.message" - ) - - except MatrixRequestError as ex: - _LOGGER.error("Could not join room %s: %s", room_id, ex) - - def _get_auth_tokens(self) -> JsonObjectType: - """Read sorted authentication tokens from disk. - - Returns the auth_tokens dictionary. - """ + rooms = [ + self.hass.async_create_task(self._join_room(room_id)) + for room_id in self._listening_rooms + ] + await asyncio.wait(rooms) + + async def _get_auth_tokens(self) -> JsonObjectType: + """Read sorted authentication tokens from disk.""" try: return load_json_object(self._session_filepath) except HomeAssistantError as ex: @@ -291,116 +295,179 @@ def _get_auth_tokens(self) -> JsonObjectType: ) return {} - def _store_auth_token(self, token): + async def _store_auth_token(self, token: str) -> None: """Store authentication token to session and persistent storage.""" - self._auth_tokens[self._mx_id] = token + self._access_tokens[self._mx_id] = token - save_json(self._session_filepath, self._auth_tokens) + await self.hass.async_add_executor_job( + save_json, self._session_filepath, self._access_tokens, True # private=True + ) - def _login(self): - """Login to the Matrix homeserver and return the client instance.""" - # Attempt to generate a valid client using either of the two possible - # login methods: - client = None + async def _login(self) -> None: + """Log in to the Matrix homeserver. - # If we have an authentication token - if self._mx_id in self._auth_tokens: - try: - client = self._login_by_token() - _LOGGER.debug("Logged in using stored token") + Attempts to use the stored access token. + If that fails, then tries using the password. + If that also fails, raises LocalProtocolError. + """ - except MatrixRequestError as ex: + # If we have an access token + if (token := self._access_tokens.get(self._mx_id)) is not None: + _LOGGER.debug("Restoring login from stored access token") + self._client.restore_login( + user_id=self._client.user_id, + device_id=self._client.device_id, + access_token=token, + ) + response = await self._client.whoami() + if isinstance(response, WhoamiError): _LOGGER.warning( - "Login by token failed, falling back to password: %d, %s", - ex.code, - ex.content, + "Restoring login from access token failed: %s, %s", + response.status_code, + response.message, + ) + self._client.access_token = ( + "" # Force a soft-logout if the homeserver didn't. + ) + elif isinstance(response, WhoamiResponse): + _LOGGER.debug( + "Successfully restored login from access token: user_id '%s', device_id '%s'", + response.user_id, + response.device_id, ) - # If we still don't have a client try password - if not client: - try: - client = self._login_by_password() - _LOGGER.debug("Logged in using password") - - except MatrixRequestError as ex: - _LOGGER.error( - "Login failed, both token and username/password invalid: %d, %s", - ex.code, - ex.content, + # If the token login did not succeed + if not self._client.logged_in: + response = await self._client.login(password=self._password) + _LOGGER.debug("Logging in using password") + + if isinstance(response, LoginError): + _LOGGER.warning( + "Login by password failed: %s, %s", + response.status_code, + response.message, ) - # Re-raise the error so _setup can catch it - raise - - return client - - def _login_by_token(self): - """Login using authentication token and return the client.""" - return MatrixClient( - base_url=self._homeserver, - token=self._auth_tokens[self._mx_id], - user_id=self._mx_id, - valid_cert_check=self._verify_tls, - ) - def _login_by_password(self): - """Login using password authentication and return the client.""" - _client = MatrixClient( - base_url=self._homeserver, valid_cert_check=self._verify_tls + if not self._client.logged_in: + raise ConfigEntryAuthFailed( + "Login failed, both token and username/password are invalid" + ) + + await self._store_auth_token(self._client.access_token) + + async def _handle_room_send( + self, target_room: RoomID, message_type: str, content: dict + ) -> None: + """Wrap _client.room_send and handle ErrorResponses.""" + response: Response = await self._client.room_send( + room_id=target_room, + message_type=message_type, + content=content, ) + if isinstance(response, ErrorResponse): + _LOGGER.error( + "Unable to deliver message to room '%s': %s", + target_room, + response, + ) + else: + _LOGGER.debug("Message delivered to room '%s'", target_room) + + async def _handle_multi_room_send( + self, target_rooms: list[RoomID], message_type: str, content: dict + ) -> None: + """Wrap _handle_room_send for multiple target_rooms.""" + _tasks = [] + for target_room in target_rooms: + _tasks.append( + self.hass.async_create_task( + self._handle_room_send( + target_room=target_room, + message_type=message_type, + content=content, + ) + ) + ) + await asyncio.wait(_tasks) - _client.login_with_password(self._mx_id, self._password) + async def _send_image(self, image_path: str, target_rooms: list[RoomID]) -> None: + """Upload an image, then send it to all target_rooms.""" + _is_allowed_path = await self.hass.async_add_executor_job( + self.hass.config.is_allowed_path, image_path + ) + if not _is_allowed_path: + _LOGGER.error("Path not allowed: %s", image_path) + return - self._store_auth_token(_client.token) + # Get required image metadata. + image = await self.hass.async_add_executor_job(Image.open, image_path) + (width, height) = image.size + mime_type = mimetypes.guess_type(image_path)[0] + file_stat = await aiofiles.os.stat(image_path) + + _LOGGER.debug("Uploading file from path, %s", image_path) + async with aiofiles.open(image_path, "r+b") as image_file: + response, _ = await self._client.upload( + image_file, + content_type=mime_type, + filename=os.path.basename(image_path), + filesize=file_stat.st_size, + ) + if isinstance(response, UploadError): + _LOGGER.error("Unable to upload image to the homeserver: %s", response) + return + if isinstance(response, UploadResponse): + _LOGGER.debug("Successfully uploaded image to the homeserver") + else: + _LOGGER.error( + "Unknown response received when uploading image to homeserver: %s", + response, + ) + return - return _client + content = { + "body": os.path.basename(image_path), + "info": { + "size": file_stat.st_size, + "mimetype": mime_type, + "w": width, + "h": height, + }, + "msgtype": "m.image", + "url": response.content_uri, + } - def _send_image(self, img, target_rooms): - _LOGGER.debug("Uploading file from path, %s", img) + await self._handle_multi_room_send( + target_rooms=target_rooms, message_type="m.room.message", content=content + ) - if not self.hass.config.is_allowed_path(img): - _LOGGER.error("Path not allowed: %s", img) - return - with open(img, "rb") as upfile: - imgfile = upfile.read() - content_type = mimetypes.guess_type(img)[0] - mxc = self._client.upload(imgfile, content_type) - for target_room in target_rooms: - try: - room = self._join_or_get_room(target_room) - room.send_image(mxc, img, mimetype=content_type) - except MatrixRequestError as ex: - _LOGGER.error( - "Unable to deliver message to room '%s': %d, %s", - target_room, - ex.code, - ex.content, - ) + async def _send_message( + self, message: str, target_rooms: list[RoomID], data: dict | None + ) -> None: + """Send a message to the Matrix server.""" + content = {"msgtype": "m.text", "body": message} + if data is not None and data.get(ATTR_FORMAT) == FORMAT_HTML: + content |= {"format": "org.matrix.custom.html", "formatted_body": message} - def _send_message(self, message, data, target_rooms): - """Send the message to the Matrix server.""" - for target_room in target_rooms: - try: - room = self._join_or_get_room(target_room) - if message is not None: - if data.get(ATTR_FORMAT) == FORMAT_HTML: - _LOGGER.debug(room.send_html(message)) - else: - _LOGGER.debug(room.send_text(message)) - except MatrixRequestError as ex: - _LOGGER.error( - "Unable to deliver message to room '%s': %d, %s", - target_room, - ex.code, - ex.content, - ) - if ATTR_IMAGES in data: - for img in data.get(ATTR_IMAGES, []): - self._send_image(img, target_rooms) + await self._handle_multi_room_send( + target_rooms=target_rooms, message_type="m.room.message", content=content + ) - def handle_send_message(self, service: ServiceCall) -> None: + if ( + data is not None + and (image_paths := data.get(ATTR_IMAGES, [])) + and len(target_rooms) > 0 + ): + image_tasks = [ + self.hass.async_create_task(self._send_image(image_path, target_rooms)) + for image_path in image_paths + ] + await asyncio.wait(image_tasks) + + async def handle_send_message(self, service: ServiceCall) -> None: """Handle the send_message service.""" - self._send_message( - service.data.get(ATTR_MESSAGE), - service.data.get(ATTR_DATA), + await self._send_message( + service.data[ATTR_MESSAGE], service.data[ATTR_TARGET], + service.data.get(ATTR_DATA), ) diff --git a/homeassistant/components/matrix/manifest.json b/homeassistant/components/matrix/manifest.json index 4bded80a71164e..74bb97d10fca95 100644 --- a/homeassistant/components/matrix/manifest.json +++ b/homeassistant/components/matrix/manifest.json @@ -1,9 +1,9 @@ { "domain": "matrix", "name": "Matrix", - "codeowners": [], + "codeowners": ["@PaarthShah"], "documentation": "https://www.home-assistant.io/integrations/matrix", "iot_class": "cloud_push", "loggers": ["matrix_client"], - "requirements": ["matrix-client==0.4.0"] + "requirements": ["matrix-nio==0.21.2", "Pillow==10.0.0"] } diff --git a/homeassistant/components/matrix/notify.py b/homeassistant/components/matrix/notify.py index 3c90e9afbc04c9..c71f91eb582ab1 100644 --- a/homeassistant/components/matrix/notify.py +++ b/homeassistant/components/matrix/notify.py @@ -1,6 +1,8 @@ """Support for Matrix notifications.""" from __future__ import annotations +from typing import Any + import voluptuous as vol from homeassistant.components.notify import ( @@ -14,6 +16,7 @@ import homeassistant.helpers.config_validation as cv from homeassistant.helpers.typing import ConfigType, DiscoveryInfoType +from . import RoomID from .const import DOMAIN, SERVICE_SEND_MESSAGE CONF_DEFAULT_ROOM = "default_room" @@ -33,16 +36,14 @@ def get_service( class MatrixNotificationService(BaseNotificationService): """Send notifications to a Matrix room.""" - def __init__(self, default_room): + def __init__(self, default_room: RoomID) -> None: """Set up the Matrix notification service.""" self._default_room = default_room - def send_message(self, message="", **kwargs): + def send_message(self, message: str = "", **kwargs: Any) -> None: """Send the message to the Matrix server.""" - target_rooms = kwargs.get(ATTR_TARGET) or [self._default_room] + target_rooms: list[RoomID] = kwargs.get(ATTR_TARGET) or [self._default_room] service_data = {ATTR_TARGET: target_rooms, ATTR_MESSAGE: message} if (data := kwargs.get(ATTR_DATA)) is not None: service_data[ATTR_DATA] = data - return self.hass.services.call( - DOMAIN, SERVICE_SEND_MESSAGE, service_data=service_data - ) + self.hass.services.call(DOMAIN, SERVICE_SEND_MESSAGE, service_data=service_data) diff --git a/mypy.ini b/mypy.ini index 82cce328c6ae5d..9802c26c3c6461 100644 --- a/mypy.ini +++ b/mypy.ini @@ -1892,6 +1892,16 @@ disallow_untyped_defs = true warn_return_any = true warn_unreachable = true +[mypy-homeassistant.components.matrix.*] +check_untyped_defs = true +disallow_incomplete_defs = true +disallow_subclassing_any = true +disallow_untyped_calls = true +disallow_untyped_decorators = true +disallow_untyped_defs = true +warn_return_any = true +warn_unreachable = true + [mypy-homeassistant.components.matter.*] check_untyped_defs = true disallow_incomplete_defs = true diff --git a/requirements_all.txt b/requirements_all.txt index f7355ea948313e..4c5497ae98c981 100644 --- a/requirements_all.txt +++ b/requirements_all.txt @@ -37,6 +37,7 @@ Mastodon.py==1.5.1 # homeassistant.components.doods # homeassistant.components.generic # homeassistant.components.image_upload +# homeassistant.components.matrix # homeassistant.components.proxy # homeassistant.components.qrcode # homeassistant.components.seven_segments @@ -1177,7 +1178,7 @@ lxml==4.9.3 mac-vendor-lookup==0.1.12 # homeassistant.components.matrix -matrix-client==0.4.0 +matrix-nio==0.21.2 # homeassistant.components.maxcube maxcube-api==0.4.3 diff --git a/requirements_test.txt b/requirements_test.txt index a2533d0ef2b629..89db04a5db83cc 100644 --- a/requirements_test.txt +++ b/requirements_test.txt @@ -33,6 +33,7 @@ requests_mock==1.11.0 respx==0.20.2 syrupy==4.2.1 tqdm==4.66.1 +types-aiofiles==22.1.0 types-atomicwrites==1.4.5.1 types-croniter==1.0.6 types-backports==0.1.3 diff --git a/requirements_test_all.txt b/requirements_test_all.txt index 3e3c6aab866e52..18e4f21914e482 100644 --- a/requirements_test_all.txt +++ b/requirements_test_all.txt @@ -33,6 +33,7 @@ HATasmota==0.7.0 # homeassistant.components.doods # homeassistant.components.generic # homeassistant.components.image_upload +# homeassistant.components.matrix # homeassistant.components.proxy # homeassistant.components.qrcode # homeassistant.components.seven_segments @@ -899,6 +900,9 @@ lxml==4.9.3 # homeassistant.components.nmap_tracker mac-vendor-lookup==0.1.12 +# homeassistant.components.matrix +matrix-nio==0.21.2 + # homeassistant.components.maxcube maxcube-api==0.4.3 diff --git a/tests/components/matrix/__init__.py b/tests/components/matrix/__init__.py new file mode 100644 index 00000000000000..a520f7e7c23d0e --- /dev/null +++ b/tests/components/matrix/__init__.py @@ -0,0 +1 @@ +"""Tests for the Matrix component.""" diff --git a/tests/components/matrix/conftest.py b/tests/components/matrix/conftest.py new file mode 100644 index 00000000000000..d0970b96019b17 --- /dev/null +++ b/tests/components/matrix/conftest.py @@ -0,0 +1,248 @@ +"""Define fixtures available for all tests.""" +from __future__ import annotations + +import re +import tempfile +from unittest.mock import patch + +from nio import ( + AsyncClient, + ErrorResponse, + JoinError, + JoinResponse, + LocalProtocolError, + LoginError, + LoginResponse, + Response, + UploadResponse, + WhoamiError, + WhoamiResponse, +) +from PIL import Image +import pytest + +from homeassistant.components.matrix import ( + CONF_COMMANDS, + CONF_EXPRESSION, + CONF_HOMESERVER, + CONF_ROOMS, + CONF_WORD, + EVENT_MATRIX_COMMAND, + MatrixBot, + RoomID, +) +from homeassistant.components.matrix.const import DOMAIN as MATRIX_DOMAIN +from homeassistant.components.matrix.notify import CONF_DEFAULT_ROOM +from homeassistant.components.notify import DOMAIN as NOTIFY_DOMAIN +from homeassistant.const import ( + CONF_NAME, + CONF_PASSWORD, + CONF_PLATFORM, + CONF_USERNAME, + CONF_VERIFY_SSL, +) +from homeassistant.core import HomeAssistant +from homeassistant.setup import async_setup_component + +from tests.common import async_capture_events + +TEST_NOTIFIER_NAME = "matrix_notify" + +TEST_DEFAULT_ROOM = "!DefaultNotificationRoom:example.com" +TEST_JOINABLE_ROOMS = ["!RoomIdString:example.com", "#RoomAliasString:example.com"] +TEST_BAD_ROOM = "!UninvitedRoom:example.com" +TEST_MXID = "@user:example.com" +TEST_DEVICE_ID = "FAKEID" +TEST_PASSWORD = "password" +TEST_TOKEN = "access_token" + +NIO_IMPORT_PREFIX = "homeassistant.components.matrix.nio." + + +class _MockAsyncClient(AsyncClient): + """Mock class to simulate MatrixBot._client's I/O methods.""" + + async def close(self): + return None + + async def join(self, room_id: RoomID): + if room_id in TEST_JOINABLE_ROOMS: + return JoinResponse(room_id=room_id) + else: + return JoinError(message="Not allowed to join this room.") + + async def login(self, *args, **kwargs): + if kwargs.get("password") == TEST_PASSWORD or kwargs.get("token") == TEST_TOKEN: + self.access_token = TEST_TOKEN + return LoginResponse( + access_token=TEST_TOKEN, + device_id="test_device", + user_id=TEST_MXID, + ) + else: + self.access_token = "" + return LoginError(message="LoginError", status_code="status_code") + + async def logout(self, *args, **kwargs): + self.access_token = "" + + async def whoami(self): + if self.access_token == TEST_TOKEN: + self.user_id = TEST_MXID + self.device_id = TEST_DEVICE_ID + return WhoamiResponse( + user_id=TEST_MXID, device_id=TEST_DEVICE_ID, is_guest=False + ) + else: + self.access_token = "" + return WhoamiError( + message="Invalid access token passed.", status_code="M_UNKNOWN_TOKEN" + ) + + async def room_send(self, *args, **kwargs): + if not self.logged_in: + raise LocalProtocolError + if kwargs["room_id"] in TEST_JOINABLE_ROOMS: + return Response() + else: + return ErrorResponse(message="Cannot send a message in this room.") + + async def sync(self, *args, **kwargs): + return None + + async def sync_forever(self, *args, **kwargs): + return None + + async def upload(self, *args, **kwargs): + return UploadResponse(content_uri="mxc://example.com/randomgibberish"), None + + +MOCK_CONFIG_DATA = { + MATRIX_DOMAIN: { + CONF_HOMESERVER: "https://matrix.example.com", + CONF_USERNAME: TEST_MXID, + CONF_PASSWORD: TEST_PASSWORD, + CONF_VERIFY_SSL: True, + CONF_ROOMS: TEST_JOINABLE_ROOMS, + CONF_COMMANDS: [ + { + CONF_WORD: "WordTrigger", + CONF_NAME: "WordTriggerEventName", + }, + { + CONF_EXPRESSION: "My name is (?P.*)", + CONF_NAME: "ExpressionTriggerEventName", + }, + ], + }, + NOTIFY_DOMAIN: { + CONF_NAME: TEST_NOTIFIER_NAME, + CONF_PLATFORM: MATRIX_DOMAIN, + CONF_DEFAULT_ROOM: TEST_DEFAULT_ROOM, + }, +} + +MOCK_WORD_COMMANDS = { + "!RoomIdString:example.com": { + "WordTrigger": { + "word": "WordTrigger", + "name": "WordTriggerEventName", + "rooms": ["!RoomIdString:example.com", "#RoomAliasString:example.com"], + } + }, + "#RoomAliasString:example.com": { + "WordTrigger": { + "word": "WordTrigger", + "name": "WordTriggerEventName", + "rooms": ["!RoomIdString:example.com", "#RoomAliasString:example.com"], + } + }, +} + +MOCK_EXPRESSION_COMMANDS = { + "!RoomIdString:example.com": [ + { + "expression": re.compile("My name is (?P.*)"), + "name": "ExpressionTriggerEventName", + "rooms": ["!RoomIdString:example.com", "#RoomAliasString:example.com"], + } + ], + "#RoomAliasString:example.com": [ + { + "expression": re.compile("My name is (?P.*)"), + "name": "ExpressionTriggerEventName", + "rooms": ["!RoomIdString:example.com", "#RoomAliasString:example.com"], + } + ], +} + + +@pytest.fixture +def mock_client(): + """Return mocked AsyncClient.""" + with patch("homeassistant.components.matrix.AsyncClient", _MockAsyncClient) as mock: + yield mock + + +@pytest.fixture +def mock_save_json(): + """Prevent saving test access_tokens.""" + with patch("homeassistant.components.matrix.save_json") as mock: + yield mock + + +@pytest.fixture +def mock_load_json(): + """Mock loading access_tokens from a file.""" + with patch( + "homeassistant.components.matrix.load_json_object", + return_value={TEST_MXID: TEST_TOKEN}, + ) as mock: + yield mock + + +@pytest.fixture +def mock_allowed_path(): + """Allow using NamedTemporaryFile for mock image.""" + with patch("homeassistant.core.Config.is_allowed_path", return_value=True) as mock: + yield mock + + +@pytest.fixture +async def matrix_bot( + hass: HomeAssistant, mock_client, mock_save_json, mock_allowed_path +) -> MatrixBot: + """Set up Matrix and Notify component. + + The resulting MatrixBot will have a mocked _client. + """ + + assert await async_setup_component(hass, MATRIX_DOMAIN, MOCK_CONFIG_DATA) + assert await async_setup_component(hass, NOTIFY_DOMAIN, MOCK_CONFIG_DATA) + await hass.async_block_till_done() + assert isinstance(matrix_bot := hass.data[MATRIX_DOMAIN], MatrixBot) + + await hass.async_start() + + return matrix_bot + + +@pytest.fixture +def matrix_events(hass: HomeAssistant): + """Track event calls.""" + return async_capture_events(hass, MATRIX_DOMAIN) + + +@pytest.fixture +def command_events(hass: HomeAssistant): + """Track event calls.""" + return async_capture_events(hass, EVENT_MATRIX_COMMAND) + + +@pytest.fixture +def image_path(tmp_path): + """Provide the Path to a mock image.""" + image = Image.new("RGBA", size=(50, 50), color=(256, 0, 0)) + image_file = tempfile.NamedTemporaryFile(dir=tmp_path) + image.save(image_file, "PNG") + return image_file diff --git a/tests/components/matrix/test_join_rooms.py b/tests/components/matrix/test_join_rooms.py new file mode 100644 index 00000000000000..54856b91ac3541 --- /dev/null +++ b/tests/components/matrix/test_join_rooms.py @@ -0,0 +1,22 @@ +"""Test MatrixBot._join.""" + +from homeassistant.components.matrix import MatrixBot + +from tests.components.matrix.conftest import TEST_BAD_ROOM, TEST_JOINABLE_ROOMS + + +async def test_join(matrix_bot: MatrixBot, caplog): + """Test joining configured rooms.""" + + # Join configured rooms. + await matrix_bot._join_rooms() + for room_id in TEST_JOINABLE_ROOMS: + assert f"Joined or already in room '{room_id}'" in caplog.messages + + # Joining a disallowed room should not raise an exception. + matrix_bot._listening_rooms = [TEST_BAD_ROOM] + await matrix_bot._join_rooms() + assert ( + f"Could not join room '{TEST_BAD_ROOM}': JoinError: Not allowed to join this room." + in caplog.messages + ) diff --git a/tests/components/matrix/test_login.py b/tests/components/matrix/test_login.py new file mode 100644 index 00000000000000..8112d98fc8c65f --- /dev/null +++ b/tests/components/matrix/test_login.py @@ -0,0 +1,118 @@ +"""Test MatrixBot._login.""" + +from pydantic.dataclasses import dataclass +import pytest + +from homeassistant.components.matrix import MatrixBot +from homeassistant.exceptions import ConfigEntryAuthFailed, HomeAssistantError + +from tests.components.matrix.conftest import ( + TEST_DEVICE_ID, + TEST_MXID, + TEST_PASSWORD, + TEST_TOKEN, +) + + +@dataclass +class LoginTestParameters: + """Dataclass of parameters representing the login parameters and expected result state.""" + + password: str + access_token: dict[str, str] + expected_login_state: bool + expected_caplog_messages: set[str] + expected_expection: type(Exception) | None = None + + +good_password_missing_token = LoginTestParameters( + password=TEST_PASSWORD, + access_token={}, + expected_login_state=True, + expected_caplog_messages={"Logging in using password"}, +) + +good_password_bad_token = LoginTestParameters( + password=TEST_PASSWORD, + access_token={TEST_MXID: "WrongToken"}, + expected_login_state=True, + expected_caplog_messages={ + "Restoring login from stored access token", + "Restoring login from access token failed: M_UNKNOWN_TOKEN, Invalid access token passed.", + "Logging in using password", + }, +) + +bad_password_good_access_token = LoginTestParameters( + password="WrongPassword", + access_token={TEST_MXID: TEST_TOKEN}, + expected_login_state=True, + expected_caplog_messages={ + "Restoring login from stored access token", + f"Successfully restored login from access token: user_id '{TEST_MXID}', device_id '{TEST_DEVICE_ID}'", + }, +) + +bad_password_bad_access_token = LoginTestParameters( + password="WrongPassword", + access_token={TEST_MXID: "WrongToken"}, + expected_login_state=False, + expected_caplog_messages={ + "Restoring login from stored access token", + "Restoring login from access token failed: M_UNKNOWN_TOKEN, Invalid access token passed.", + "Logging in using password", + "Login by password failed: status_code, LoginError", + }, + expected_expection=ConfigEntryAuthFailed, +) + +bad_password_missing_access_token = LoginTestParameters( + password="WrongPassword", + access_token={}, + expected_login_state=False, + expected_caplog_messages={ + "Logging in using password", + "Login by password failed: status_code, LoginError", + }, + expected_expection=ConfigEntryAuthFailed, +) + + +@pytest.mark.parametrize( + "params", + [ + good_password_missing_token, + good_password_bad_token, + bad_password_good_access_token, + bad_password_bad_access_token, + bad_password_missing_access_token, + ], +) +async def test_login( + matrix_bot: MatrixBot, caplog: pytest.LogCaptureFixture, params: LoginTestParameters +): + """Test logging in with the given parameters and expected state.""" + await matrix_bot._client.logout() + matrix_bot._password = params.password + matrix_bot._access_tokens = params.access_token + + if params.expected_expection: + with pytest.raises(params.expected_expection): + await matrix_bot._login() + else: + await matrix_bot._login() + assert matrix_bot._client.logged_in == params.expected_login_state + assert set(caplog.messages).issuperset(params.expected_caplog_messages) + + +async def test_get_auth_tokens(matrix_bot: MatrixBot, mock_load_json): + """Test loading access_tokens from a mocked file.""" + + # Test loading good tokens. + loaded_tokens = await matrix_bot._get_auth_tokens() + assert loaded_tokens == {TEST_MXID: TEST_TOKEN} + + # Test miscellaneous error from hass. + mock_load_json.side_effect = HomeAssistantError() + loaded_tokens = await matrix_bot._get_auth_tokens() + assert loaded_tokens == {} diff --git a/tests/components/matrix/test_matrix_bot.py b/tests/components/matrix/test_matrix_bot.py new file mode 100644 index 00000000000000..0b150a629fe23a --- /dev/null +++ b/tests/components/matrix/test_matrix_bot.py @@ -0,0 +1,88 @@ +"""Configure and test MatrixBot.""" +from nio import MatrixRoom, RoomMessageText + +from homeassistant.components.matrix import ( + DOMAIN as MATRIX_DOMAIN, + SERVICE_SEND_MESSAGE, + MatrixBot, +) +from homeassistant.components.notify import DOMAIN as NOTIFY_DOMAIN +from homeassistant.core import HomeAssistant + +from .conftest import ( + MOCK_EXPRESSION_COMMANDS, + MOCK_WORD_COMMANDS, + TEST_JOINABLE_ROOMS, + TEST_NOTIFIER_NAME, +) + + +async def test_services(hass: HomeAssistant, matrix_bot: MatrixBot): + """Test hass/MatrixBot state.""" + + services = hass.services.async_services() + + # Verify that the matrix service is registered + assert (matrix_service := services.get(MATRIX_DOMAIN)) + assert SERVICE_SEND_MESSAGE in matrix_service + + # Verify that the matrix notifier is registered + assert (notify_service := services.get(NOTIFY_DOMAIN)) + assert TEST_NOTIFIER_NAME in notify_service + + +async def test_commands(hass, matrix_bot: MatrixBot, command_events): + """Test that the configured commands were parsed correctly.""" + + assert len(command_events) == 0 + + assert matrix_bot._word_commands == MOCK_WORD_COMMANDS + assert matrix_bot._expression_commands == MOCK_EXPRESSION_COMMANDS + + room_id = TEST_JOINABLE_ROOMS[0] + room = MatrixRoom(room_id=room_id, own_user_id=matrix_bot._mx_id) + + # Test single-word command. + word_command_message = RoomMessageText( + body="!WordTrigger arg1 arg2", + formatted_body=None, + format=None, + source={ + "event_id": "fake_event_id", + "sender": "@SomeUser:example.com", + "origin_server_ts": 123456789, + }, + ) + await matrix_bot._handle_room_message(room, word_command_message) + await hass.async_block_till_done() + assert len(command_events) == 1 + event = command_events.pop() + assert event.data == { + "command": "WordTriggerEventName", + "sender": "@SomeUser:example.com", + "room": room_id, + "args": ["arg1", "arg2"], + } + + # Test expression command. + room = MatrixRoom(room_id=room_id, own_user_id=matrix_bot._mx_id) + expression_command_message = RoomMessageText( + body="My name is FakeName", + formatted_body=None, + format=None, + source={ + "event_id": "fake_event_id", + "sender": "@SomeUser:example.com", + "origin_server_ts": 123456789, + }, + ) + await matrix_bot._handle_room_message(room, expression_command_message) + await hass.async_block_till_done() + assert len(command_events) == 1 + event = command_events.pop() + assert event.data == { + "command": "ExpressionTriggerEventName", + "sender": "@SomeUser:example.com", + "room": room_id, + "args": {"name": "FakeName"}, + } diff --git a/tests/components/matrix/test_send_message.py b/tests/components/matrix/test_send_message.py new file mode 100644 index 00000000000000..34964f2b09132a --- /dev/null +++ b/tests/components/matrix/test_send_message.py @@ -0,0 +1,71 @@ +"""Test the send_message service.""" + +from homeassistant.components.matrix import ( + ATTR_FORMAT, + ATTR_IMAGES, + DOMAIN as MATRIX_DOMAIN, + MatrixBot, +) +from homeassistant.components.matrix.const import FORMAT_HTML, SERVICE_SEND_MESSAGE +from homeassistant.components.notify import ATTR_DATA, ATTR_MESSAGE, ATTR_TARGET +from homeassistant.core import HomeAssistant + +from tests.components.matrix.conftest import TEST_BAD_ROOM, TEST_JOINABLE_ROOMS + + +async def test_send_message( + hass: HomeAssistant, matrix_bot: MatrixBot, image_path, matrix_events, caplog +): + """Test the send_message service.""" + assert len(matrix_events) == 0 + await matrix_bot._login() + + # Send a message without an attached image. + data = {ATTR_MESSAGE: "Test message", ATTR_TARGET: TEST_JOINABLE_ROOMS} + await hass.services.async_call( + MATRIX_DOMAIN, SERVICE_SEND_MESSAGE, data, blocking=True + ) + + for room_id in TEST_JOINABLE_ROOMS: + assert f"Message delivered to room '{room_id}'" in caplog.messages + + # Send an HTML message without an attached image. + data = { + ATTR_MESSAGE: "Test message", + ATTR_TARGET: TEST_JOINABLE_ROOMS, + ATTR_DATA: {ATTR_FORMAT: FORMAT_HTML}, + } + await hass.services.async_call( + MATRIX_DOMAIN, SERVICE_SEND_MESSAGE, data, blocking=True + ) + + for room_id in TEST_JOINABLE_ROOMS: + assert f"Message delivered to room '{room_id}'" in caplog.messages + + # Send a message with an attached image. + data[ATTR_DATA] = {ATTR_IMAGES: [image_path.name]} + await hass.services.async_call( + MATRIX_DOMAIN, SERVICE_SEND_MESSAGE, data, blocking=True + ) + + for room_id in TEST_JOINABLE_ROOMS: + assert f"Message delivered to room '{room_id}'" in caplog.messages + + +async def test_unsendable_message( + hass: HomeAssistant, matrix_bot: MatrixBot, matrix_events, caplog +): + """Test the send_message service with an invalid room.""" + assert len(matrix_events) == 0 + await matrix_bot._login() + + data = {ATTR_MESSAGE: "Test message", ATTR_TARGET: TEST_BAD_ROOM} + + await hass.services.async_call( + MATRIX_DOMAIN, SERVICE_SEND_MESSAGE, data, blocking=True + ) + + assert ( + f"Unable to deliver message to room '{TEST_BAD_ROOM}': ErrorResponse: Cannot send a message in this room." + in caplog.messages + ) From d88ee0dbe0253ec5e8895af1836bd543ec2e2c7b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Hjelseth=20H=C3=B8yer?= Date: Sat, 2 Sep 2023 15:08:49 +0200 Subject: [PATCH 02/12] Update Tibber library to 0.28.2 (#99115) --- homeassistant/components/tibber/manifest.json | 2 +- requirements_all.txt | 2 +- requirements_test_all.txt | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/homeassistant/components/tibber/manifest.json b/homeassistant/components/tibber/manifest.json index c668430914fae7..1d8120a4321e38 100644 --- a/homeassistant/components/tibber/manifest.json +++ b/homeassistant/components/tibber/manifest.json @@ -8,5 +8,5 @@ "iot_class": "cloud_polling", "loggers": ["tibber"], "quality_scale": "silver", - "requirements": ["pyTibber==0.28.0"] + "requirements": ["pyTibber==0.28.2"] } diff --git a/requirements_all.txt b/requirements_all.txt index 4c5497ae98c981..d72b41f442e08a 100644 --- a/requirements_all.txt +++ b/requirements_all.txt @@ -1548,7 +1548,7 @@ pyRFXtrx==0.30.1 pySDCP==1 # homeassistant.components.tibber -pyTibber==0.28.0 +pyTibber==0.28.2 # homeassistant.components.dlink pyW215==0.7.0 diff --git a/requirements_test_all.txt b/requirements_test_all.txt index 18e4f21914e482..3049cb33268d82 100644 --- a/requirements_test_all.txt +++ b/requirements_test_all.txt @@ -1163,7 +1163,7 @@ pyElectra==1.2.0 pyRFXtrx==0.30.1 # homeassistant.components.tibber -pyTibber==0.28.0 +pyTibber==0.28.2 # homeassistant.components.dlink pyW215==0.7.0 From 26b1222faedd72be7c6f0f53209976ca7a34f978 Mon Sep 17 00:00:00 2001 From: Jc2k Date: Sat, 2 Sep 2023 15:21:05 +0100 Subject: [PATCH 03/12] Support tracking private bluetooth devices (#99465) Co-authored-by: J. Nick Koston --- .strict-typing | 1 + CODEOWNERS | 2 + .../components/private_ble_device/__init__.py | 19 ++ .../private_ble_device/config_flow.py | 60 +++++ .../components/private_ble_device/const.py | 2 + .../private_ble_device/coordinator.py | 236 ++++++++++++++++++ .../private_ble_device/device_tracker.py | 75 ++++++ .../components/private_ble_device/entity.py | 71 ++++++ .../private_ble_device/manifest.json | 10 + .../private_ble_device/strings.json | 20 ++ homeassistant/generated/config_flows.py | 1 + homeassistant/generated/integrations.json | 6 + mypy.ini | 10 + requirements_all.txt | 1 + requirements_test_all.txt | 1 + .../components/private_ble_device/__init__.py | 78 ++++++ .../components/private_ble_device/conftest.py | 1 + .../private_ble_device/test_config_flow.py | 132 ++++++++++ .../private_ble_device/test_device_tracker.py | 183 ++++++++++++++ 19 files changed, 909 insertions(+) create mode 100644 homeassistant/components/private_ble_device/__init__.py create mode 100644 homeassistant/components/private_ble_device/config_flow.py create mode 100644 homeassistant/components/private_ble_device/const.py create mode 100644 homeassistant/components/private_ble_device/coordinator.py create mode 100644 homeassistant/components/private_ble_device/device_tracker.py create mode 100644 homeassistant/components/private_ble_device/entity.py create mode 100644 homeassistant/components/private_ble_device/manifest.json create mode 100644 homeassistant/components/private_ble_device/strings.json create mode 100644 tests/components/private_ble_device/__init__.py create mode 100644 tests/components/private_ble_device/conftest.py create mode 100644 tests/components/private_ble_device/test_config_flow.py create mode 100644 tests/components/private_ble_device/test_device_tracker.py diff --git a/.strict-typing b/.strict-typing index 3059c42f33fab6..2a6e9b04cbe7ac 100644 --- a/.strict-typing +++ b/.strict-typing @@ -255,6 +255,7 @@ homeassistant.components.persistent_notification.* homeassistant.components.pi_hole.* homeassistant.components.ping.* homeassistant.components.powerwall.* +homeassistant.components.private_ble_device.* homeassistant.components.proximity.* homeassistant.components.prusalink.* homeassistant.components.pure_energie.* diff --git a/CODEOWNERS b/CODEOWNERS index bf6fdaf9fc5713..b937c2769fce18 100644 --- a/CODEOWNERS +++ b/CODEOWNERS @@ -951,6 +951,8 @@ build.json @home-assistant/supervisor /tests/components/poolsense/ @haemishkyd /homeassistant/components/powerwall/ @bdraco @jrester @daniel-simpson /tests/components/powerwall/ @bdraco @jrester @daniel-simpson +/homeassistant/components/private_ble_device/ @Jc2k +/tests/components/private_ble_device/ @Jc2k /homeassistant/components/profiler/ @bdraco /tests/components/profiler/ @bdraco /homeassistant/components/progettihwsw/ @ardaseremet diff --git a/homeassistant/components/private_ble_device/__init__.py b/homeassistant/components/private_ble_device/__init__.py new file mode 100644 index 00000000000000..c4666ccc02fa70 --- /dev/null +++ b/homeassistant/components/private_ble_device/__init__.py @@ -0,0 +1,19 @@ +"""Private BLE Device integration.""" +from __future__ import annotations + +from homeassistant.config_entries import ConfigEntry +from homeassistant.const import Platform +from homeassistant.core import HomeAssistant + +PLATFORMS = [Platform.DEVICE_TRACKER] + + +async def async_setup_entry(hass: HomeAssistant, entry: ConfigEntry) -> bool: + """Set up tracking of a private bluetooth device from a config entry.""" + await hass.config_entries.async_forward_entry_setups(entry, PLATFORMS) + return True + + +async def async_unload_entry(hass: HomeAssistant, entry: ConfigEntry) -> bool: + """Unload entities for a config entry.""" + return await hass.config_entries.async_unload_platforms(entry, PLATFORMS) diff --git a/homeassistant/components/private_ble_device/config_flow.py b/homeassistant/components/private_ble_device/config_flow.py new file mode 100644 index 00000000000000..5bf130a0396fff --- /dev/null +++ b/homeassistant/components/private_ble_device/config_flow.py @@ -0,0 +1,60 @@ +"""Config flow for the BLE Tracker.""" +from __future__ import annotations + +import base64 +import binascii +import logging + +import voluptuous as vol + +from homeassistant.components import bluetooth +from homeassistant.config_entries import ConfigFlow +from homeassistant.data_entry_flow import FlowResult + +from .const import DOMAIN +from .coordinator import async_last_service_info + +_LOGGER = logging.getLogger(__name__) + +CONF_IRK = "irk" + + +class BLEDeviceTrackerConfigFlow(ConfigFlow, domain=DOMAIN): + """Handle a config flow for BLE Device Tracker.""" + + VERSION = 1 + + async def async_step_user( + self, user_input: dict[str, str] | None = None + ) -> FlowResult: + """Set up by user.""" + errors: dict[str, str] = {} + + if not bluetooth.async_scanner_count(self.hass, connectable=False): + return self.async_abort(reason="bluetooth_not_available") + + if user_input is not None: + irk = user_input[CONF_IRK] + if irk.startswith("irk:"): + irk = irk[4:] + + if irk.endswith("="): + irk_bytes = bytes(reversed(base64.b64decode(irk))) + else: + irk_bytes = binascii.unhexlify(irk) + + if len(irk_bytes) != 16: + errors[CONF_IRK] = "irk_not_valid" + elif not (service_info := async_last_service_info(self.hass, irk_bytes)): + errors[CONF_IRK] = "irk_not_found" + else: + await self.async_set_unique_id(irk_bytes.hex()) + return self.async_create_entry( + title=service_info.name or "BLE Device Tracker", + data={CONF_IRK: irk_bytes.hex()}, + ) + + data_schema = vol.Schema({CONF_IRK: str}) + return self.async_show_form( + step_id="user", data_schema=data_schema, errors=errors + ) diff --git a/homeassistant/components/private_ble_device/const.py b/homeassistant/components/private_ble_device/const.py new file mode 100644 index 00000000000000..086fd06bfd5780 --- /dev/null +++ b/homeassistant/components/private_ble_device/const.py @@ -0,0 +1,2 @@ +"""Constants for Private BLE Device.""" +DOMAIN = "private_ble_device" diff --git a/homeassistant/components/private_ble_device/coordinator.py b/homeassistant/components/private_ble_device/coordinator.py new file mode 100644 index 00000000000000..863b283385175b --- /dev/null +++ b/homeassistant/components/private_ble_device/coordinator.py @@ -0,0 +1,236 @@ +"""Central manager for tracking devices with random but resolvable MAC addresses.""" +from __future__ import annotations + +from collections.abc import Callable +import logging +from typing import cast + +from bluetooth_data_tools import get_cipher_for_irk, resolve_private_address +from cryptography.hazmat.primitives.ciphers import Cipher + +from homeassistant.components import bluetooth +from homeassistant.components.bluetooth.match import BluetoothCallbackMatcher +from homeassistant.core import HomeAssistant + +from .const import DOMAIN + +_LOGGER = logging.getLogger(__name__) + +UnavailableCallback = Callable[[bluetooth.BluetoothServiceInfoBleak], None] +Cancellable = Callable[[], None] + + +def async_last_service_info( + hass: HomeAssistant, irk: bytes +) -> bluetooth.BluetoothServiceInfoBleak | None: + """Find a BluetoothServiceInfoBleak for the irk. + + This iterates over all currently visible mac addresses and checks them against `irk`. + It returns the newest. + """ + + # This can't use existing data collected by the coordinator - its called when + # the coordinator doesn't know about the IRK, so doesn't optimise this lookup. + + cur: bluetooth.BluetoothServiceInfoBleak | None = None + cipher = get_cipher_for_irk(irk) + + for service_info in bluetooth.async_discovered_service_info(hass, False): + if resolve_private_address(cipher, service_info.address): + if not cur or cur.time < service_info.time: + cur = service_info + + return cur + + +class PrivateDevicesCoordinator: + """Monitor private bluetooth devices and correlate them with known IRK. + + This class should not be instanced directly - use `async_get_coordinator` to get an instance. + + There is a single shared coordinator for all instances of this integration. This is to avoid + unnecessary hashing (AES) operations as much as possible. + """ + + def __init__(self, hass: HomeAssistant) -> None: + """Initialize the manager.""" + self.hass = hass + + self._irks: dict[bytes, Cipher] = {} + self._unavailable_callbacks: dict[bytes, list[UnavailableCallback]] = {} + self._service_info_callbacks: dict[ + bytes, list[bluetooth.BluetoothCallback] + ] = {} + + self._mac_to_irk: dict[str, bytes] = {} + self._irk_to_mac: dict[bytes, str] = {} + + # These MAC addresses have been compared to the IRK list + # They are unknown, so we can ignore them. + self._ignored: dict[str, Cancellable] = {} + + self._unavailability_trackers: dict[bytes, Cancellable] = {} + self._listener_cancel: Cancellable | None = None + + def _async_ensure_started(self) -> None: + if not self._listener_cancel: + self._listener_cancel = bluetooth.async_register_callback( + self.hass, + self._async_track_service_info, + BluetoothCallbackMatcher(connectable=False), + bluetooth.BluetoothScanningMode.ACTIVE, + ) + + def _async_ensure_stopped(self) -> None: + if self._listener_cancel: + self._listener_cancel() + self._listener_cancel = None + + for cancel in self._ignored.values(): + cancel() + self._ignored.clear() + + def _async_track_unavailable( + self, service_info: bluetooth.BluetoothServiceInfoBleak + ) -> None: + # This should be called when the current MAC address associated with an IRK goes away. + if resolved := self._mac_to_irk.get(service_info.address): + if callbacks := self._unavailable_callbacks.get(resolved): + for cb in callbacks: + cb(service_info) + return + + def _async_irk_resolved_to_mac(self, irk: bytes, mac: str) -> None: + if previous_mac := self._irk_to_mac.get(irk): + self._mac_to_irk.pop(previous_mac, None) + + self._mac_to_irk[mac] = irk + self._irk_to_mac[irk] = mac + + # Stop ignoring this MAC + self._ignored.pop(mac, None) + + # Ignore availability events for the previous address + if cancel := self._unavailability_trackers.pop(irk, None): + cancel() + + # Track available for new address + self._unavailability_trackers[irk] = bluetooth.async_track_unavailable( + self.hass, self._async_track_unavailable, mac, False + ) + + def _async_track_service_info( + self, + service_info: bluetooth.BluetoothServiceInfoBleak, + change: bluetooth.BluetoothChange, + ) -> None: + mac = service_info.address + + if mac in self._ignored: + return + + if resolved := self._mac_to_irk.get(mac): + if callbacks := self._service_info_callbacks.get(resolved): + for cb in callbacks: + cb(service_info, change) + return + + for irk, cipher in self._irks.items(): + if resolve_private_address(cipher, service_info.address): + self._async_irk_resolved_to_mac(irk, mac) + if callbacks := self._service_info_callbacks.get(irk): + for cb in callbacks: + cb(service_info, change) + return + + def _unignore(service_info: bluetooth.BluetoothServiceInfoBleak) -> None: + self._ignored.pop(service_info.address, None) + + self._ignored[mac] = bluetooth.async_track_unavailable( + self.hass, _unignore, mac, False + ) + + def _async_maybe_learn_irk(self, irk: bytes) -> None: + """Add irk to list of irks that we can use to resolve RPAs.""" + if irk not in self._irks: + if service_info := async_last_service_info(self.hass, irk): + self._async_irk_resolved_to_mac(irk, service_info.address) + self._irks[irk] = get_cipher_for_irk(irk) + + def _async_maybe_forget_irk(self, irk: bytes) -> None: + """If no downstream caller is tracking this irk, lets forget it.""" + if irk in self._service_info_callbacks or irk in self._unavailable_callbacks: + return + + # Ignore availability events for this irk as no + # one is listening. + if cancel := self._unavailability_trackers.pop(irk, None): + cancel() + + del self._irks[irk] + + if mac := self._irk_to_mac.pop(irk, None): + self._mac_to_irk.pop(mac, None) + + if not self._mac_to_irk: + self._async_ensure_stopped() + + def async_track_service_info( + self, callback: bluetooth.BluetoothCallback, irk: bytes + ) -> Cancellable: + """Receive a callback when a new advertisement is received for an irk. + + Returns a callback that can be used to cancel the registration. + """ + self._async_ensure_started() + self._async_maybe_learn_irk(irk) + + callbacks = self._service_info_callbacks.setdefault(irk, []) + callbacks.append(callback) + + def _unsubscribe() -> None: + callbacks.remove(callback) + if not callbacks: + self._service_info_callbacks.pop(irk, None) + self._async_maybe_forget_irk(irk) + + return _unsubscribe + + def async_track_unavailable( + self, + callback: UnavailableCallback, + irk: bytes, + ) -> Cancellable: + """Register to receive a callback when an irk is unavailable. + + Returns a callback that can be used to cancel the registration. + """ + self._async_ensure_started() + self._async_maybe_learn_irk(irk) + + callbacks = self._unavailable_callbacks.setdefault(irk, []) + callbacks.append(callback) + + def _unsubscribe() -> None: + callbacks.remove(callback) + if not callbacks: + self._unavailable_callbacks.pop(irk, None) + + self._async_maybe_forget_irk(irk) + + return _unsubscribe + + +def async_get_coordinator(hass: HomeAssistant) -> PrivateDevicesCoordinator: + """Create or return an existing PrivateDeviceManager. + + There should only be one per HomeAssistant instance. Associating private + mac addresses with an IRK involves AES operations. We don't want to + duplicate that work. + """ + if existing := hass.data.get(DOMAIN): + return cast(PrivateDevicesCoordinator, existing) + + pdm = hass.data[DOMAIN] = PrivateDevicesCoordinator(hass) + + return pdm diff --git a/homeassistant/components/private_ble_device/device_tracker.py b/homeassistant/components/private_ble_device/device_tracker.py new file mode 100644 index 00000000000000..64e23b25ebec78 --- /dev/null +++ b/homeassistant/components/private_ble_device/device_tracker.py @@ -0,0 +1,75 @@ +"""Tracking for bluetooth low energy devices.""" +from __future__ import annotations + +from collections.abc import Mapping +import logging + +from homeassistant.components import bluetooth +from homeassistant.components.device_tracker import SourceType +from homeassistant.components.device_tracker.config_entry import BaseTrackerEntity +from homeassistant.config_entries import ConfigEntry +from homeassistant.const import STATE_HOME, STATE_NOT_HOME +from homeassistant.core import HomeAssistant, callback +from homeassistant.helpers.entity_platform import AddEntitiesCallback + +from .entity import BasePrivateDeviceEntity + +_LOGGER = logging.getLogger(__name__) + + +async def async_setup_entry( + hass: HomeAssistant, + config_entry: ConfigEntry, + async_add_entities: AddEntitiesCallback, +) -> None: + """Load Device Tracker entities for a config entry.""" + async_add_entities([BasePrivateDeviceTracker(config_entry)]) + + +class BasePrivateDeviceTracker(BasePrivateDeviceEntity, BaseTrackerEntity): + """A trackable Private Bluetooth Device.""" + + _attr_should_poll = False + _attr_has_entity_name = True + _attr_name = None + + @property + def extra_state_attributes(self) -> Mapping[str, str]: + """Return extra state attributes for this device.""" + if last_info := self._last_info: + return { + "current_address": last_info.address, + "source": last_info.source, + } + return {} + + @callback + def _async_track_unavailable( + self, service_info: bluetooth.BluetoothServiceInfoBleak + ) -> None: + self._last_info = None + self.async_write_ha_state() + + @callback + def _async_track_service_info( + self, + service_info: bluetooth.BluetoothServiceInfoBleak, + change: bluetooth.BluetoothChange, + ) -> None: + self._last_info = service_info + self.async_write_ha_state() + + @property + def state(self) -> str: + """Return the state of the device.""" + return STATE_HOME if self._last_info else STATE_NOT_HOME + + @property + def source_type(self) -> SourceType: + """Return the source type, eg gps or router, of the device.""" + return SourceType.BLUETOOTH_LE + + @property + def icon(self) -> str: + """Return device icon.""" + return "mdi:bluetooth-connect" if self._last_info else "mdi:bluetooth-off" diff --git a/homeassistant/components/private_ble_device/entity.py b/homeassistant/components/private_ble_device/entity.py new file mode 100644 index 00000000000000..ae632213506609 --- /dev/null +++ b/homeassistant/components/private_ble_device/entity.py @@ -0,0 +1,71 @@ +"""Tracking for bluetooth low energy devices.""" +from __future__ import annotations + +from abc import abstractmethod +import binascii + +from homeassistant.components import bluetooth +from homeassistant.config_entries import ConfigEntry +from homeassistant.core import callback +from homeassistant.helpers.device_registry import DeviceInfo +from homeassistant.helpers.entity import Entity + +from .const import DOMAIN +from .coordinator import async_get_coordinator, async_last_service_info + + +class BasePrivateDeviceEntity(Entity): + """Base Private Bluetooth Entity.""" + + _attr_should_poll = False + _attr_has_entity_name = True + + def __init__(self, config_entry: ConfigEntry) -> None: + """Set up a new BleScanner entity.""" + irk = config_entry.data["irk"] + + self._attr_unique_id = irk + + self._attr_device_info = DeviceInfo( + name=f"Private BLE Device {irk[:6]}", + identifiers={(DOMAIN, irk)}, + ) + + self._entry = config_entry + self._irk = binascii.unhexlify(irk) + self._last_info: bluetooth.BluetoothServiceInfoBleak | None = None + + async def async_added_to_hass(self) -> None: + """Configure entity when it is added to Home Assistant.""" + coordinator = async_get_coordinator(self.hass) + self.async_on_remove( + coordinator.async_track_service_info( + self._async_track_service_info, self._irk + ) + ) + self.async_on_remove( + coordinator.async_track_unavailable( + self._async_track_unavailable, self._irk + ) + ) + + if service_info := async_last_service_info(self.hass, self._irk): + self._async_track_service_info( + service_info, bluetooth.BluetoothChange.ADVERTISEMENT + ) + + @abstractmethod + @callback + def _async_track_unavailable( + self, service_info: bluetooth.BluetoothServiceInfoBleak + ) -> None: + """Respond when the bluetooth device being tracked is no longer visible.""" + + @abstractmethod + @callback + def _async_track_service_info( + self, + service_info: bluetooth.BluetoothServiceInfoBleak, + change: bluetooth.BluetoothChange, + ) -> None: + """Respond when the bluetooth device being tracked broadcasted updated information.""" diff --git a/homeassistant/components/private_ble_device/manifest.json b/homeassistant/components/private_ble_device/manifest.json new file mode 100644 index 00000000000000..3497138178cfba --- /dev/null +++ b/homeassistant/components/private_ble_device/manifest.json @@ -0,0 +1,10 @@ +{ + "domain": "private_ble_device", + "name": "Private BLE Device", + "codeowners": ["@Jc2k"], + "config_flow": true, + "dependencies": ["bluetooth_adapters"], + "documentation": "https://www.home-assistant.io/integrations/private_ble_device", + "iot_class": "local_push", + "requirements": ["bluetooth-data-tools==1.11.0"] +} diff --git a/homeassistant/components/private_ble_device/strings.json b/homeassistant/components/private_ble_device/strings.json new file mode 100644 index 00000000000000..c62ea5c4d50f01 --- /dev/null +++ b/homeassistant/components/private_ble_device/strings.json @@ -0,0 +1,20 @@ +{ + "config": { + "flow_title": "[%key:component::bluetooth::config::flow_title%]", + "step": { + "user": { + "description": "What is the IRK (Identity Resolving Key) of the BLE device you want to track?", + "data": { + "irk": "IRK" + } + } + }, + "error": { + "irk_not_found": "The provided IRK does not match any BLE devices that Home Assistant can see.", + "irk_not_valid": "The key does not look like a valid IRK." + }, + "abort": { + "bluetooth_not_available": "At least one Bluetooth adapter or remote bluetooth proxy must be configured to track Private BLE Devices." + } + } +} diff --git a/homeassistant/generated/config_flows.py b/homeassistant/generated/config_flows.py index 7d84dc87cbe047..6c992fd4b5e4b7 100644 --- a/homeassistant/generated/config_flows.py +++ b/homeassistant/generated/config_flows.py @@ -351,6 +351,7 @@ "point", "poolsense", "powerwall", + "private_ble_device", "profiler", "progettihwsw", "prosegur", diff --git a/homeassistant/generated/integrations.json b/homeassistant/generated/integrations.json index c357b5aed4c9aa..a9e19441693b57 100644 --- a/homeassistant/generated/integrations.json +++ b/homeassistant/generated/integrations.json @@ -4320,6 +4320,12 @@ "config_flow": true, "iot_class": "cloud_polling" }, + "private_ble_device": { + "name": "Private BLE Device", + "integration_type": "hub", + "config_flow": true, + "iot_class": "local_push" + }, "profiler": { "name": "Profiler", "integration_type": "hub", diff --git a/mypy.ini b/mypy.ini index 9802c26c3c6461..178b82fd359c0e 100644 --- a/mypy.ini +++ b/mypy.ini @@ -2312,6 +2312,16 @@ disallow_untyped_defs = true warn_return_any = true warn_unreachable = true +[mypy-homeassistant.components.private_ble_device.*] +check_untyped_defs = true +disallow_incomplete_defs = true +disallow_subclassing_any = true +disallow_untyped_calls = true +disallow_untyped_decorators = true +disallow_untyped_defs = true +warn_return_any = true +warn_unreachable = true + [mypy-homeassistant.components.proximity.*] check_untyped_defs = true disallow_incomplete_defs = true diff --git a/requirements_all.txt b/requirements_all.txt index d72b41f442e08a..be7a06399d2a58 100644 --- a/requirements_all.txt +++ b/requirements_all.txt @@ -550,6 +550,7 @@ bluetooth-auto-recovery==1.2.1 # homeassistant.components.esphome # homeassistant.components.ld2410_ble # homeassistant.components.led_ble +# homeassistant.components.private_ble_device bluetooth-data-tools==1.11.0 # homeassistant.components.bond diff --git a/requirements_test_all.txt b/requirements_test_all.txt index 3049cb33268d82..5362d5ac2b5e59 100644 --- a/requirements_test_all.txt +++ b/requirements_test_all.txt @@ -461,6 +461,7 @@ bluetooth-auto-recovery==1.2.1 # homeassistant.components.esphome # homeassistant.components.ld2410_ble # homeassistant.components.led_ble +# homeassistant.components.private_ble_device bluetooth-data-tools==1.11.0 # homeassistant.components.bond diff --git a/tests/components/private_ble_device/__init__.py b/tests/components/private_ble_device/__init__.py new file mode 100644 index 00000000000000..df9929293a1526 --- /dev/null +++ b/tests/components/private_ble_device/__init__.py @@ -0,0 +1,78 @@ +"""Tests for private_ble_device.""" + +from datetime import timedelta +import time +from unittest.mock import patch + +from home_assistant_bluetooth import BluetoothServiceInfoBleak + +from homeassistant import config_entries +from homeassistant.components.private_ble_device.const import DOMAIN +from homeassistant.core import HomeAssistant +from homeassistant.util import dt as dt_util + +from tests.common import MockConfigEntry, async_fire_time_changed +from tests.components.bluetooth import ( + generate_advertisement_data, + generate_ble_device, + inject_bluetooth_service_info_bleak, +) + +MAC_RPA_VALID_1 = "40:01:02:0a:c4:a6" +MAC_RPA_VALID_2 = "40:02:03:d2:74:ce" +MAC_RPA_INVALID = "40:00:00:d2:74:ce" +MAC_STATIC = "00:01:ff:a0:3a:76" + +DUMMY_IRK = "00000000000000000000000000000000" + + +async def async_mock_config_entry(hass: HomeAssistant, irk: str = DUMMY_IRK) -> None: + """Create a test device for a dummy IRK.""" + entry = MockConfigEntry( + version=1, + domain=DOMAIN, + entry_id=irk, + data={"irk": irk}, + title="Private BLE Device 000000", + ) + entry.add_to_hass(hass) + + assert await hass.config_entries.async_setup(entry.entry_id) + assert entry.state is config_entries.ConfigEntryState.LOADED + await hass.async_block_till_done() + + +async def async_inject_broadcast( + hass: HomeAssistant, + mac: str = MAC_RPA_VALID_1, + mfr_data: bytes = b"", + broadcast_time: float | None = None, +) -> None: + """Inject an advertisement.""" + inject_bluetooth_service_info_bleak( + hass, + BluetoothServiceInfoBleak( + name="Test Test Test", + address=mac, + rssi=-63, + service_data={}, + manufacturer_data={1: mfr_data}, + service_uuids=[], + source="local", + device=generate_ble_device(mac, "Test Test Test"), + advertisement=generate_advertisement_data(local_name="Not it"), + time=broadcast_time or time.monotonic(), + connectable=False, + ), + ) + await hass.async_block_till_done() + + +async def async_move_time_forwards(hass: HomeAssistant, offset: float): + """Mock time advancing from now to now+offset.""" + with patch( + "homeassistant.components.bluetooth.manager.MONOTONIC_TIME", + return_value=time.monotonic() + offset, + ): + async_fire_time_changed(hass, dt_util.utcnow() + timedelta(seconds=offset)) + await hass.async_block_till_done() diff --git a/tests/components/private_ble_device/conftest.py b/tests/components/private_ble_device/conftest.py new file mode 100644 index 00000000000000..b33dc1d4ea2109 --- /dev/null +++ b/tests/components/private_ble_device/conftest.py @@ -0,0 +1 @@ +"""private_ble_device fixtures.""" diff --git a/tests/components/private_ble_device/test_config_flow.py b/tests/components/private_ble_device/test_config_flow.py new file mode 100644 index 00000000000000..aa8ea0d905c514 --- /dev/null +++ b/tests/components/private_ble_device/test_config_flow.py @@ -0,0 +1,132 @@ +"""Tests for private bluetooth device config flow.""" +from unittest.mock import patch + +from homeassistant import config_entries +from homeassistant.components.private_ble_device import const +from homeassistant.core import HomeAssistant +from homeassistant.data_entry_flow import FlowResult, FlowResultType +from homeassistant.helpers.service_info.bluetooth import BluetoothServiceInfo + +from tests.components.bluetooth import inject_bluetooth_service_info + + +def assert_form_error(result: FlowResult, key: str, value: str) -> None: + """Assert that a flow returned a form error.""" + assert result["type"] == "form" + assert result["errors"] + assert result["errors"][key] == value + + +async def test_setup_user_no_bluetooth( + hass: HomeAssistant, mock_bluetooth_adapters: None +) -> None: + """Test setting up via user interaction when bluetooth is not enabled.""" + result = await hass.config_entries.flow.async_init( + const.DOMAIN, + context={"source": config_entries.SOURCE_USER}, + ) + assert result["type"] == FlowResultType.ABORT + assert result["reason"] == "bluetooth_not_available" + + +async def test_invalid_irk(hass: HomeAssistant, enable_bluetooth: None) -> None: + """Test invalid irk.""" + result = await hass.config_entries.flow.async_init( + const.DOMAIN, context={"source": config_entries.SOURCE_USER} + ) + assert result["type"] == "form" + + result = await hass.config_entries.flow.async_configure( + result["flow_id"], user_input={"irk": "irk:000000"} + ) + assert_form_error(result, "irk", "irk_not_valid") + + +async def test_irk_not_found(hass: HomeAssistant, enable_bluetooth: None) -> None: + """Test irk not found.""" + result = await hass.config_entries.flow.async_init( + const.DOMAIN, context={"source": config_entries.SOURCE_USER} + ) + assert result["type"] == "form" + + result = await hass.config_entries.flow.async_configure( + result["flow_id"], + user_input={"irk": "irk:00000000000000000000000000000000"}, + ) + assert_form_error(result, "irk", "irk_not_found") + + +async def test_flow_works(hass: HomeAssistant, enable_bluetooth: None) -> None: + """Test config flow works.""" + + inject_bluetooth_service_info( + hass, + BluetoothServiceInfo( + name="Test Test Test", + address="40:01:02:0a:c4:a6", + rssi=-63, + service_data={}, + manufacturer_data={}, + service_uuids=[], + source="local", + ), + ) + + result = await hass.config_entries.flow.async_init( + const.DOMAIN, context={"source": config_entries.SOURCE_USER} + ) + assert result["type"] == "form" + + # Check you can finish the flow + with patch( + "homeassistant.components.private_ble_device.async_setup_entry", + return_value=True, + ): + result = await hass.config_entries.flow.async_configure( + result["flow_id"], + user_input={"irk": "irk:00000000000000000000000000000000"}, + ) + + assert result["type"] == FlowResultType.CREATE_ENTRY + assert result["title"] == "Test Test Test" + assert result["data"] == {"irk": "00000000000000000000000000000000"} + assert result["result"].unique_id == "00000000000000000000000000000000" + + +async def test_flow_works_by_base64( + hass: HomeAssistant, enable_bluetooth: None +) -> None: + """Test config flow works.""" + + inject_bluetooth_service_info( + hass, + BluetoothServiceInfo( + name="Test Test Test", + address="40:01:02:0a:c4:a6", + rssi=-63, + service_data={}, + manufacturer_data={}, + service_uuids=[], + source="local", + ), + ) + + result = await hass.config_entries.flow.async_init( + const.DOMAIN, context={"source": config_entries.SOURCE_USER} + ) + assert result["type"] == "form" + + # Check you can finish the flow + with patch( + "homeassistant.components.private_ble_device.async_setup_entry", + return_value=True, + ): + result = await hass.config_entries.flow.async_configure( + result["flow_id"], + user_input={"irk": "AAAAAAAAAAAAAAAAAAAAAA=="}, + ) + + assert result["type"] == FlowResultType.CREATE_ENTRY + assert result["title"] == "Test Test Test" + assert result["data"] == {"irk": "00000000000000000000000000000000"} + assert result["result"].unique_id == "00000000000000000000000000000000" diff --git a/tests/components/private_ble_device/test_device_tracker.py b/tests/components/private_ble_device/test_device_tracker.py new file mode 100644 index 00000000000000..776ba503983e8f --- /dev/null +++ b/tests/components/private_ble_device/test_device_tracker.py @@ -0,0 +1,183 @@ +"""Tests for polling measures.""" + + +import time + +from homeassistant.components.bluetooth.advertisement_tracker import ( + ADVERTISING_TIMES_NEEDED, +) +from homeassistant.core import HomeAssistant + +from . import ( + MAC_RPA_VALID_1, + MAC_RPA_VALID_2, + MAC_STATIC, + async_inject_broadcast, + async_mock_config_entry, + async_move_time_forwards, +) + +from tests.components.bluetooth.test_advertisement_tracker import ONE_HOUR_SECONDS + + +async def test_tracker_created(hass: HomeAssistant, enable_bluetooth: None) -> None: + """Test creating a tracker entity when no devices have been seen.""" + await async_mock_config_entry(hass) + + state = hass.states.get("device_tracker.private_ble_device_000000") + assert state + assert state.state == "not_home" + + +async def test_tracker_ignore_other_rpa( + hass: HomeAssistant, enable_bluetooth: None +) -> None: + """Test that tracker ignores RPA's that don't match us.""" + await async_mock_config_entry(hass) + await async_inject_broadcast(hass, MAC_STATIC) + + state = hass.states.get("device_tracker.private_ble_device_000000") + assert state + assert state.state == "not_home" + + +async def test_tracker_already_home( + hass: HomeAssistant, enable_bluetooth: None +) -> None: + """Test creating a tracker and the device was already discovered by HA.""" + await async_inject_broadcast(hass, MAC_RPA_VALID_1) + await async_mock_config_entry(hass) + + state = hass.states.get("device_tracker.private_ble_device_000000") + assert state + assert state.state == "home" + + +async def test_tracker_arrive_home(hass: HomeAssistant, enable_bluetooth: None) -> None: + """Test transition from not_home to home.""" + await async_mock_config_entry(hass) + await async_inject_broadcast(hass, MAC_RPA_VALID_1, b"1") + state = hass.states.get("device_tracker.private_ble_device_000000") + assert state + assert state.state == "home" + assert state.attributes["current_address"] == "40:01:02:0a:c4:a6" + assert state.attributes["source"] == "local" + + await async_inject_broadcast(hass, MAC_STATIC, b"1") + state = hass.states.get("device_tracker.private_ble_device_000000") + assert state + assert state.state == "home" + + # Test same wrong mac address again to exercise some caching + await async_inject_broadcast(hass, MAC_STATIC, b"2") + state = hass.states.get("device_tracker.private_ble_device_000000") + assert state + assert state.state == "home" + + # And test original mac address again. + # Use different mfr data so that event bubbles up + await async_inject_broadcast(hass, MAC_RPA_VALID_1, b"2") + state = hass.states.get("device_tracker.private_ble_device_000000") + assert state + assert state.state == "home" + assert state.attributes["current_address"] == "40:01:02:0a:c4:a6" + + +async def test_tracker_isolation(hass: HomeAssistant, enable_bluetooth: None) -> None: + """Test creating 2 tracker entities doesn't confuse anything.""" + await async_mock_config_entry(hass) + await async_mock_config_entry(hass, irk="1" * 32) + + # This broadcast should only impact the first entity + await async_inject_broadcast(hass, MAC_RPA_VALID_1, b"1") + + state = hass.states.get("device_tracker.private_ble_device_000000") + assert state + assert state.state == "home" + + state = hass.states.get("device_tracker.private_ble_device_111111") + assert state + assert state.state == "not_home" + + +async def test_tracker_mac_rotate(hass: HomeAssistant, enable_bluetooth: None) -> None: + """Test MAC address rotation.""" + await async_inject_broadcast(hass, MAC_RPA_VALID_1) + await async_mock_config_entry(hass) + + state = hass.states.get("device_tracker.private_ble_device_000000") + assert state + assert state.state == "home" + assert state.attributes["current_address"] == MAC_RPA_VALID_1 + + await async_inject_broadcast(hass, MAC_RPA_VALID_2) + state = hass.states.get("device_tracker.private_ble_device_000000") + assert state + assert state.state == "home" + assert state.attributes["current_address"] == MAC_RPA_VALID_2 + + +async def test_tracker_start_stale(hass: HomeAssistant, enable_bluetooth: None) -> None: + """Test edge case where we find an existing stale record, and it expires before we see any more.""" + time.monotonic() + + await async_inject_broadcast(hass, MAC_RPA_VALID_1) + await async_mock_config_entry(hass) + + state = hass.states.get("device_tracker.private_ble_device_000000") + assert state + assert state.state == "home" + + await async_move_time_forwards( + hass, ((ADVERTISING_TIMES_NEEDED - 1) * ONE_HOUR_SECONDS) + ) + state = hass.states.get("device_tracker.private_ble_device_000000") + assert state + assert state.state == "not_home" + + +async def test_tracker_leave_home(hass: HomeAssistant, enable_bluetooth: None) -> None: + """Test tracker notices we have left.""" + time.monotonic() + + await async_mock_config_entry(hass) + await async_inject_broadcast(hass, MAC_RPA_VALID_1) + + state = hass.states.get("device_tracker.private_ble_device_000000") + assert state + assert state.state == "home" + + await async_move_time_forwards( + hass, ((ADVERTISING_TIMES_NEEDED - 1) * ONE_HOUR_SECONDS) + ) + state = hass.states.get("device_tracker.private_ble_device_000000") + assert state + assert state.state == "not_home" + + +async def test_old_tracker_leave_home( + hass: HomeAssistant, enable_bluetooth: None +) -> None: + """Test tracker ignores an old stale mac address timing out.""" + start_time = time.monotonic() + + await async_mock_config_entry(hass) + + await async_inject_broadcast(hass, MAC_RPA_VALID_2, broadcast_time=start_time) + await async_inject_broadcast(hass, MAC_RPA_VALID_2, broadcast_time=start_time + 15) + + state = hass.states.get("device_tracker.private_ble_device_000000") + assert state + assert state.state == "home" + + # First address has timed out - still home + await async_move_time_forwards(hass, 910) + state = hass.states.get("device_tracker.private_ble_device_000000") + assert state + assert state.state == "home" + + # Second address has time out - now away + await async_move_time_forwards(hass, 920) + state = hass.states.get("device_tracker.private_ble_device_000000") + assert state + assert state.state == "not_home" From 9e9aa163f70ada036c59f5a1dbd9df269ff704d2 Mon Sep 17 00:00:00 2001 From: Joost Lekkerkerker Date: Sat, 2 Sep 2023 16:42:32 +0200 Subject: [PATCH 04/12] Use shorthand attributes in hlk_sw16 (#99383) --- homeassistant/components/hlk_sw16/__init__.py | 13 ++----------- 1 file changed, 2 insertions(+), 11 deletions(-) diff --git a/homeassistant/components/hlk_sw16/__init__.py b/homeassistant/components/hlk_sw16/__init__.py index f80972da6130fb..9be0b5203fd904 100644 --- a/homeassistant/components/hlk_sw16/__init__.py +++ b/homeassistant/components/hlk_sw16/__init__.py @@ -147,12 +147,8 @@ def __init__(self, device_port, entry_id, client): self._device_port = device_port self._is_on = None self._client = client - self._name = device_port - - @property - def unique_id(self): - """Return a unique ID.""" - return f"{self._entry_id}_{self._device_port}" + self._attr_name = device_port + self._attr_unique_id = f"{self._entry_id}_{self._device_port}" @callback def handle_event_callback(self, event): @@ -161,11 +157,6 @@ def handle_event_callback(self, event): self._is_on = event self.async_write_ha_state() - @property - def name(self): - """Return a name for the device.""" - return self._name - @property def available(self): """Return True if entity is available.""" From 7168e71860ff5c04aafbf5de240026b2963043ff Mon Sep 17 00:00:00 2001 From: Erik Montnemery Date: Sat, 2 Sep 2023 16:51:06 +0200 Subject: [PATCH 05/12] Log unexpected exceptions causing recorder shutdown (#99445) --- homeassistant/components/recorder/core.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/homeassistant/components/recorder/core.py b/homeassistant/components/recorder/core.py index ffdc3807039d3a..bbaff24ff778fb 100644 --- a/homeassistant/components/recorder/core.py +++ b/homeassistant/components/recorder/core.py @@ -692,6 +692,10 @@ def run(self) -> None: """Run the recorder thread.""" try: self._run() + except Exception: # pylint: disable=broad-exception-caught + _LOGGER.exception( + "Recorder._run threw unexpected exception, recorder shutting down" + ) finally: # Ensure shutdown happens cleanly if # anything goes wrong in the run loop From defd9e400179f568a053cf4c86471830e03206fc Mon Sep 17 00:00:00 2001 From: Erik Montnemery Date: Sat, 2 Sep 2023 17:09:46 +0200 Subject: [PATCH 06/12] Don't compile missing statistics when running tests (#99446) --- tests/conftest.py | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/tests/conftest.py b/tests/conftest.py index f90984e1c7bc8f..739dfa5d292937 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1276,6 +1276,11 @@ def hass_recorder( hass = get_test_home_assistant() nightly = recorder.Recorder.async_nightly_tasks if enable_nightly_purge else None stats = recorder.Recorder.async_periodic_statistics if enable_statistics else None + compile_missing = ( + recorder.Recorder._schedule_compile_missing_statistics + if enable_statistics + else None + ) schema_validate = ( migration._find_schema_errors if enable_schema_validation @@ -1327,6 +1332,10 @@ def hass_recorder( "homeassistant.components.recorder.Recorder._migrate_entity_ids", side_effect=migrate_entity_ids, autospec=True, + ), patch( + "homeassistant.components.recorder.Recorder._schedule_compile_missing_statistics", + side_effect=compile_missing, + autospec=True, ): def setup_recorder(config: dict[str, Any] | None = None) -> HomeAssistant: @@ -1399,6 +1408,11 @@ async def async_setup_recorder_instance( if enable_schema_validation else itertools.repeat(set()) ) + compile_missing = ( + recorder.Recorder._schedule_compile_missing_statistics + if enable_statistics + else None + ) migrate_states_context_ids = ( recorder.Recorder._migrate_states_context_ids if enable_migrate_context_ids @@ -1445,6 +1459,10 @@ async def async_setup_recorder_instance( "homeassistant.components.recorder.Recorder._migrate_entity_ids", side_effect=migrate_entity_ids, autospec=True, + ), patch( + "homeassistant.components.recorder.Recorder._schedule_compile_missing_statistics", + side_effect=compile_missing, + autospec=True, ): async def async_setup_recorder( From 6e743a5bb2b7a56cd68d0663943ea6e220e141d1 Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Sat, 2 Sep 2023 11:55:11 -0500 Subject: [PATCH 07/12] Switch mqtt to use async_call_later where possible (#99486) --- homeassistant/components/mqtt/binary_sensor.py | 18 +++++++++--------- homeassistant/components/mqtt/sensor.py | 18 +++++++++--------- 2 files changed, 18 insertions(+), 18 deletions(-) diff --git a/homeassistant/components/mqtt/binary_sensor.py b/homeassistant/components/mqtt/binary_sensor.py index 0d4b2c4a7b4572..b5c7bc987896f4 100644 --- a/homeassistant/components/mqtt/binary_sensor.py +++ b/homeassistant/components/mqtt/binary_sensor.py @@ -29,7 +29,7 @@ import homeassistant.helpers.config_validation as cv from homeassistant.helpers.entity_platform import AddEntitiesCallback import homeassistant.helpers.event as evt -from homeassistant.helpers.event import async_track_point_in_utc_time +from homeassistant.helpers.event import async_call_later from homeassistant.helpers.restore_state import RestoreEntity from homeassistant.helpers.typing import ConfigType, DiscoveryInfoType from homeassistant.util import dt as dt_util @@ -128,15 +128,17 @@ async def mqtt_async_added_to_hass(self) -> None: expiration_at: datetime = last_state.last_changed + timedelta( seconds=self._expire_after ) - if expiration_at < (time_now := dt_util.utcnow()): + remain_seconds = (expiration_at - dt_util.utcnow()).total_seconds() + + if remain_seconds <= 0: # Skip reactivating the binary_sensor _LOGGER.debug("Skip state recovery after reload for %s", self.entity_id) return self._expired = False self._attr_is_on = last_state.state == STATE_ON - self._expiration_trigger = async_track_point_in_utc_time( - self.hass, self._value_is_expired, expiration_at + self._expiration_trigger = async_call_later( + self.hass, remain_seconds, self._value_is_expired ) _LOGGER.debug( ( @@ -144,7 +146,7 @@ async def mqtt_async_added_to_hass(self) -> None: " expiring %s" ), self.entity_id, - expiration_at - time_now, + remain_seconds, ) async def async_will_remove_from_hass(self) -> None: @@ -202,10 +204,8 @@ def state_message_received(msg: ReceiveMessage) -> None: self._expiration_trigger() # Set new trigger - expiration_at = dt_util.utcnow() + timedelta(seconds=self._expire_after) - - self._expiration_trigger = async_track_point_in_utc_time( - self.hass, self._value_is_expired, expiration_at + self._expiration_trigger = async_call_later( + self.hass, self._expire_after, self._value_is_expired ) payload = self._value_template(msg.payload) diff --git a/homeassistant/components/mqtt/sensor.py b/homeassistant/components/mqtt/sensor.py index ae94b0df0ce68c..70c8d505b4f8f2 100644 --- a/homeassistant/components/mqtt/sensor.py +++ b/homeassistant/components/mqtt/sensor.py @@ -32,7 +32,7 @@ from homeassistant.core import CALLBACK_TYPE, HomeAssistant, State, callback import homeassistant.helpers.config_validation as cv from homeassistant.helpers.entity_platform import AddEntitiesCallback -from homeassistant.helpers.event import async_track_point_in_utc_time +from homeassistant.helpers.event import async_call_later from homeassistant.helpers.typing import ConfigType, DiscoveryInfoType from homeassistant.util import dt as dt_util @@ -162,15 +162,17 @@ async def mqtt_async_added_to_hass(self) -> None: and not self._expiration_trigger ): expiration_at = last_state.last_changed + timedelta(seconds=_expire_after) - if expiration_at < (time_now := dt_util.utcnow()): + remain_seconds = (expiration_at - dt_util.utcnow()).total_seconds() + + if remain_seconds <= 0: # Skip reactivating the sensor _LOGGER.debug("Skip state recovery after reload for %s", self.entity_id) return self._expired = False self._attr_native_value = last_sensor_data.native_value - self._expiration_trigger = async_track_point_in_utc_time( - self.hass, self._value_is_expired, expiration_at + self._expiration_trigger = async_call_later( + self.hass, remain_seconds, self._value_is_expired ) _LOGGER.debug( ( @@ -178,7 +180,7 @@ async def mqtt_async_added_to_hass(self) -> None: " expiring %s" ), self.entity_id, - expiration_at - time_now, + remain_seconds, ) async def async_will_remove_from_hass(self) -> None: @@ -235,10 +237,8 @@ def _update_state(msg: ReceiveMessage) -> None: self._expiration_trigger() # Set new trigger - expiration_at = dt_util.utcnow() + timedelta(seconds=self._expire_after) - - self._expiration_trigger = async_track_point_in_utc_time( - self.hass, self._value_is_expired, expiration_at + self._expiration_trigger = async_call_later( + self.hass, self._expire_after, self._value_is_expired ) payload = self._template(msg.payload, PayloadSentinel.DEFAULT) From acd9cfa929646a6a1fcad807e3e4b5c6c02565d4 Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Sat, 2 Sep 2023 12:08:07 -0500 Subject: [PATCH 08/12] Speed up calls to the all states api (#99462) --- homeassistant/components/api/__init__.py | 21 +++++++++----- tests/components/api/test_init.py | 37 ++++++++++++++++++++++-- 2 files changed, 47 insertions(+), 11 deletions(-) diff --git a/homeassistant/components/api/__init__.py b/homeassistant/components/api/__init__.py index 7b13833ccaba1e..b427341546e5d1 100644 --- a/homeassistant/components/api/__init__.py +++ b/homeassistant/components/api/__init__.py @@ -9,6 +9,7 @@ from aiohttp.web_exceptions import HTTPBadRequest import voluptuous as vol +from homeassistant.auth.models import User from homeassistant.auth.permissions.const import POLICY_READ from homeassistant.bootstrap import DATA_LOGGING from homeassistant.components.http import HomeAssistantView, require_admin @@ -189,16 +190,20 @@ class APIStatesView(HomeAssistantView): name = "api:states" @ha.callback - def get(self, request): + def get(self, request: web.Request) -> web.Response: """Get current states.""" - user = request["hass_user"] + user: User = request["hass_user"] + hass: HomeAssistant = request.app["hass"] + if user.is_admin: + return self.json([state.as_dict() for state in hass.states.async_all()]) entity_perm = user.permissions.check_entity - states = [ - state - for state in request.app["hass"].states.async_all() - if entity_perm(state.entity_id, "read") - ] - return self.json(states) + return self.json( + [ + state.as_dict() + for state in hass.states.async_all() + if entity_perm(state.entity_id, "read") + ] + ) class APIEntityStateView(HomeAssistantView): diff --git a/tests/components/api/test_init.py b/tests/components/api/test_init.py index 116529b02a4515..f61988eff5a270 100644 --- a/tests/components/api/test_init.py +++ b/tests/components/api/test_init.py @@ -9,6 +9,7 @@ import voluptuous as vol from homeassistant import const +from homeassistant.auth.models import Credentials from homeassistant.auth.providers.legacy_api_password import ( LegacyApiPasswordAuthProvider, ) @@ -17,7 +18,7 @@ from homeassistant.core import HomeAssistant from homeassistant.setup import async_setup_component -from tests.common import MockUser, async_mock_service +from tests.common import CLIENT_ID, MockUser, async_mock_service from tests.typing import ClientSessionGenerator @@ -569,11 +570,41 @@ async def test_event_stream_requires_admin( assert resp.status == HTTPStatus.UNAUTHORIZED -async def test_states_view_filters( +async def test_states( hass: HomeAssistant, mock_api_client: TestClient, hass_admin_user: MockUser +) -> None: + """Test fetching all states as admin.""" + hass.states.async_set("test.entity", "hello") + resp = await mock_api_client.get(const.URL_API_STATES) + assert resp.status == HTTPStatus.OK + json = await resp.json() + assert len(json) == 1 + assert json[0]["entity_id"] == "test.entity" + + +async def test_states_view_filters( + hass: HomeAssistant, + hass_read_only_user: MockUser, + hass_client: ClientSessionGenerator, ) -> None: """Test filtering only visible states.""" - hass_admin_user.mock_policy({"entities": {"entity_ids": {"test.entity": True}}}) + assert not hass_read_only_user.is_admin + hass_read_only_user.mock_policy({"entities": {"entity_ids": {"test.entity": True}}}) + await async_setup_component(hass, "api", {}) + read_only_user_credential = Credentials( + id="mock-read-only-credential-id", + auth_provider_type="homeassistant", + auth_provider_id=None, + data={"username": "readonly"}, + is_new=False, + ) + await hass.auth.async_link_user(hass_read_only_user, read_only_user_credential) + + refresh_token = await hass.auth.async_create_refresh_token( + hass_read_only_user, CLIENT_ID, credential=read_only_user_credential + ) + token = hass.auth.async_create_access_token(refresh_token) + mock_api_client = await hass_client(token) hass.states.async_set("test.entity", "hello") hass.states.async_set("test.not_visible_entity", "invisible") resp = await mock_api_client.get(const.URL_API_STATES) From 6974d211e55cbcd042c3216c9c0c38801e41f01e Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Sat, 2 Sep 2023 12:14:33 -0500 Subject: [PATCH 09/12] Switch isy994 to use async_call_later (#99487) async_track_point_in_utc_time is not needed here since we only need to call at timedelta(hours=25) --- homeassistant/components/isy994/binary_sensor.py | 14 +++----------- 1 file changed, 3 insertions(+), 11 deletions(-) diff --git a/homeassistant/components/isy994/binary_sensor.py b/homeassistant/components/isy994/binary_sensor.py index aa7c3d551473bf..27f1887bd92a47 100644 --- a/homeassistant/components/isy994/binary_sensor.py +++ b/homeassistant/components/isy994/binary_sensor.py @@ -23,9 +23,8 @@ from homeassistant.core import CALLBACK_TYPE, HomeAssistant, callback from homeassistant.helpers.device_registry import DeviceInfo from homeassistant.helpers.entity_platform import AddEntitiesCallback -from homeassistant.helpers.event import async_track_point_in_utc_time +from homeassistant.helpers.event import async_call_later from homeassistant.helpers.restore_state import RestoreEntity -from homeassistant.util import dt as dt_util from .const import ( _LOGGER, @@ -496,15 +495,8 @@ def timer_elapsed(now: datetime) -> None: self._heartbeat_timer = None self.async_write_ha_state() - point_in_time = dt_util.utcnow() + timedelta(hours=25) - _LOGGER.debug( - "Heartbeat timer starting. Now: %s Then: %s", - dt_util.utcnow(), - point_in_time, - ) - - self._heartbeat_timer = async_track_point_in_utc_time( - self.hass, timer_elapsed, point_in_time + self._heartbeat_timer = async_call_later( + self.hass, timedelta(hours=25), timer_elapsed ) @callback From c3841f8734b24aa68045d4172b04149c34ddb98e Mon Sep 17 00:00:00 2001 From: Jan Bouwhuis Date: Sat, 2 Sep 2023 19:26:11 +0200 Subject: [PATCH 10/12] Rework on mqtt certificate tests (#99503) * Shared fixture on TEMP_DIR_NAME mock in MQTT tests * Improve mqtt certificate file tests * Update tests/components/mqtt/test_util.py Co-authored-by: J. Nick Koston * Update tests/components/mqtt/conftest.py Co-authored-by: J. Nick Koston * Avoid blocking code * typo in sub function --------- Co-authored-by: J. Nick Koston --- tests/components/mqtt/conftest.py | 21 +++++++ tests/components/mqtt/test_config_flow.py | 11 ++-- tests/components/mqtt/test_util.py | 74 ++++++++++++++++------- 3 files changed, 77 insertions(+), 29 deletions(-) diff --git a/tests/components/mqtt/conftest.py b/tests/components/mqtt/conftest.py index ebe86c1f1dfb7d..91ece381f6d463 100644 --- a/tests/components/mqtt/conftest.py +++ b/tests/components/mqtt/conftest.py @@ -1,5 +1,9 @@ """Test fixtures for mqtt component.""" +from collections.abc import Generator +from random import getrandbits +from unittest.mock import patch + import pytest from tests.components.light.conftest import mock_light_profiles # noqa: F401 @@ -8,3 +12,20 @@ @pytest.fixture(autouse=True) def patch_hass_config(mock_hass_config: None) -> None: """Patch configuration.yaml.""" + + +@pytest.fixture +def temp_dir_prefix() -> str: + """Set an alternate temp dir prefix.""" + return "test" + + +@pytest.fixture +def mock_temp_dir(temp_dir_prefix: str) -> Generator[None, None, str]: + """Mock the certificate temp directory.""" + with patch( + # Patch temp dir name to avoid tests fail running in parallel + "homeassistant.components.mqtt.util.TEMP_DIR_NAME", + f"home-assistant-mqtt-{temp_dir_prefix}-{getrandbits(10):03x}", + ) as mocked_temp_dir: + yield mocked_temp_dir diff --git a/tests/components/mqtt/test_config_flow.py b/tests/components/mqtt/test_config_flow.py index f0681a537da557..c2a7e0065ce04f 100644 --- a/tests/components/mqtt/test_config_flow.py +++ b/tests/components/mqtt/test_config_flow.py @@ -2,7 +2,6 @@ from collections.abc import Generator, Iterator from contextlib import contextmanager from pathlib import Path -from random import getrandbits from ssl import SSLError from typing import Any from unittest.mock import AsyncMock, MagicMock, patch @@ -131,7 +130,9 @@ def mock_try_connection_time_out() -> Generator[MagicMock, None, None]: @pytest.fixture -def mock_process_uploaded_file(tmp_path: Path) -> Generator[MagicMock, None, None]: +def mock_process_uploaded_file( + tmp_path: Path, mock_temp_dir: str +) -> Generator[MagicMock, None, None]: """Mock upload certificate files.""" file_id_ca = str(uuid4()) file_id_cert = str(uuid4()) @@ -159,11 +160,7 @@ def _mock_process_uploaded_file( with patch( "homeassistant.components.mqtt.config_flow.process_uploaded_file", side_effect=_mock_process_uploaded_file, - ) as mock_upload, patch( - # Patch temp dir name to avoid tests fail running in parallel - "homeassistant.components.mqtt.util.TEMP_DIR_NAME", - "home-assistant-mqtt" + f"-{getrandbits(10):03x}", - ): + ) as mock_upload: mock_upload.file_id = { mqtt.CONF_CERTIFICATE: file_id_ca, mqtt.CONF_CLIENT_CERT: file_id_cert, diff --git a/tests/components/mqtt/test_util.py b/tests/components/mqtt/test_util.py index e93a5e376bbb96..941072bc224e18 100644 --- a/tests/components/mqtt/test_util.py +++ b/tests/components/mqtt/test_util.py @@ -1,7 +1,9 @@ """Test MQTT utils.""" from collections.abc import Callable +from pathlib import Path from random import getrandbits +import tempfile from unittest.mock import patch import pytest @@ -14,17 +16,6 @@ from tests.typing import MqttMockHAClient, MqttMockPahoClient -@pytest.fixture(autouse=True) -def mock_temp_dir(): - """Mock the certificate temp directory.""" - with patch( - # Patch temp dir name to avoid tests fail running in parallel - "homeassistant.components.mqtt.util.TEMP_DIR_NAME", - "home-assistant-mqtt" + f"-{getrandbits(10):03x}", - ) as mocked_temp_dir: - yield mocked_temp_dir - - @pytest.mark.parametrize( ("option", "content", "file_created"), [ @@ -34,31 +25,50 @@ def mock_temp_dir(): (mqtt.CONF_CLIENT_KEY, "### PRIVATE KEY ###", True), ], ) +@pytest.mark.parametrize("temp_dir_prefix", ["create-test"]) async def test_async_create_certificate_temp_files( - hass: HomeAssistant, mock_temp_dir, option, content, file_created + hass: HomeAssistant, + mock_temp_dir: str, + option: str, + content: str, + file_created: bool, ) -> None: """Test creating and reading and recovery certificate files.""" config = {option: content} - await mqtt.util.async_create_certificate_temp_files(hass, config) - file_path = mqtt.util.get_file_path(option) + temp_dir = Path(tempfile.gettempdir()) / mock_temp_dir + + # Create old file to be able to assert it is removed with auto option + def _ensure_old_file_exists() -> None: + if not temp_dir.exists(): + temp_dir.mkdir(0o700) + temp_file = temp_dir / option + with open(temp_file, "wb") as old_file: + old_file.write(b"old content") + old_file.close() + + await hass.async_add_executor_job(_ensure_old_file_exists) + await mqtt.util.async_create_certificate_temp_files(hass, config) + file_path = await hass.async_add_executor_job(mqtt.util.get_file_path, option) assert bool(file_path) is file_created assert ( - mqtt.util.migrate_certificate_file_to_content(file_path or content) == content + await hass.async_add_executor_job( + mqtt.util.migrate_certificate_file_to_content, file_path or content + ) + == content ) # Make sure certificate temp files are recovered - if file_path: - # Overwrite content of file (except for auto option) - file = open(file_path, "wb") - file.write(b"invalid") - file.close() + await hass.async_add_executor_job(_ensure_old_file_exists) await mqtt.util.async_create_certificate_temp_files(hass, config) - file_path2 = mqtt.util.get_file_path(option) + file_path2 = await hass.async_add_executor_job(mqtt.util.get_file_path, option) assert bool(file_path2) is file_created assert ( - mqtt.util.migrate_certificate_file_to_content(file_path2 or content) == content + await hass.async_add_executor_job( + mqtt.util.migrate_certificate_file_to_content, file_path2 or content + ) + == content ) assert file_path == file_path2 @@ -71,6 +81,26 @@ async def test_reading_non_exitisting_certificate_file() -> None: ) +@pytest.mark.parametrize("temp_dir_prefix", "unknown") +async def test_return_default_get_file_path( + hass: HomeAssistant, mock_temp_dir: str +) -> None: + """Test get_file_path returns default.""" + + def _get_file_path(file_path: Path) -> bool: + return ( + not file_path.exists() + and mqtt.util.get_file_path("some_option", "mydefault") == "mydefault" + ) + + with patch( + "homeassistant.components.mqtt.util.TEMP_DIR_NAME", + f"home-assistant-mqtt-other-{getrandbits(10):03x}", + ) as mock_temp_dir: + tempdir = Path(tempfile.gettempdir()) / mock_temp_dir + assert await hass.async_add_executor_job(_get_file_path, tempdir) + + @patch("homeassistant.components.mqtt.PLATFORMS", []) async def test_waiting_for_client_not_loaded( hass: HomeAssistant, From 1048f47a915c3b89bdcac5dca54320b7535e1431 Mon Sep 17 00:00:00 2001 From: Allen Porter Date: Sat, 2 Sep 2023 10:38:41 -0700 Subject: [PATCH 11/12] Code cleanup for nest device info (#99511) --- homeassistant/components/nest/device_info.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/homeassistant/components/nest/device_info.py b/homeassistant/components/nest/device_info.py index 35e32ccf1bc98f..f269e3e89d648d 100644 --- a/homeassistant/components/nest/device_info.py +++ b/homeassistant/components/nest/device_info.py @@ -66,10 +66,7 @@ def device_name(self) -> str | None: @property def device_model(self) -> str | None: """Return device model information.""" - # The API intentionally returns minimal information about specific - # devices, instead relying on traits, but we can infer a generic model - # name based on the type - return DEVICE_TYPE_MAP.get(self._device.type or "", None) + return DEVICE_TYPE_MAP.get(self._device.type) if self._device.type else None @property def suggested_area(self) -> str | None: From 1ab2e900f9a61f35be4dcaa2551140d7a5cd79b3 Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Sat, 2 Sep 2023 12:43:27 -0500 Subject: [PATCH 12/12] Improve lingering timer checks (#99472) --- homeassistant/core.py | 4 ++++ tests/conftest.py | 35 +++++++++++++++++++++++++++-------- 2 files changed, 31 insertions(+), 8 deletions(-) diff --git a/homeassistant/core.py b/homeassistant/core.py index 18c5c355ae979f..89269ae9158cf9 100644 --- a/homeassistant/core.py +++ b/homeassistant/core.py @@ -294,6 +294,10 @@ def __new__(cls, config_dir: str) -> HomeAssistant: _hass.hass = hass return hass + def __repr__(self) -> str: + """Return the representation.""" + return f"" + def __init__(self, config_dir: str) -> None: """Initialize new Home Assistant object.""" self.loop = asyncio.get_running_loop() diff --git a/tests/conftest.py b/tests/conftest.py index 739dfa5d292937..99db088449670c 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -3,12 +3,13 @@ import asyncio from collections.abc import AsyncGenerator, Callable, Coroutine, Generator -from contextlib import asynccontextmanager +from contextlib import asynccontextmanager, contextmanager import functools import gc import itertools import logging import os +import reprlib import sqlite3 import ssl import threading @@ -302,6 +303,21 @@ def skip_stop_scripts( yield +@contextmanager +def long_repr_strings() -> Generator[None, None, None]: + """Increase reprlib maxstring and maxother to 300.""" + arepr = reprlib.aRepr + original_maxstring = arepr.maxstring + original_maxother = arepr.maxother + arepr.maxstring = 300 + arepr.maxother = 300 + try: + yield + finally: + arepr.maxstring = original_maxstring + arepr.maxother = original_maxother + + @pytest.fixture(autouse=True) def verify_cleanup( event_loop: asyncio.AbstractEventLoop, @@ -335,13 +351,16 @@ def verify_cleanup( for handle in event_loop._scheduled: # type: ignore[attr-defined] if not handle.cancelled(): - if expected_lingering_timers: - _LOGGER.warning("Lingering timer after test %r", handle) - elif handle._args and isinstance(job := handle._args[0], HassJob): - pytest.fail(f"Lingering timer after job {repr(job)}") - else: - pytest.fail(f"Lingering timer after test {repr(handle)}") - handle.cancel() + with long_repr_strings(): + if expected_lingering_timers: + _LOGGER.warning("Lingering timer after test %r", handle) + elif handle._args and isinstance(job := handle._args[-1], HassJob): + if job.cancel_on_shutdown: + continue + pytest.fail(f"Lingering timer after job {repr(job)}") + else: + pytest.fail(f"Lingering timer after test {repr(handle)}") + handle.cancel() # Verify no threads where left behind. threads = frozenset(threading.enumerate()) - threads_before