From 67bd86af118007d1f9a127dd7205b3d32cd275dd Mon Sep 17 00:00:00 2001 From: Sergey Kuznetsov Date: Sat, 5 Sep 2020 17:46:39 +0300 Subject: [PATCH 001/103] Change version to 1.2.0 --- info/info.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/info/info.json b/info/info.json index 4823fa9..a020d51 100644 --- a/info/info.json +++ b/info/info.json @@ -1,6 +1,6 @@ { "name": "Yandex.Disk Bot", - "version": "1.1.0", + "version": "1.2.0", "description": "This bot integrates Yandex.Disk into Telegram.", "about": "Work with Yandex.Disk.", "telegram": "Ya_Disk_Bot", From d407c9546d8bb1a1767a5de2670a40977336160d Mon Sep 17 00:00:00 2001 From: Sergey Kuznetsov Date: Sat, 5 Sep 2020 18:26:39 +0300 Subject: [PATCH 002/103] Edit venv paths in .vscode --- .vscode/settings.json | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.vscode/settings.json b/.vscode/settings.json index 51dbe5c..cde25bb 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -1,9 +1,9 @@ { - "python.pythonPath": "./venv/bin/python", + "python.pythonPath": "venv/bin/python", "python.autoComplete.extraPaths": [ "./src/*" ], - "cornflakes.linter.executablePath": "./venv/bin/flake8", + "cornflakes.linter.executablePath": "venv/bin/flake8", "files.exclude": { "venv": true, "**/__pycache__": true, From fb1e2e0f0cfd0c68d2ccfeb4be5e8f9ed2d31973 Mon Sep 17 00:00:00 2001 From: Sergey Kuznetsov Date: Sat, 5 Sep 2020 18:26:56 +0300 Subject: [PATCH 003/103] Change python runtime version to 3.8.2 --- CHANGELOG.md | 11 +++++++++++ runtime.txt | 2 +- 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index de1eb89..5a2e1a3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,14 @@ +# 1.2.0 + +## Telegram Bot + +## Project + +## Improved + +- Upgrade `python` to 3.8.2. + + # 1.1.0 (May 9, 2020) ## Telegram Bot diff --git a/runtime.txt b/runtime.txt index 73b1cf8..724c203 100644 --- a/runtime.txt +++ b/runtime.txt @@ -1 +1 @@ -python-3.8.0 +python-3.8.2 From f64193f1e4af75c3f1a73eb7fac35863f9d0598f Mon Sep 17 00:00:00 2001 From: Sergey Kuznetsov Date: Sat, 5 Sep 2020 19:36:00 +0300 Subject: [PATCH 004/103] Improve documentation for redirects --- src/app.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/app.py b/src/app.py index c18d839..c9ad18a 100644 --- a/src/app.py +++ b/src/app.py @@ -72,6 +72,12 @@ def configure_blueprints(app: Flask) -> None: def configure_redirects(app: Flask) -> None: """ Configures redirects. + + Note: all redirects to static content should be handled by + HTTP Reverse Proxy Server, not by WSGI HTTP Server. + We are keeping this redirect to static favicon only for + development builds where usually only Flask itself is used + (we want to see favicon at development stage - it is only the reason). """ @app.route("/favicon.ico") def favicon(): From cf88fc625a97cc4f3bc5ffb6bc4f6f92517a75e7 Mon Sep 17 00:00:00 2001 From: Sergey Kuznetsov Date: Sat, 5 Sep 2020 19:36:57 +0300 Subject: [PATCH 005/103] Redirect to favicon will be handled by nginx --- CHANGELOG.md | 4 ++++ src/configs/nginx.conf | 4 ++++ 2 files changed, 8 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 5a2e1a3..4950ab7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,10 @@ - Upgrade `python` to 3.8.2. +## Changed + +- Redirect to favicon will be handled by nginx. + # 1.1.0 (May 9, 2020) diff --git a/src/configs/nginx.conf b/src/configs/nginx.conf index 2e64c37..ebff794 100644 --- a/src/configs/nginx.conf +++ b/src/configs/nginx.conf @@ -33,6 +33,10 @@ http { alias src/static/robots/robots.txt; } + location /favicon.ico { + alias src/static/favicons/favicon.ico; + } + location /static/ { root src; autoindex off; From 5aa164e7d5099c7af4693ca96cc5aef5ffc56b86 Mon Sep 17 00:00:00 2001 From: Sergey Kuznetsov Date: Sun, 6 Sep 2020 14:54:17 +0300 Subject: [PATCH 006/103] Add reason for command abort --- .../webhook/commands/common/responses.py | 28 +++- .../webhook/commands/create_folder.py | 8 +- .../telegram_bot/webhook/commands/upload.py | 143 ++++++++++++------ 3 files changed, 127 insertions(+), 52 deletions(-) diff --git a/src/blueprints/telegram_bot/webhook/commands/common/responses.py b/src/blueprints/telegram_bot/webhook/commands/common/responses.py index 7dffb69..8cfc595 100644 --- a/src/blueprints/telegram_bot/webhook/commands/common/responses.py +++ b/src/blueprints/telegram_bot/webhook/commands/common/responses.py @@ -1,8 +1,20 @@ +from enum import IntEnum, unique + from src.api import telegram +@unique +class AbortReason(IntEnum): + """ + Reason for `abort_command`. + """ + UNKNOWN = 1 + NO_SUITABLE_DATA = 2 + + def abort_command( chat_telegram_id: int, + reason: AbortReason, edit_message: int = None, reply_to_message: int = None ) -> None: @@ -15,11 +27,17 @@ def abort_command( - if `reply_to_message` Telegram ID specified, then that message will be used for reply message. """ - text = ( - "I can't handle this because " - "you didn't send any suitable data " - "for that command." - ) + texts = { + AbortReason.UNKNOWN: ( + "I can't handle this because something is wrong." + ), + AbortReason.NO_SUITABLE_DATA: ( + "I can't handle this because " + "you didn't send any suitable data " + "for that command." + ) + } + text = texts[reason] if (edit_message is not None): telegram.edit_message_text( diff --git a/src/blueprints/telegram_bot/webhook/commands/create_folder.py b/src/blueprints/telegram_bot/webhook/commands/create_folder.py index 69bf7cd..a980f2f 100644 --- a/src/blueprints/telegram_bot/webhook/commands/create_folder.py +++ b/src/blueprints/telegram_bot/webhook/commands/create_folder.py @@ -3,7 +3,8 @@ from src.api import telegram from .common.responses import ( cancel_command, - abort_command + abort_command, + AbortReason ) from .common.decorators import ( yd_access_token_required, @@ -33,7 +34,10 @@ def handle(): ).strip() if not (folder_name): - return abort_command(chat.telegram_id) + return abort_command( + chat.telegram_id, + AbortReason.NO_SUITABLE_DATA + ) access_token = user.yandex_disk_token.get_access_token() last_status_code = None diff --git a/src/blueprints/telegram_bot/webhook/commands/upload.py b/src/blueprints/telegram_bot/webhook/commands/upload.py index bc151c0..d320e05 100644 --- a/src/blueprints/telegram_bot/webhook/commands/upload.py +++ b/src/blueprints/telegram_bot/webhook/commands/upload.py @@ -11,7 +11,8 @@ ) from .common.responses import ( abort_command, - cancel_command + cancel_command, + AbortReason ) from .common.yandex_api import ( upload_file_with_url, @@ -22,6 +23,23 @@ ) +class MessageHealth: + """ + Health status of Telegram message. + """ + def __init__( + self, + ok: bool, + abort_reason: Union[AbortReason, None] = None + ) -> None: + """ + :param ok: Message is valid for subsequent handling. + :param abort_reason: Reason of abort. `None` if `ok = True`. + """ + self.ok = ok + self.abort_reason = None + + class AttachmentHandler(metaclass=ABCMeta): """ Handles uploading of attachment of Telegram message. @@ -53,14 +71,16 @@ def telegram_action(self) -> str: pass @abstractmethod - def message_is_valid( + def check_message_health( self, message: telegram_interface.Message - ) -> bool: + ) -> MessageHealth: """ :param message: Incoming Telegram message. - :returns: Message is valid and should be handled. + :returns: See `MessageHealth` documentation. + Message will be handled by next operators only + in case of `ok = true`. """ pass @@ -106,14 +126,21 @@ def upload(self) -> None: message = g.telegram_message user = g.db_user chat = g.db_chat + message_health = self.check_message_health(message) - if not (self.message_is_valid(message)): - return abort_command(chat.telegram_id) + if not (message_health.ok): + return abort_command( + chat.telegram_id, + message_health.abort_reason or AbortReason.UNKNOWN + ) attachment = self.get_attachment(message) if (attachment is None): - return abort_command(chat.telegram_id) + return abort_command( + chat.telegram_id, + AbortReason.NO_SUITABLE_DATA + ) try: telegram.send_chat_action( @@ -237,16 +264,21 @@ def handle(): def telegram_action(self): return "upload_photo" - def message_is_valid(self, message: telegram_interface.Message): + def check_message_health(self, message: telegram_interface.Message): + health = MessageHealth(True) raw_data = message.raw_data - return ( + if not ( isinstance( raw_data.get("photo"), list ) and len(raw_data["photo"]) > 0 - ) + ): + health.ok = False + health.abort_reason = AbortReason.NO_SUITABLE_DATA + + return health def get_attachment(self, message: telegram_interface.Message): raw_data = message.raw_data @@ -276,16 +308,20 @@ def handle(): def telegram_action(self): return "upload_document" - def message_is_valid(self, message: telegram_interface.Message): - return ( - isinstance( - message.raw_data.get("document"), - dict - ) - ) + def check_message_health(self, message: telegram_interface.Message): + health = MessageHealth(True) + value = self.get_attachment(message) + + if not ( + isinstance(value, dict) + ): + health.ok = False + health.abort_reason = AbortReason.NO_SUITABLE_DATA + + return health def get_attachment(self, message: telegram_interface.Message): - return message.raw_data["document"] + return message.raw_data.get("document") def create_file_name(self, attachment, file): return ( @@ -307,16 +343,20 @@ def handle(): def telegram_action(self): return "upload_audio" - def message_is_valid(self, message: telegram_interface.Message): - return ( - isinstance( - message.raw_data.get("audio"), - dict - ) - ) + def check_message_health(self, message: telegram_interface.Message): + health = MessageHealth(True) + value = self.get_attachment(message) + + if not ( + isinstance(value, dict) + ): + health.ok = False + health.abort_reason = AbortReason.NO_SUITABLE_DATA + + return health def get_attachment(self, message: telegram_interface.Message): - return message.raw_data["audio"] + return message.raw_data.get("audio") def create_file_name(self, attachment, file): name = file["file_unique_id"] @@ -348,16 +388,20 @@ def handle(): def telegram_action(self): return "upload_video" - def message_is_valid(self, message: telegram_interface.Message): - return ( - isinstance( - message.raw_data.get("video"), - dict - ) - ) + def check_message_health(self, message: telegram_interface.Message): + health = MessageHealth(True) + value = self.get_attachment(message) + + if not ( + isinstance(value, dict) + ): + health.ok = False + health.abort_reason = AbortReason.NO_SUITABLE_DATA + + return health def get_attachment(self, message: telegram_interface.Message): - return message.raw_data["video"] + return message.raw_data.get("video") def create_file_name(self, attachment, file): name = file["file_unique_id"] @@ -383,16 +427,20 @@ def handle(): def telegram_action(self): return "upload_audio" - def message_is_valid(self, message: telegram_interface.Message): - return ( - isinstance( - message.raw_data.get("voice"), - dict - ) - ) + def check_message_health(self, message: telegram_interface.Message): + health = MessageHealth(True) + value = self.get_attachment(message) + + if not ( + isinstance(value, dict) + ): + health.ok = False + health.abort_reason = AbortReason.NO_SUITABLE_DATA + + return health def get_attachment(self, message: telegram_interface.Message): - return message.raw_data["voice"] + return message.raw_data.get("voice") def create_file_name(self, attachment, file): name = file["file_unique_id"] @@ -418,13 +466,18 @@ def handle(): def telegram_action(self): return "upload_document" - def message_is_valid(self, message: telegram_interface.Message): + def check_message_health(self, message: telegram_interface.Message): + health = MessageHealth(True) value = self.get_attachment(message) - return ( + if not ( isinstance(value, str) and len(value) > 0 - ) + ): + health.ok = False + health.abort_reason = AbortReason.NO_SUITABLE_DATA + + return health def get_attachment(self, message: telegram_interface.Message): return message.get_entity_value("url") From 5d2c5dfae5a675941ebf69ffdd313d3a6639b5bf Mon Sep 17 00:00:00 2001 From: Sergey Kuznetsov Date: Sun, 6 Sep 2020 14:55:16 +0300 Subject: [PATCH 007/103] Refactoring of CHANGELOG --- CHANGELOG.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 4950ab7..1a28523 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,11 +4,11 @@ ## Project -## Improved +### Improved - Upgrade `python` to 3.8.2. -## Changed +### Changed - Redirect to favicon will be handled by nginx. From f4286689b4e436b88e94789d940b393616c784eb Mon Sep 17 00:00:00 2001 From: Sergey Kuznetsov Date: Sun, 6 Sep 2020 16:31:38 +0300 Subject: [PATCH 008/103] Refactoring of commands/upload.py --- .../telegram_bot/webhook/commands/upload.py | 242 ++++++++---------- 1 file changed, 113 insertions(+), 129 deletions(-) diff --git a/src/blueprints/telegram_bot/webhook/commands/upload.py b/src/blueprints/telegram_bot/webhook/commands/upload.py index d320e05..5c89302 100644 --- a/src/blueprints/telegram_bot/webhook/commands/upload.py +++ b/src/blueprints/telegram_bot/webhook/commands/upload.py @@ -43,6 +43,11 @@ def __init__( class AttachmentHandler(metaclass=ABCMeta): """ Handles uploading of attachment of Telegram message. + + - most of attachments will be treated as files. + - some of the not abstract class functions are common + for most attachments. If you need specific logic in some + function, then override this function. """ def __init__(self) -> None: # Sended message to Telegram user. @@ -70,21 +75,25 @@ def telegram_action(self) -> str: """ pass + @property @abstractmethod - def check_message_health( - self, - message: telegram_interface.Message - ) -> MessageHealth: + def raw_data_key(self) -> str: """ - :param message: Incoming Telegram message. - - :returns: See `MessageHealth` documentation. - Message will be handled by next operators only - in case of `ok = true`. + :returns: Key in message, under this key + stored needed raw data. Example: 'audio'. + See https://core.telegram.org/bots/api#message """ pass + @property @abstractmethod + def raw_data_type(self) -> type: + """ + :returns: Expected type of raw data. + Example: 'dict'. `None` never should be returned! + """ + pass + def get_attachment( self, message: telegram_interface.Message @@ -93,15 +102,34 @@ def get_attachment( :param message: Incoming Telegram message. :returns: Attachment of message (photo object, - file object, audio object, etc.). If `None`, - uploading will be aborted. If `dict`, it must have `file_id` - and `file_unique_id` properties. If `str`, it is assumed - as direct file URL. See - https://core.telegram.org/bots/api/#available-types + file object, audio object, etc.). If `None`, then + uploading should be aborted. If `dict`, it will + have `file_id` and `file_unique_id` properties. + If `str`, it should be assumed as direct file URL. + See https://core.telegram.org/bots/api/#available-types """ - pass + return message.raw_data.get(self.raw_data_key) + + def check_message_health( + self, + message: telegram_interface.Message + ) -> MessageHealth: + """ + :param message: Incoming Telegram message. + + :returns: See `MessageHealth` documentation. + Message should be handled by next operators only + in case of `ok = true`. + """ + health = MessageHealth(True) + value = self.get_attachment(message) + + if not (isinstance(value, self.raw_data_type)): + health.ok = False + health.abort_reason = AbortReason.NO_SUITABLE_DATA + + return health - @abstractmethod def create_file_name( self, attachment: Union[dict, str], @@ -115,7 +143,34 @@ def create_file_name( :returns: Name of file which will be uploaded. """ - pass + name = ( + attachment.get("file_name") or + file["file_unique_id"] + ) + extension = "" + + if (isinstance(attachment, dict)): + extension = self.get_mime_type(attachment) + + if (extension): + name = f"{name}.{extension}" + + return name + + def get_mime_type(self, attachment: dict) -> str: + """ + :param attachment: `dict` result from `self.get_attachment()`. + + :returns: Empty string in case if `attachment` doesn't + have required key. Otherwise mime type of this attachment. + """ + result = "" + + if ("mime_type" in attachment): + types = attachment["mime_type"].split("/") + result = types[1] + + return result @yd_access_token_required @get_db_data @@ -264,25 +319,16 @@ def handle(): def telegram_action(self): return "upload_photo" - def check_message_health(self, message: telegram_interface.Message): - health = MessageHealth(True) - raw_data = message.raw_data - - if not ( - isinstance( - raw_data.get("photo"), - list - ) and - len(raw_data["photo"]) > 0 - ): - health.ok = False - health.abort_reason = AbortReason.NO_SUITABLE_DATA + @property + def raw_data_key(self): + return "photo" - return health + @property + def raw_data_type(self): + return list def get_attachment(self, message: telegram_interface.Message): - raw_data = message.raw_data - photos = raw_data["photo"] + photos = message.raw_data["photo"] biggest_photo = photos[0] for photo in photos[1:]: @@ -291,9 +337,6 @@ def get_attachment(self, message: telegram_interface.Message): return biggest_photo - def create_file_name(self, attachment, file): - return file["file_unique_id"] - class FileHandler(AttachmentHandler): """ @@ -308,26 +351,13 @@ def handle(): def telegram_action(self): return "upload_document" - def check_message_health(self, message: telegram_interface.Message): - health = MessageHealth(True) - value = self.get_attachment(message) - - if not ( - isinstance(value, dict) - ): - health.ok = False - health.abort_reason = AbortReason.NO_SUITABLE_DATA - - return health - - def get_attachment(self, message: telegram_interface.Message): - return message.raw_data.get("document") + @property + def raw_data_key(self): + return "document" - def create_file_name(self, attachment, file): - return ( - attachment.get("file_name") or - file["file_unique_id"] - ) + @property + def raw_data_type(self): + return dict class AudioHandler(AttachmentHandler): @@ -343,20 +373,13 @@ def handle(): def telegram_action(self): return "upload_audio" - def check_message_health(self, message: telegram_interface.Message): - health = MessageHealth(True) - value = self.get_attachment(message) - - if not ( - isinstance(value, dict) - ): - health.ok = False - health.abort_reason = AbortReason.NO_SUITABLE_DATA - - return health + @property + def raw_data_key(self): + return "audio" - def get_attachment(self, message: telegram_interface.Message): - return message.raw_data.get("audio") + @property + def raw_data_type(self): + return dict def create_file_name(self, attachment, file): name = file["file_unique_id"] @@ -367,9 +390,9 @@ def create_file_name(self, attachment, file): if ("performer" in attachment): name = f"{attachment['performer']} - {name}" - if ("mime_type" in attachment): - types = attachment["mime_type"].split("/") - extension = types[1] + extension = self.get_mime_type(attachment) + + if (extension): name = f"{name}.{extension}" return name @@ -388,30 +411,13 @@ def handle(): def telegram_action(self): return "upload_video" - def check_message_health(self, message: telegram_interface.Message): - health = MessageHealth(True) - value = self.get_attachment(message) - - if not ( - isinstance(value, dict) - ): - health.ok = False - health.abort_reason = AbortReason.NO_SUITABLE_DATA - - return health - - def get_attachment(self, message: telegram_interface.Message): - return message.raw_data.get("video") - - def create_file_name(self, attachment, file): - name = file["file_unique_id"] - - if ("mime_type" in attachment): - types = attachment["mime_type"].split("/") - extension = types[1] - name = f"{name}.{extension}" + @property + def raw_data_key(self): + return "video" - return name + @property + def raw_data_type(self): + return dict class VoiceHandler(AttachmentHandler): @@ -427,30 +433,13 @@ def handle(): def telegram_action(self): return "upload_audio" - def check_message_health(self, message: telegram_interface.Message): - health = MessageHealth(True) - value = self.get_attachment(message) - - if not ( - isinstance(value, dict) - ): - health.ok = False - health.abort_reason = AbortReason.NO_SUITABLE_DATA - - return health - - def get_attachment(self, message: telegram_interface.Message): - return message.raw_data.get("voice") - - def create_file_name(self, attachment, file): - name = file["file_unique_id"] - - if ("mime_type" in attachment): - types = attachment["mime_type"].split("/") - extension = types[1] - name = f"{name}.{extension}" + @property + def raw_data_key(self): + return "voice" - return name + @property + def raw_data_type(self): + return dict class URLHandler(AttachmentHandler): @@ -466,21 +455,16 @@ def handle(): def telegram_action(self): return "upload_document" - def check_message_health(self, message: telegram_interface.Message): - health = MessageHealth(True) - value = self.get_attachment(message) - - if not ( - isinstance(value, str) and - len(value) > 0 - ): - health.ok = False - health.abort_reason = AbortReason.NO_SUITABLE_DATA + @property + def raw_data_key(self): + return "url" - return health + @property + def raw_data_type(self): + return str def get_attachment(self, message: telegram_interface.Message): - return message.get_entity_value("url") + return message.get_entity_value(self.raw_data_key) def create_file_name(self, attachment, file): return attachment.split("/")[-1] From e31ffc5123135edf885d8dc73e099e745a3585e9 Mon Sep 17 00:00:00 2001 From: Sergey Kuznetsov Date: Sun, 6 Sep 2020 17:08:33 +0300 Subject: [PATCH 009/103] Add handling of file size limit #3 --- CHANGELOG.md | 5 +++ .../webhook/commands/common/responses.py | 8 +++++ .../telegram_bot/webhook/commands/help.py | 5 +++ .../telegram_bot/webhook/commands/upload.py | 36 +++++++++++++++++-- src/configs/flask.py | 4 +++ 5 files changed, 55 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 1a28523..75aa02c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,10 @@ ## Telegram Bot +### Added + +- Handling of files that exceed file size limit in 20 MB. At the moment the bot will asnwer with warning message, not trying to upload that file. [#3](https://github.com/Amaimersion/yandex-disk-telegram-bot/issues/3) + ## Project ### Improved @@ -11,6 +15,7 @@ ### Changed - Redirect to favicon will be handled by nginx. +- Biggest photo (from single photo file) will be selected based on total pixels count, not based on height. # 1.1.0 (May 9, 2020) diff --git a/src/blueprints/telegram_bot/webhook/commands/common/responses.py b/src/blueprints/telegram_bot/webhook/commands/common/responses.py index 8cfc595..3e49ab4 100644 --- a/src/blueprints/telegram_bot/webhook/commands/common/responses.py +++ b/src/blueprints/telegram_bot/webhook/commands/common/responses.py @@ -1,5 +1,7 @@ from enum import IntEnum, unique +from flask import current_app + from src.api import telegram @@ -10,6 +12,7 @@ class AbortReason(IntEnum): """ UNKNOWN = 1 NO_SUITABLE_DATA = 2 + EXCEED_FILE_SIZE_LIMIT = 3 def abort_command( @@ -35,6 +38,11 @@ def abort_command( "I can't handle this because " "you didn't send any suitable data " "for that command." + ), + AbortReason.EXCEED_FILE_SIZE_LIMIT: ( + "I can't handle file of such a large size. " + "At the moment my limit is " + f"{current_app.config['TELEGRAM_API_MAX_FILE_SIZE'] / 1000 / 1000} MB." # noqa ) } text = texts[reason] diff --git a/src/blueprints/telegram_bot/webhook/commands/help.py b/src/blueprints/telegram_bot/webhook/commands/help.py index 79548db..619c49c 100644 --- a/src/blueprints/telegram_bot/webhook/commands/help.py +++ b/src/blueprints/telegram_bot/webhook/commands/help.py @@ -13,6 +13,9 @@ def handle(): yd_upload_default_folder = current_app.config[ "YANDEX_DISK_API_DEFAULT_UPLOAD_FOLDER" ] + file_size_limit_in_mb = current_app.config[ + "TELEGRAM_API_MAX_FILE_SIZE" + ] / 1000 / 1000 text = ( "You can control me by sending these commands:" @@ -21,6 +24,8 @@ def handle(): "\n" f'For uploading "{to_code(yd_upload_default_folder)}" folder is used by default.' "\n" + f'Maximum size of every upload (except URL) is {file_size_limit_in_mb} MB.' + "\n" f"{CommandsNames.UPLOAD_PHOTO.value} — upload a photo. " "Original name will be not saved, quality of photo will be decreased. " "You can send photo without this command." diff --git a/src/blueprints/telegram_bot/webhook/commands/upload.py b/src/blueprints/telegram_bot/webhook/commands/upload.py index 5c89302..bac24e4 100644 --- a/src/blueprints/telegram_bot/webhook/commands/upload.py +++ b/src/blueprints/telegram_bot/webhook/commands/upload.py @@ -127,6 +127,12 @@ def check_message_health( if not (isinstance(value, self.raw_data_type)): health.ok = False health.abort_reason = AbortReason.NO_SUITABLE_DATA + elif ( + isinstance(value, dict) and + self.is_too_big_file(value) + ): + health.ok = False + health.abort_reason = AbortReason.EXCEED_FILE_SIZE_LIMIT return health @@ -172,6 +178,23 @@ def get_mime_type(self, attachment: dict) -> str: return result + def is_too_big_file(self, file: dict) -> bool: + """ + Checks if size of file exceeds limit size of upload. + + :param file: `dict` value from `self.get_attachment()`. + + :returns: File size exceeds upload limit size. + Always `False` if file size is unknown. + """ + limit = current_app.config["TELEGRAM_API_MAX_FILE_SIZE"] + size = limit + + if ("file_size" in file): + size = file["file_size"] + + return (size > limit) + @yd_access_token_required @get_db_data def upload(self) -> None: @@ -329,11 +352,18 @@ def raw_data_type(self): def get_attachment(self, message: telegram_interface.Message): photos = message.raw_data["photo"] - biggest_photo = photos[0] + biggest_photo = None + biggest_pixels_count = -1 + + for photo in photos: + if (self.is_too_big_file(photo)): + continue + + current_pixels_count = photo["width"] * photo["height"] - for photo in photos[1:]: - if (photo["height"] > biggest_photo["height"]): + if (current_pixels_count > biggest_pixels_count): biggest_photo = photo + biggest_pixels_count = current_pixels_count return biggest_photo diff --git a/src/configs/flask.py b/src/configs/flask.py index 26802ab..89dba81 100644 --- a/src/configs/flask.py +++ b/src/configs/flask.py @@ -39,6 +39,10 @@ class Config: # stop waiting for a Telegram response # after a given number of seconds TELEGRAM_API_TIMEOUT = 5 + # maximum file size in bytes that bot + # can handle by itself. + # It is Telegram limit, not bot + TELEGRAM_API_MAX_FILE_SIZE = 20 * 1000 * 1000 # Yandex OAuth API # stop waiting for a Yandex response From 5cbc849371ff93c15211a85361990d9d8e4f1006 Mon Sep 17 00:00:00 2001 From: Sergey Kuznetsov Date: Sun, 6 Sep 2020 17:28:08 +0300 Subject: [PATCH 010/103] Fix and refactoring of commands/upload.py --- .../telegram_bot/webhook/commands/upload.py | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/src/blueprints/telegram_bot/webhook/commands/upload.py b/src/blueprints/telegram_bot/webhook/commands/upload.py index bac24e4..1d2ee95 100644 --- a/src/blueprints/telegram_bot/webhook/commands/upload.py +++ b/src/blueprints/telegram_bot/webhook/commands/upload.py @@ -127,6 +127,12 @@ def check_message_health( if not (isinstance(value, self.raw_data_type)): health.ok = False health.abort_reason = AbortReason.NO_SUITABLE_DATA + elif ( + (type(value) in [str]) and + (len(value) == 0) + ): + health.ok = False + health.abort_reason = AbortReason.NO_SUITABLE_DATA elif ( isinstance(value, dict) and self.is_too_big_file(value) @@ -348,10 +354,11 @@ def raw_data_key(self): @property def raw_data_type(self): - return list + # dict, not list, because we will select biggest photo + return dict def get_attachment(self, message: telegram_interface.Message): - photos = message.raw_data["photo"] + photos = message.raw_data.get(self.raw_data_key, []) biggest_photo = None biggest_pixels_count = -1 From 6e5d68c9d4107a4a314c412bb0aa03dd33f3f236 Mon Sep 17 00:00:00 2001 From: Sergey Kuznetsov Date: Sun, 6 Sep 2020 17:28:33 +0300 Subject: [PATCH 011/103] Refactoring of commands/help.py --- src/blueprints/telegram_bot/webhook/commands/help.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/blueprints/telegram_bot/webhook/commands/help.py b/src/blueprints/telegram_bot/webhook/commands/help.py index 619c49c..3c0fb9e 100644 --- a/src/blueprints/telegram_bot/webhook/commands/help.py +++ b/src/blueprints/telegram_bot/webhook/commands/help.py @@ -13,9 +13,9 @@ def handle(): yd_upload_default_folder = current_app.config[ "YANDEX_DISK_API_DEFAULT_UPLOAD_FOLDER" ] - file_size_limit_in_mb = current_app.config[ + file_size_limit_in_mb = int(current_app.config[ "TELEGRAM_API_MAX_FILE_SIZE" - ] / 1000 / 1000 + ] / 1000 / 1000) text = ( "You can control me by sending these commands:" From ef485d3032b13e7c82b296576ff65e17ff682ae8 Mon Sep 17 00:00:00 2001 From: Sergey Kuznetsov Date: Mon, 7 Sep 2020 13:42:58 +0300 Subject: [PATCH 012/103] Refactoring of commands/upload.py --- .../telegram_bot/webhook/commands/upload.py | 13 +++++-------- 1 file changed, 5 insertions(+), 8 deletions(-) diff --git a/src/blueprints/telegram_bot/webhook/commands/upload.py b/src/blueprints/telegram_bot/webhook/commands/upload.py index 1d2ee95..1778d86 100644 --- a/src/blueprints/telegram_bot/webhook/commands/upload.py +++ b/src/blueprints/telegram_bot/webhook/commands/upload.py @@ -155,14 +155,11 @@ def create_file_name( :returns: Name of file which will be uploaded. """ - name = ( - attachment.get("file_name") or - file["file_unique_id"] - ) - extension = "" - - if (isinstance(attachment, dict)): - extension = self.get_mime_type(attachment) + if (isinstance(attachment, str)): + return attachment + + name = attachment.get("file_name") or file["file_unique_id"] + extension = self.get_mime_type(attachment) if (extension): name = f"{name}.{extension}" From 35c9bd792462ac778855367186a581b4fc83ec3e Mon Sep 17 00:00:00 2001 From: Sergey Kuznetsov Date: Mon, 7 Sep 2020 13:58:25 +0300 Subject: [PATCH 013/103] upload: Disable adding of mime type for files --- src/blueprints/telegram_bot/webhook/commands/upload.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/blueprints/telegram_bot/webhook/commands/upload.py b/src/blueprints/telegram_bot/webhook/commands/upload.py index 1778d86..746dc73 100644 --- a/src/blueprints/telegram_bot/webhook/commands/upload.py +++ b/src/blueprints/telegram_bot/webhook/commands/upload.py @@ -393,6 +393,10 @@ def raw_data_key(self): def raw_data_type(self): return dict + def get_mime_type(self, attachment): + # file name already contains type + return "" + class AudioHandler(AttachmentHandler): """ From e2d1af54377b4aa39450a2af8e204ae405efe06b Mon Sep 17 00:00:00 2001 From: Sergey Kuznetsov Date: Mon, 7 Sep 2020 14:26:24 +0300 Subject: [PATCH 014/103] Change file limit system from decimal to binary --- .../telegram_bot/webhook/commands/common/responses.py | 2 +- src/blueprints/telegram_bot/webhook/commands/help.py | 2 +- src/configs/flask.py | 9 +++++++-- 3 files changed, 9 insertions(+), 4 deletions(-) diff --git a/src/blueprints/telegram_bot/webhook/commands/common/responses.py b/src/blueprints/telegram_bot/webhook/commands/common/responses.py index 3e49ab4..7f88d93 100644 --- a/src/blueprints/telegram_bot/webhook/commands/common/responses.py +++ b/src/blueprints/telegram_bot/webhook/commands/common/responses.py @@ -42,7 +42,7 @@ def abort_command( AbortReason.EXCEED_FILE_SIZE_LIMIT: ( "I can't handle file of such a large size. " "At the moment my limit is " - f"{current_app.config['TELEGRAM_API_MAX_FILE_SIZE'] / 1000 / 1000} MB." # noqa + f"{current_app.config['TELEGRAM_API_MAX_FILE_SIZE'] / 1024 / 1024} MB." # noqa ) } text = texts[reason] diff --git a/src/blueprints/telegram_bot/webhook/commands/help.py b/src/blueprints/telegram_bot/webhook/commands/help.py index 3c0fb9e..a7a09ea 100644 --- a/src/blueprints/telegram_bot/webhook/commands/help.py +++ b/src/blueprints/telegram_bot/webhook/commands/help.py @@ -15,7 +15,7 @@ def handle(): ] file_size_limit_in_mb = int(current_app.config[ "TELEGRAM_API_MAX_FILE_SIZE" - ] / 1000 / 1000) + ] / 1024 / 1024) text = ( "You can control me by sending these commands:" diff --git a/src/configs/flask.py b/src/configs/flask.py index 89dba81..ed7733e 100644 --- a/src/configs/flask.py +++ b/src/configs/flask.py @@ -41,8 +41,13 @@ class Config: TELEGRAM_API_TIMEOUT = 5 # maximum file size in bytes that bot # can handle by itself. - # It is Telegram limit, not bot - TELEGRAM_API_MAX_FILE_SIZE = 20 * 1000 * 1000 + # It is Telegram limit, not bot. + # Binary system should be used, not decimal. + # For example, MebiBytes (M = 1024 * 1024), + # not MegaBytes (MB = 1000 * 1000). + # In Linux you can use `truncate -s 20480K test.txt` + # to create exactly 20M file + TELEGRAM_API_MAX_FILE_SIZE = 20 * 1024 * 1024 # Yandex OAuth API # stop waiting for a Yandex response From 654620d7df71a96d9952da697bcaee09316b7569 Mon Sep 17 00:00:00 2001 From: Sergey Kuznetsov Date: Thu, 10 Sep 2020 14:48:11 +0300 Subject: [PATCH 015/103] Add /publish function --- CHANGELOG.md | 1 + README.md | 1 + info/info.json | 4 ++ src/api/yandex/__init__.py | 3 +- src/api/yandex/methods.py | 14 ++++ .../telegram_bot/webhook/commands/__init__.py | 1 + .../webhook/commands/common/names.py | 1 + .../webhook/commands/common/yandex_api.py | 38 +++++++++++ .../telegram_bot/webhook/commands/help.py | 4 ++ .../telegram_bot/webhook/commands/publish.py | 68 +++++++++++++++++++ src/blueprints/telegram_bot/webhook/views.py | 3 +- 11 files changed, 136 insertions(+), 2 deletions(-) create mode 100644 src/blueprints/telegram_bot/webhook/commands/publish.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 75aa02c..8b84383 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,7 @@ ### Added +- `/publish`: publishing of files or folders. - Handling of files that exceed file size limit in 20 MB. At the moment the bot will asnwer with warning message, not trying to upload that file. [#3](https://github.com/Amaimersion/yandex-disk-telegram-bot/issues/3) ## Project diff --git a/README.md b/README.md index d89f9b1..2b3fee3 100644 --- a/README.md +++ b/README.md @@ -34,6 +34,7 @@ - uploading of audio. - uploading of video. - uploading of voice. +- publishing of files or folders. - creating of folders. diff --git a/info/info.json b/info/info.json index a020d51..afa9e24 100644 --- a/info/info.json +++ b/info/info.json @@ -45,6 +45,10 @@ "command": "upload_url", "description": "Upload a file using direct URL" }, + { + "command": "publish", + "description": "Publish a file or folder using OS path" + }, { "command": "create_folder", "description": "Create a folder using OS path" diff --git a/src/api/yandex/__init__.py b/src/api/yandex/__init__.py index e63ce3a..7aa6151 100644 --- a/src/api/yandex/__init__.py +++ b/src/api/yandex/__init__.py @@ -1,7 +1,8 @@ from .methods import ( get_access_token, upload_file_with_url, - create_folder + create_folder, + publish ) from .requests import ( make_link_request, diff --git a/src/api/yandex/methods.py b/src/api/yandex/methods.py index e1ce723..ee07a82 100644 --- a/src/api/yandex/methods.py +++ b/src/api/yandex/methods.py @@ -39,3 +39,17 @@ def create_folder(user_token: str, **kwargs): data=kwargs, user_token=user_token ) + + +def publish(user_token: str, **kwargs): + """ + https://yandex.ru/dev/disk/api/reference/publish-docpage/ + + - see `api/request.py` documentation for more. + """ + return make_disk_request( + http_method="PUT", + api_method="resources/publish", + data=kwargs, + user_token=user_token + ) diff --git a/src/blueprints/telegram_bot/webhook/commands/__init__.py b/src/blueprints/telegram_bot/webhook/commands/__init__.py index 6eb71fe..8b0bc6f 100644 --- a/src/blueprints/telegram_bot/webhook/commands/__init__.py +++ b/src/blueprints/telegram_bot/webhook/commands/__init__.py @@ -12,3 +12,4 @@ from .upload import handle_voice as upload_voice_handler from .upload import handle_url as upload_url_handler from .create_folder import handle as create_folder_handler +from .publish import handle as publish_handler diff --git a/src/blueprints/telegram_bot/webhook/commands/common/names.py b/src/blueprints/telegram_bot/webhook/commands/common/names.py index ecf0500..89c3755 100644 --- a/src/blueprints/telegram_bot/webhook/commands/common/names.py +++ b/src/blueprints/telegram_bot/webhook/commands/common/names.py @@ -19,3 +19,4 @@ class CommandsNames(Enum): UPLOAD_VOICE = "/upload_voice" UPLOAD_URL = "/upload_url" CREATE_FOLDER = "/create_folder" + PUBLISH = "/publish" diff --git a/src/blueprints/telegram_bot/webhook/commands/common/yandex_api.py b/src/blueprints/telegram_bot/webhook/commands/common/yandex_api.py index edeba79..f0aac40 100644 --- a/src/blueprints/telegram_bot/webhook/commands/common/yandex_api.py +++ b/src/blueprints/telegram_bot/webhook/commands/common/yandex_api.py @@ -41,6 +41,15 @@ class YandexAPIUploadFileError(Exception): pass +class YandexAPIPublishItemError(Exception): + """ + Unable to pubish an item from Yandex.Disk. + + - may contain human-readable error message. + """ + pass + + class YandexAPIExceededNumberOfStatusChecksError(Exception): """ There was too much attempts to check status @@ -105,6 +114,35 @@ def create_folder( return last_status_code +def publish_item( + user_access_token: str, + absolute_item_path: str +) -> None: + """ + Publish an item that already exists on Yandex.Disk. + + :raises: YandexAPIRequestError + :raises: YandexAPIPublishItemError + """ + try: + response = yandex.publish( + user_access_token, + path=absolute_item_path + ) + except Exception as error: + raise YandexAPIRequestError(error) + + response = response["content"] + is_error = is_error_yandex_response(response) + + if (is_error): + raise YandexAPIPublishItemError( + create_yandex_error_text( + response + ) + ) + + def upload_file_with_url( user_access_token: str, folder_path: str, diff --git a/src/blueprints/telegram_bot/webhook/commands/help.py b/src/blueprints/telegram_bot/webhook/commands/help.py index a7a09ea..b2f10f5 100644 --- a/src/blueprints/telegram_bot/webhook/commands/help.py +++ b/src/blueprints/telegram_bot/webhook/commands/help.py @@ -50,6 +50,10 @@ def handle(): "Original name will be saved. " "You can send direct URL to a file without this command." "\n" + f"{CommandsNames.PUBLISH.value} — publish a file or folder that already exists. " + "Send full name of item to publish with this command. " + f'Example: {to_code(f"/{yd_upload_default_folder}/files/photo.jpeg")}' + "\n" f"{CommandsNames.CREATE_FOLDER.value}— create a folder. " "Send folder name to create with this command. " "Folder name should starts from root, " diff --git a/src/blueprints/telegram_bot/webhook/commands/publish.py b/src/blueprints/telegram_bot/webhook/commands/publish.py new file mode 100644 index 0000000..1245e52 --- /dev/null +++ b/src/blueprints/telegram_bot/webhook/commands/publish.py @@ -0,0 +1,68 @@ +from flask import g + +from src.api import telegram +from .common.responses import ( + cancel_command, + abort_command, + AbortReason +) +from .common.decorators import ( + yd_access_token_required, + get_db_data +) +from .common.yandex_api import ( + publish_item, + YandexAPIPublishItemError, + YandexAPIRequestError +) +from . import CommandsNames + + +@yd_access_token_required +@get_db_data +def handle(): + """ + Handles `/publish` command. + """ + message = g.telegram_message + user = g.db_user + chat = g.db_chat + message_text = message.get_text() + path = message_text.replace( + CommandsNames.PUBLISH.value, + "" + ).strip() + + if not (path): + return abort_command( + chat.telegram_id, + AbortReason.NO_SUITABLE_DATA + ) + + access_token = user.yandex_disk_token.get_access_token() + + try: + publish_item( + access_token, + path + ) + except YandexAPIRequestError as error: + print(error) + return cancel_command(chat.telegram_id) + except YandexAPIPublishItemError as error: + error_text = ( + str(error) or + "Unknown Yandex.Disk error" + ) + + telegram.send_message( + chat_id=chat.telegram_id, + text=error_text + ) + + return + + telegram.send_message( + chat_id=chat.telegram_id, + text="Published" + ) diff --git a/src/blueprints/telegram_bot/webhook/views.py b/src/blueprints/telegram_bot/webhook/views.py index 004a89f..361eafb 100644 --- a/src/blueprints/telegram_bot/webhook/views.py +++ b/src/blueprints/telegram_bot/webhook/views.py @@ -73,7 +73,8 @@ def route_command(command: str) -> None: CommandNames.UPLOAD_VIDEO.value: commands.upload_video_handler, CommandNames.UPLOAD_VOICE.value: commands.upload_voice_handler, CommandNames.UPLOAD_URL.value: commands.upload_url_handler, - CommandNames.CREATE_FOLDER.value: commands.create_folder_handler + CommandNames.CREATE_FOLDER.value: commands.create_folder_handler, + CommandNames.PUBLISH.value: commands.publish_handler } method = routes.get(command, commands.unknown_handler) From f1240be6299cd861d0ae609813490d99eab0d209 Mon Sep 17 00:00:00 2001 From: Sergey Kuznetsov Date: Thu, 10 Sep 2020 15:21:12 +0300 Subject: [PATCH 016/103] Refactoring of help message --- src/blueprints/telegram_bot/webhook/commands/help.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/blueprints/telegram_bot/webhook/commands/help.py b/src/blueprints/telegram_bot/webhook/commands/help.py index b2f10f5..8afb01e 100644 --- a/src/blueprints/telegram_bot/webhook/commands/help.py +++ b/src/blueprints/telegram_bot/webhook/commands/help.py @@ -54,7 +54,7 @@ def handle(): "Send full name of item to publish with this command. " f'Example: {to_code(f"/{yd_upload_default_folder}/files/photo.jpeg")}' "\n" - f"{CommandsNames.CREATE_FOLDER.value}— create a folder. " + f"{CommandsNames.CREATE_FOLDER.value} — create a folder. " "Send folder name to create with this command. " "Folder name should starts from root, " f'nested folders should be separated with "{to_code("/")}" character.' From 8901b70411bc86260dc46b320eeda82920bffa69 Mon Sep 17 00:00:00 2001 From: Sergey Kuznetsov Date: Sat, 12 Sep 2020 14:07:36 +0300 Subject: [PATCH 017/103] Add /space function --- CHANGELOG.md | 1 + README.md | 1 + info/info.json | 4 + requirements.txt | 6 + src/api/telegram/__init__.py | 3 +- src/api/telegram/methods.py | 16 ++ src/api/telegram/requests.py | 28 ++- src/api/yandex/__init__.py | 3 +- src/api/yandex/methods.py | 14 ++ .../telegram_bot/webhook/commands/__init__.py | 1 + .../webhook/commands/common/names.py | 1 + .../webhook/commands/common/yandex_api.py | 28 +++ .../telegram_bot/webhook/commands/help.py | 2 + .../telegram_bot/webhook/commands/space.py | 183 ++++++++++++++++++ src/blueprints/telegram_bot/webhook/views.py | 3 +- 15 files changed, 286 insertions(+), 8 deletions(-) create mode 100644 src/blueprints/telegram_bot/webhook/commands/space.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 8b84383..299525a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,7 @@ ### Added - `/publish`: publishing of files or folders. +- `/space`: getting of information about remaining Yandex.Disk space. - Handling of files that exceed file size limit in 20 MB. At the moment the bot will asnwer with warning message, not trying to upload that file. [#3](https://github.com/Amaimersion/yandex-disk-telegram-bot/issues/3) ## Project diff --git a/README.md b/README.md index 2b3fee3..8d7173b 100644 --- a/README.md +++ b/README.md @@ -36,6 +36,7 @@ - uploading of voice. - publishing of files or folders. - creating of folders. +- getting of information about the disk. ## Requirements diff --git a/info/info.json b/info/info.json index afa9e24..6248fd8 100644 --- a/info/info.json +++ b/info/info.json @@ -53,6 +53,10 @@ "command": "create_folder", "description": "Create a folder using OS path" }, + { + "command": "space", + "description": "Information about remaining space" + }, { "command": "grant_access", "description": "Grant me an access to your Yandex.Disk" diff --git a/requirements.txt b/requirements.txt index f225ee4..3f3edfe 100644 --- a/requirements.txt +++ b/requirements.txt @@ -17,9 +17,13 @@ gunicorn==20.0.4 idna==2.9 itsdangerous==1.1.0 Jinja2==2.11.1 +kaleido==0.0.3.post1 Mako==1.1.2 MarkupSafe==1.1.1 mccabe==0.6.1 +numpy==1.19.2 +pandas==1.1.2 +plotly==4.10.0 psycopg2-binary==2.8.4 pycodestyle==2.5.0 pycparser==2.20 @@ -28,7 +32,9 @@ PyJWT==1.7.1 python-dateutil==2.8.1 python-dotenv==0.12.0 python-editor==1.0.4 +pytz==2020.1 requests==2.23.0 +retrying==1.3.3 six==1.14.0 SQLAlchemy==1.3.15 text-unidecode==1.3 diff --git a/src/api/telegram/__init__.py b/src/api/telegram/__init__.py index edce8c5..b88af85 100644 --- a/src/api/telegram/__init__.py +++ b/src/api/telegram/__init__.py @@ -2,7 +2,8 @@ send_message, get_file, send_chat_action, - edit_message_text + edit_message_text, + send_photo ) from .requests import ( create_file_download_url diff --git a/src/api/telegram/methods.py b/src/api/telegram/methods.py index 7a3152e..0c41e97 100644 --- a/src/api/telegram/methods.py +++ b/src/api/telegram/methods.py @@ -35,3 +35,19 @@ def edit_message_text(**kwargs): - see `api/request.py` documentation for more. """ return make_request("editMessageText", kwargs) + + +def send_photo(**kwargs): + """ + https://core.telegram.org/bots/api#sendphoto + + - see `api/request.py` documentation for more. + - specify `photo` as tuple from `files` in + https://requests.readthedocs.io/en/latest/api/#requests.request + """ + key = "photo" + files = { + key: kwargs.pop(key) + } + + return make_request("sendPhoto", data=kwargs, files=files) diff --git a/src/api/telegram/requests.py b/src/api/telegram/requests.py index 5051c18..761836e 100644 --- a/src/api/telegram/requests.py +++ b/src/api/telegram/requests.py @@ -43,30 +43,48 @@ def create_file_download_url(file_path: str) -> str: ) -def make_request(method_name: str, data: dict) -> dict: +def make_request( + method_name: str, + data: dict, + files: dict = None +) -> dict: """ Makes HTTP request to Telegram Bot API. - see `api/request.py` documentation for more. :param method_name: Name of API method in URL. - :param data: JSON data to send. + :param data: JSON data to send. It will be sent as + `application/json` payload. + :param files: Files data to send. If specified, then + `data` will be sent as query string. Files will be sent + as `multipart/form-data` payload. See `files` for more - + https://requests.readthedocs.io/en/latest/api/#requests.request :raises TelegramBotApiException: See `telegram/exceptions.py` documentation for more. """ url = create_bot_url(method_name) timeout = current_app.config["TELEGRAM_API_TIMEOUT"] + payload = { + "json": data + } + + if files: + payload = { + "files": files, + "params": data + } + result = request( content_type="json", method="POST", url=url, - json=data, timeout=timeout, allow_redirects=False, - verify=True + verify=True, + **payload ) - ok = result["content"]["ok"] if not ok: diff --git a/src/api/yandex/__init__.py b/src/api/yandex/__init__.py index 7aa6151..38542ab 100644 --- a/src/api/yandex/__init__.py +++ b/src/api/yandex/__init__.py @@ -2,7 +2,8 @@ get_access_token, upload_file_with_url, create_folder, - publish + publish, + get_disk_info ) from .requests import ( make_link_request, diff --git a/src/api/yandex/methods.py b/src/api/yandex/methods.py index ee07a82..cf71f5b 100644 --- a/src/api/yandex/methods.py +++ b/src/api/yandex/methods.py @@ -53,3 +53,17 @@ def publish(user_token: str, **kwargs): data=kwargs, user_token=user_token ) + + +def get_disk_info(user_token: str, **kwargs): + """ + https://yandex.ru/dev/disk/api/reference/capacity-docpage/ + + - see `api/request.py` documentation for more. + """ + return make_disk_request( + http_method="GET", + api_method="", + data=kwargs, + user_token=user_token + ) diff --git a/src/blueprints/telegram_bot/webhook/commands/__init__.py b/src/blueprints/telegram_bot/webhook/commands/__init__.py index 8b0bc6f..44c66f4 100644 --- a/src/blueprints/telegram_bot/webhook/commands/__init__.py +++ b/src/blueprints/telegram_bot/webhook/commands/__init__.py @@ -13,3 +13,4 @@ from .upload import handle_url as upload_url_handler from .create_folder import handle as create_folder_handler from .publish import handle as publish_handler +from .space import handle as space_handler diff --git a/src/blueprints/telegram_bot/webhook/commands/common/names.py b/src/blueprints/telegram_bot/webhook/commands/common/names.py index 89c3755..4e08dbd 100644 --- a/src/blueprints/telegram_bot/webhook/commands/common/names.py +++ b/src/blueprints/telegram_bot/webhook/commands/common/names.py @@ -20,3 +20,4 @@ class CommandsNames(Enum): UPLOAD_URL = "/upload_url" CREATE_FOLDER = "/create_folder" PUBLISH = "/publish" + SPACE = "/space" diff --git a/src/blueprints/telegram_bot/webhook/commands/common/yandex_api.py b/src/blueprints/telegram_bot/webhook/commands/common/yandex_api.py index f0aac40..f162828 100644 --- a/src/blueprints/telegram_bot/webhook/commands/common/yandex_api.py +++ b/src/blueprints/telegram_bot/webhook/commands/common/yandex_api.py @@ -242,6 +242,34 @@ def upload_file_with_url( raise YandexAPIExceededNumberOfStatusChecksError() +def get_disk_info(user_access_token: str) -> dict: + """ + See for interface: + - https://yandex.ru/dev/disk/api/reference/capacity-docpage/ + - https://dev.yandex.net/disk-polygon/#!/v147disk + + :returns: Information about user Yandex.Disk. + + :raises: YandexAPIRequestError + """ + try: + response = yandex.get_disk_info(user_access_token) + except Exception as error: + raise YandexAPIRequestError(error) + + response = response["content"] + is_error = is_error_yandex_response(response) + + if (is_error): + raise YandexAPIUploadFileError( + create_yandex_error_text( + response + ) + ) + + return response + + def is_error_yandex_response(data: dict) -> bool: """ :returns: Yandex response contains error or not. diff --git a/src/blueprints/telegram_bot/webhook/commands/help.py b/src/blueprints/telegram_bot/webhook/commands/help.py index 8afb01e..5d4eed9 100644 --- a/src/blueprints/telegram_bot/webhook/commands/help.py +++ b/src/blueprints/telegram_bot/webhook/commands/help.py @@ -58,6 +58,8 @@ def handle(): "Send folder name to create with this command. " "Folder name should starts from root, " f'nested folders should be separated with "{to_code("/")}" character.' + "\n" + f"{CommandsNames.SPACE.value} — get information about remaining Yandex.Disk space. " "\n\n" "Yandex.Disk Access" "\n" diff --git a/src/blueprints/telegram_bot/webhook/commands/space.py b/src/blueprints/telegram_bot/webhook/commands/space.py new file mode 100644 index 0000000..16fad7b --- /dev/null +++ b/src/blueprints/telegram_bot/webhook/commands/space.py @@ -0,0 +1,183 @@ +from string import ascii_letters, digits +from datetime import datetime, timezone + +from flask import g +from plotly.graph_objects import Pie, Figure +from plotly.express import colors +from plotly.io import to_image + +from src.api import telegram +from .common.responses import ( + cancel_command, + AbortReason +) +from .common.decorators import ( + yd_access_token_required, + get_db_data +) +from .common.yandex_api import ( + get_disk_info, + YandexAPIRequestError +) +from . import CommandsNames + + +@yd_access_token_required +@get_db_data +def handle(): + """ + Handles `/publish` command. + """ + user = g.db_user + chat = g.db_chat + access_token = user.yandex_disk_token.get_access_token() + disk_info = None + + try: + disk_info = get_disk_info(access_token) + except YandexAPIRequestError as error: + print(error) + return cancel_command(chat.telegram_id) + + current_date = get_current_date() + jpeg_image = create_space_chart( + total_space=disk_info["total_space"], + used_space=disk_info["used_space"], + trash_size=disk_info["trash_size"], + caption=current_date + ) + + telegram.send_photo( + chat_id=chat.telegram_id, + photo=( + f"{to_filename(current_date)}.jpg", + jpeg_image, + "image/jpeg" + ), + caption=f"Yandex.Disk space at {current_date}" + ) + + +def create_space_chart( + total_space: int, + used_space: int, + trash_size: int, + caption: str = None +) -> bytes: + """ + Creates Yandex.Disk space chart. + + - all sizes (total, used, trash) should be + specified in binary bytes (B). They will be + converted to binary gigabytes (GB). + + :returns: JPEG image as bytes. + """ + free_space = b_to_gb(total_space - used_space - trash_size) + total_space = b_to_gb(total_space) + used_space = b_to_gb(used_space) + trash_size = b_to_gb(trash_size) + + chart = Pie( + labels=[ + "Used Space", + "Trash Size", + "Free Space" + ], + values=[ + used_space, + trash_size, + free_space + ], + text=[ + "Used", + "Trash", + "Free" + ], + marker={ + "colors": [ + colors.sequential.Rainbow[3], + colors.sequential.Rainbow[8], + colors.sequential.Rainbow[5] + ], + "line": { + "width": 0.2 + } + }, + sort=False, + direction="clockwise", + texttemplate=( + "%{text}
" + "%{value:.2f} GB
" + "%{percent}" + ), + textposition="outside", + hole=0.5 + ) + figure = Figure( + data=chart, + layout={ + "title": { + "text": caption, + "font": { + "size": 20 + } + }, + "annotations": [ + { + "align": "center", + "showarrow": False, + "text": ( + "Total
" + f"{total_space:.2f} GB
" + "100%" + ) + } + ], + "width": 1000, + "height": 800, + "font": { + "size": 27 + }, + "margin": { + "t": 140, + "b": 40, + "r": 230, + "l": 150 + } + } + ) + + return to_image(figure, format="jpeg") + + +def b_to_gb(value: int) -> int: + """ + Converts binary bytes to binary gigabytes. + """ + return (value / 1024 / 1024 / 1024) + + +def get_current_date() -> str: + """ + :returns: Current date as string representation. + """ + now = datetime.now(timezone.utc) + + return now.strftime("%d.%m.%Y %H:%M %Z") + + +def to_filename(value: str) -> str: + """ + :returns: Valid filename. + """ + valid_chars = f"-_.{ascii_letters}{digits}" + filename = value.lower() + filename = ( + filename + .replace(" ", "_") + .replace(":", "_") + ) + filename = "".join(x for x in filename if x in valid_chars) + + return filename diff --git a/src/blueprints/telegram_bot/webhook/views.py b/src/blueprints/telegram_bot/webhook/views.py index 361eafb..8f65ff1 100644 --- a/src/blueprints/telegram_bot/webhook/views.py +++ b/src/blueprints/telegram_bot/webhook/views.py @@ -74,7 +74,8 @@ def route_command(command: str) -> None: CommandNames.UPLOAD_VOICE.value: commands.upload_voice_handler, CommandNames.UPLOAD_URL.value: commands.upload_url_handler, CommandNames.CREATE_FOLDER.value: commands.create_folder_handler, - CommandNames.PUBLISH.value: commands.publish_handler + CommandNames.PUBLISH.value: commands.publish_handler, + CommandNames.SPACE.value: commands.space_handler } method = routes.get(command, commands.unknown_handler) From bdb41ee3152b470c2fbf86a1d2f8720c54d4aaa8 Mon Sep 17 00:00:00 2001 From: Sergey Kuznetsov Date: Sat, 12 Sep 2020 14:24:14 +0300 Subject: [PATCH 018/103] Increase left margin of space chart --- src/blueprints/telegram_bot/webhook/commands/space.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/blueprints/telegram_bot/webhook/commands/space.py b/src/blueprints/telegram_bot/webhook/commands/space.py index 16fad7b..97b56ca 100644 --- a/src/blueprints/telegram_bot/webhook/commands/space.py +++ b/src/blueprints/telegram_bot/webhook/commands/space.py @@ -143,7 +143,7 @@ def create_space_chart( "t": 140, "b": 40, "r": 230, - "l": 150 + "l": 165 } } ) From ca627363d1009316d24e678907ac98aca0708656 Mon Sep 17 00:00:00 2001 From: Sergey Kuznetsov Date: Sun, 13 Sep 2020 12:50:39 +0300 Subject: [PATCH 019/103] Add /unpublish function --- CHANGELOG.md | 1 + README.md | 2 +- info/info.json | 4 ++ src/api/yandex/__init__.py | 1 + src/api/yandex/methods.py | 14 ++++ .../telegram_bot/webhook/commands/__init__.py | 1 + .../webhook/commands/common/names.py | 1 + .../webhook/commands/common/yandex_api.py | 38 +++++++++++ .../telegram_bot/webhook/commands/help.py | 4 ++ .../webhook/commands/unpublish.py | 68 +++++++++++++++++++ src/blueprints/telegram_bot/webhook/views.py | 1 + 11 files changed, 134 insertions(+), 1 deletion(-) create mode 100644 src/blueprints/telegram_bot/webhook/commands/unpublish.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 299525a..4a09a3c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,7 @@ ### Added - `/publish`: publishing of files or folders. +- `/unpublish`: unpublishing of files or folders. - `/space`: getting of information about remaining Yandex.Disk space. - Handling of files that exceed file size limit in 20 MB. At the moment the bot will asnwer with warning message, not trying to upload that file. [#3](https://github.com/Amaimersion/yandex-disk-telegram-bot/issues/3) diff --git a/README.md b/README.md index 8d7173b..d8af996 100644 --- a/README.md +++ b/README.md @@ -34,7 +34,7 @@ - uploading of audio. - uploading of video. - uploading of voice. -- publishing of files or folders. +- publishing and unpublishing of files or folders. - creating of folders. - getting of information about the disk. diff --git a/info/info.json b/info/info.json index 6248fd8..04293d9 100644 --- a/info/info.json +++ b/info/info.json @@ -49,6 +49,10 @@ "command": "publish", "description": "Publish a file or folder using OS path" }, + { + "command": "unpublish", + "description": "Unpublish a file or folder using OS path" + }, { "command": "create_folder", "description": "Create a folder using OS path" diff --git a/src/api/yandex/__init__.py b/src/api/yandex/__init__.py index 38542ab..42ad11a 100644 --- a/src/api/yandex/__init__.py +++ b/src/api/yandex/__init__.py @@ -3,6 +3,7 @@ upload_file_with_url, create_folder, publish, + unpublish, get_disk_info ) from .requests import ( diff --git a/src/api/yandex/methods.py b/src/api/yandex/methods.py index cf71f5b..32b9ac2 100644 --- a/src/api/yandex/methods.py +++ b/src/api/yandex/methods.py @@ -55,6 +55,20 @@ def publish(user_token: str, **kwargs): ) +def unpublish(user_token: str, **kwargs): + """ + https://yandex.ru/dev/disk/api/reference/publish-docpage/ + + - see `api/request.py` documentation for more. + """ + return make_disk_request( + http_method="PUT", + api_method="resources/unpublish", + data=kwargs, + user_token=user_token + ) + + def get_disk_info(user_token: str, **kwargs): """ https://yandex.ru/dev/disk/api/reference/capacity-docpage/ diff --git a/src/blueprints/telegram_bot/webhook/commands/__init__.py b/src/blueprints/telegram_bot/webhook/commands/__init__.py index 44c66f4..fed196a 100644 --- a/src/blueprints/telegram_bot/webhook/commands/__init__.py +++ b/src/blueprints/telegram_bot/webhook/commands/__init__.py @@ -13,4 +13,5 @@ from .upload import handle_url as upload_url_handler from .create_folder import handle as create_folder_handler from .publish import handle as publish_handler +from .unpublish import handle as unpublish_handler from .space import handle as space_handler diff --git a/src/blueprints/telegram_bot/webhook/commands/common/names.py b/src/blueprints/telegram_bot/webhook/commands/common/names.py index 4e08dbd..d9f3da7 100644 --- a/src/blueprints/telegram_bot/webhook/commands/common/names.py +++ b/src/blueprints/telegram_bot/webhook/commands/common/names.py @@ -20,4 +20,5 @@ class CommandsNames(Enum): UPLOAD_URL = "/upload_url" CREATE_FOLDER = "/create_folder" PUBLISH = "/publish" + UNPUBLISH = "/unpublish" SPACE = "/space" diff --git a/src/blueprints/telegram_bot/webhook/commands/common/yandex_api.py b/src/blueprints/telegram_bot/webhook/commands/common/yandex_api.py index f162828..60183ff 100644 --- a/src/blueprints/telegram_bot/webhook/commands/common/yandex_api.py +++ b/src/blueprints/telegram_bot/webhook/commands/common/yandex_api.py @@ -50,6 +50,15 @@ class YandexAPIPublishItemError(Exception): pass +class YandexAPIUnpublishItemError(Exception): + """ + Unable to unpublish an item from Yandex.Disk. + + - may contain human-readable error message. + """ + pass + + class YandexAPIExceededNumberOfStatusChecksError(Exception): """ There was too much attempts to check status @@ -143,6 +152,35 @@ def publish_item( ) +def unpublish_item( + user_access_token: str, + absolute_item_path: str +) -> None: + """ + Unpublish an item that already exists on Yandex.Disk. + + :raises: YandexAPIRequestError + :raises: YandexAPIUnpublishItemError + """ + try: + response = yandex.unpublish( + user_access_token, + path=absolute_item_path + ) + except Exception as error: + raise YandexAPIRequestError(error) + + response = response["content"] + is_error = is_error_yandex_response(response) + + if (is_error): + raise YandexAPIUnpublishItemError( + create_yandex_error_text( + response + ) + ) + + def upload_file_with_url( user_access_token: str, folder_path: str, diff --git a/src/blueprints/telegram_bot/webhook/commands/help.py b/src/blueprints/telegram_bot/webhook/commands/help.py index 5d4eed9..66ee466 100644 --- a/src/blueprints/telegram_bot/webhook/commands/help.py +++ b/src/blueprints/telegram_bot/webhook/commands/help.py @@ -54,6 +54,10 @@ def handle(): "Send full name of item to publish with this command. " f'Example: {to_code(f"/{yd_upload_default_folder}/files/photo.jpeg")}' "\n" + f"{CommandsNames.UNPUBLISH.value} — unpublish a file or folder that already exists. " + "Send full name of item to unpublish with this command. " + f'Example: {to_code(f"/{yd_upload_default_folder}/files/photo.jpeg")}' + "\n" f"{CommandsNames.CREATE_FOLDER.value} — create a folder. " "Send folder name to create with this command. " "Folder name should starts from root, " diff --git a/src/blueprints/telegram_bot/webhook/commands/unpublish.py b/src/blueprints/telegram_bot/webhook/commands/unpublish.py new file mode 100644 index 0000000..80457d3 --- /dev/null +++ b/src/blueprints/telegram_bot/webhook/commands/unpublish.py @@ -0,0 +1,68 @@ +from flask import g + +from src.api import telegram +from .common.responses import ( + cancel_command, + abort_command, + AbortReason +) +from .common.decorators import ( + yd_access_token_required, + get_db_data +) +from .common.yandex_api import ( + unpublish_item, + YandexAPIUnpublishItemError, + YandexAPIRequestError +) +from . import CommandsNames + + +@yd_access_token_required +@get_db_data +def handle(): + """ + Handles `/unpublish` command. + """ + message = g.telegram_message + user = g.db_user + chat = g.db_chat + message_text = message.get_text() + path = message_text.replace( + CommandsNames.UNPUBLISH.value, + "" + ).strip() + + if not (path): + return abort_command( + chat.telegram_id, + AbortReason.NO_SUITABLE_DATA + ) + + access_token = user.yandex_disk_token.get_access_token() + + try: + unpublish_item( + access_token, + path + ) + except YandexAPIRequestError as error: + print(error) + return cancel_command(chat.telegram_id) + except YandexAPIUnpublishItemError as error: + error_text = ( + str(error) or + "Unknown Yandex.Disk error" + ) + + telegram.send_message( + chat_id=chat.telegram_id, + text=error_text + ) + + return + + telegram.send_message( + chat_id=chat.telegram_id, + text="Unpublished" + ) diff --git a/src/blueprints/telegram_bot/webhook/views.py b/src/blueprints/telegram_bot/webhook/views.py index 8f65ff1..9b10389 100644 --- a/src/blueprints/telegram_bot/webhook/views.py +++ b/src/blueprints/telegram_bot/webhook/views.py @@ -75,6 +75,7 @@ def route_command(command: str) -> None: CommandNames.UPLOAD_URL.value: commands.upload_url_handler, CommandNames.CREATE_FOLDER.value: commands.create_folder_handler, CommandNames.PUBLISH.value: commands.publish_handler, + CommandNames.UNPUBLISH.value: commands.unpublish_handler, CommandNames.SPACE.value: commands.space_handler } method = routes.get(command, commands.unknown_handler) From 6ca1ec19903ea5bca19449da1164abdd9555452d Mon Sep 17 00:00:00 2001 From: Sergey Kuznetsov Date: Sun, 13 Sep 2020 14:04:30 +0300 Subject: [PATCH 020/103] web site: 404 -> 302 --- CHANGELOG.md | 4 ++++ src/app.py | 20 ++++++++++++++++++++ src/configs/flask.py | 1 + 3 files changed, 25 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 4a09a3c..9cd20ab 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -15,6 +15,10 @@ - Upgrade `python` to 3.8.2. +### Added + +- Web Site: 302 (redirect to Telegram) will be returned instead of 404 (not found page). + ### Changed - Redirect to favicon will be handled by nginx. diff --git a/src/app.py b/src/app.py index c9ad18a..986cb07 100644 --- a/src/app.py +++ b/src/app.py @@ -31,6 +31,7 @@ def create_app(config_name: str = None) -> Flask: configure_db(app) configure_blueprints(app) configure_redirects(app) + configure_error_handlers(app) return app @@ -87,3 +88,22 @@ def favicon(): filename="favicons/favicon.ico" ) ) + + +def configure_error_handlers(app: Flask) -> None: + """ + Configures error handlers. + """ + @app.errorhandler(404) + def not_found(error): + """ + We will redirect all requests rather than send "Not Found" + error, because we using web pages only for exceptional cases. + It is expected that all interaction with user should go + through Telegram when possible. + """ + return redirect( + location=app.config["PROJECT_URL_FOR_BOT"], + # temporary, in case if some routes will be added in future + code=302 + ) diff --git a/src/configs/flask.py b/src/configs/flask.py index ed7733e..af53de6 100644 --- a/src/configs/flask.py +++ b/src/configs/flask.py @@ -25,6 +25,7 @@ class Config: PROJECT_URL_FOR_ISSUE = "https://github.com/Amaimersion/yandex-disk-telegram-bot/issues/new?template=bug_report.md" # noqa PROJECT_URL_FOR_REQUEST = "https://github.com/Amaimersion/yandex-disk-telegram-bot/issues/new?template=feature_request.md" # noqa PROJECT_URL_FOR_QUESTION = "https://github.com/Amaimersion/yandex-disk-telegram-bot/issues/new?template=question.md" # noqa + PROJECT_URL_FOR_BOT = "https://t.me/Ya_Disk_Bot" # Flask DEBUG = False From 1ebd27440436ecdb2091b099bebb2444fd86d720 Mon Sep 17 00:00:00 2001 From: Sergey Kuznetsov Date: Thu, 22 Oct 2020 10:22:03 +0300 Subject: [PATCH 021/103] Add information about upload limit --- README.md | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index d8af996..bbed7e1 100644 --- a/README.md +++ b/README.md @@ -28,12 +28,12 @@ ## Features -- uploading of photos. -- uploading of files. +- uploading of photos (limit is 20 MB). +- uploading of files (limit is 20 MB). - uploading of files using direct URL. -- uploading of audio. -- uploading of video. -- uploading of voice. +- uploading of audio (limit is 20 MB). +- uploading of video (limit is 20 MB). +- uploading of voice (limit is 20 MB). - publishing and unpublishing of files or folders. - creating of folders. - getting of information about the disk. From f5db22df9f00147a037abc2ee69c438a30c4f7f5 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 27 Oct 2020 22:28:03 +0000 Subject: [PATCH 022/103] Bump cryptography from 2.8 to 3.2 Bumps [cryptography](https://github.com/pyca/cryptography) from 2.8 to 3.2. - [Release notes](https://github.com/pyca/cryptography/releases) - [Changelog](https://github.com/pyca/cryptography/blob/master/CHANGELOG.rst) - [Commits](https://github.com/pyca/cryptography/compare/2.8...3.2) Signed-off-by: dependabot[bot] --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index f225ee4..df9b2cf 100644 --- a/requirements.txt +++ b/requirements.txt @@ -4,7 +4,7 @@ certifi==2019.11.28 cffi==1.14.0 chardet==3.0.4 click==7.1.1 -cryptography==2.8 +cryptography==3.2 entrypoints==0.3 Faker==4.0.2 flake8==3.7.9 From 7edd91662df8ce6082efe8ce0db7201d805eb6b7 Mon Sep 17 00:00:00 2001 From: Sergey Kuznetsov Date: Sun, 1 Nov 2020 14:26:24 +0300 Subject: [PATCH 023/103] Upgrade all requirements --- CHANGELOG.md | 1 + requirements.txt | 55 ++++++++++++++++++++++++------------------------ 2 files changed, 29 insertions(+), 27 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 9cd20ab..168b4fb 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -14,6 +14,7 @@ ### Improved - Upgrade `python` to 3.8.2. +- All requirements upgraded to latest version. ### Added diff --git a/requirements.txt b/requirements.txt index 787d962..0892963 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,43 +1,44 @@ -alembic==1.4.2 -autopep8==1.5 -certifi==2019.11.28 -cffi==1.14.0 +alembic==1.4.3 +autopep8==1.5.4 +certifi==2020.6.20 +cffi==1.14.3 chardet==3.0.4 -click==7.1.1 -cryptography==3.2 +click==7.1.2 +cryptography==3.2.1 entrypoints==0.3 -Faker==4.0.2 -flake8==3.7.9 -Flask==1.1.1 +Faker==4.14.0 +flake8==3.8.4 +Flask==1.1.2 Flask-Migrate==2.5.3 -Flask-SQLAlchemy==2.4.1 +Flask-SQLAlchemy==2.4.4 gevent==1.5.0 -greenlet==0.4.15 +greenlet==0.4.17 gunicorn==20.0.4 -idna==2.9 +idna==2.10 itsdangerous==1.1.0 -Jinja2==2.11.1 -kaleido==0.0.3.post1 -Mako==1.1.2 +Jinja2==2.11.2 +kaleido==0.1.0a3 +Mako==1.1.3 MarkupSafe==1.1.1 mccabe==0.6.1 -numpy==1.19.2 -pandas==1.1.2 -plotly==4.10.0 -psycopg2-binary==2.8.4 -pycodestyle==2.5.0 +numpy==1.19.3 +pandas==1.1.4 +plotly==4.12.0 +psycopg2-binary==2.8.6 +pycodestyle==2.6.0 pycparser==2.20 -pyflakes==2.1.1 +pyflakes==2.2.0 PyJWT==1.7.1 python-dateutil==2.8.1 -python-dotenv==0.12.0 +python-dotenv==0.15.0 python-editor==1.0.4 pytz==2020.1 -requests==2.23.0 +requests==2.24.0 retrying==1.3.3 -six==1.14.0 -SQLAlchemy==1.3.15 +six==1.15.0 +SQLAlchemy==1.3.20 text-unidecode==1.3 -urllib3==1.25.8 -Werkzeug==1.0.0 +toml==0.10.2 +urllib3==1.25.11 +Werkzeug==1.0.1 -e . From 50cd38be2d93b714e146fd4c5abea6cc7a9ffbfe Mon Sep 17 00:00:00 2001 From: Sergey Kuznetsov Date: Sun, 1 Nov 2020 14:27:43 +0300 Subject: [PATCH 024/103] Upgrdae python runtime to 3.8.5 --- CHANGELOG.md | 2 +- runtime.txt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 168b4fb..5a96f88 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -13,7 +13,7 @@ ### Improved -- Upgrade `python` to 3.8.2. +- Upgrade `python` to 3.8.5. - All requirements upgraded to latest version. ### Added diff --git a/runtime.txt b/runtime.txt index 724c203..43b47fb 100644 --- a/runtime.txt +++ b/runtime.txt @@ -1 +1 @@ -python-3.8.2 +python-3.8.5 From dde0df544d5210c6b645dd65f6b6374e71df7e7f Mon Sep 17 00:00:00 2001 From: Sergey Kuznetsov Date: Sun, 1 Nov 2020 14:38:11 +0300 Subject: [PATCH 025/103] Downgrade greenlet package for gevent compability --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 0892963..a515653 100644 --- a/requirements.txt +++ b/requirements.txt @@ -12,7 +12,7 @@ Flask==1.1.2 Flask-Migrate==2.5.3 Flask-SQLAlchemy==2.4.4 gevent==1.5.0 -greenlet==0.4.17 +greenlet==0.4.15 gunicorn==20.0.4 idna==2.10 itsdangerous==1.1.0 From a1e8035aa7c05e23f37e948b45d6362e76e46e7f Mon Sep 17 00:00:00 2001 From: Sergey Kuznetsov Date: Sun, 1 Nov 2020 14:44:41 +0300 Subject: [PATCH 026/103] 302 instead of 404 will be only in production mode --- CHANGELOG.md | 2 +- src/app.py | 30 ++++++++++++++++-------------- 2 files changed, 17 insertions(+), 15 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 5a96f88..6f29e96 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -18,7 +18,7 @@ ### Added -- Web Site: 302 (redirect to Telegram) will be returned instead of 404 (not found page). +- Web Site: 302 (redirect to Telegram) will be returned instead of 404 (not found page), but only in production mode. ### Changed diff --git a/src/app.py b/src/app.py index 986cb07..c1f3801 100644 --- a/src/app.py +++ b/src/app.py @@ -7,7 +7,8 @@ from flask import ( Flask, redirect, - url_for + url_for, + current_app ) from .configs import flask_config @@ -94,16 +95,17 @@ def configure_error_handlers(app: Flask) -> None: """ Configures error handlers. """ - @app.errorhandler(404) - def not_found(error): - """ - We will redirect all requests rather than send "Not Found" - error, because we using web pages only for exceptional cases. - It is expected that all interaction with user should go - through Telegram when possible. - """ - return redirect( - location=app.config["PROJECT_URL_FOR_BOT"], - # temporary, in case if some routes will be added in future - code=302 - ) + if not app.config["DEBUG"]: + @app.errorhandler(404) + def not_found(error): + """ + We will redirect all requests rather than send "Not Found" + error, because we using web pages only for exceptional cases. + It is expected that all interaction with user should go + through Telegram when possible. + """ + return redirect( + location=app.config["PROJECT_URL_FOR_BOT"], + # temporary, in case if some routes will be added in future + code=302 + ) From 9044c57aafcc88c14bda079a02e2f5384c1ccbde Mon Sep 17 00:00:00 2001 From: Sergey Kuznetsov Date: Sun, 1 Nov 2020 16:23:01 +0300 Subject: [PATCH 027/103] Add support for different .env files based on environment --- .gitignore | 3 ++- CHANGELOG.md | 1 + README.md | 2 +- src/configs/flask.py | 29 +++++++++++++++++++++++------ 4 files changed, 27 insertions(+), 8 deletions(-) diff --git a/.gitignore b/.gitignore index 82a2cff..84ffc37 100644 --- a/.gitignore +++ b/.gitignore @@ -13,4 +13,5 @@ venv # Secrets instance -.env +.env* +!.env.example diff --git a/CHANGELOG.md b/CHANGELOG.md index 6f29e96..3cc38bd 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -18,6 +18,7 @@ ### Added +- Support for different env-type files (based on current environment). Initially it was only for production. - Web Site: 302 (redirect to Telegram) will be returned instead of 404 (not found page), but only in production mode. ### Changed diff --git a/README.md b/README.md index bbed7e1..296ce6b 100644 --- a/README.md +++ b/README.md @@ -224,7 +224,7 @@ heroku config:set GUNICORN_WORKER_CONNECTIONS= git checkout -b heroku ``` -7. Make sure `.env` file is created and filled. Remove it from `.gitignore`. Don't forget: don't ever push it anywhere but Heroku. +7. Make sure `.env.production` file is created and filled. Remove it from `.gitignore`. Don't forget: don't ever push it anywhere but Heroku. 8. Add changes for pushing to Heroku: diff --git a/src/configs/flask.py b/src/configs/flask.py index af53de6..a175006 100644 --- a/src/configs/flask.py +++ b/src/configs/flask.py @@ -7,7 +7,25 @@ from dotenv import load_dotenv -load_dotenv() +def load_env(): + config_name = os.getenv("CONFIG_NAME") + file_names = { + "production": ".env.production", + "development": ".env.development", + "testing": ".env.testing" + } + file_name = file_names.get(config_name) + + if (file_name is None): + raise Exception( + "Unable to map configuration name " + "and .env.* files" + ) + + load_dotenv(file_name) + + +load_env() class Config: @@ -33,7 +51,10 @@ class Config: SECRET_KEY = os.getenv("FLASK_SECRET_KEY") # Flask SQLAlchemy - SQLALCHEMY_DATABASE_URI = os.getenv("DATABASE_URL") + SQLALCHEMY_DATABASE_URI = os.getenv( + "DATABASE_URL", + "sqlite:///temp.sqlite" + ) SQLALCHEMY_TRACK_MODIFICATIONS = False # Telegram API @@ -95,16 +116,12 @@ class ProductionConfig(Config): class DevelopmentConfig(Config): DEBUG = True TESTING = False - SECRET_KEY = "q8bjscr0sLmAf50gXRFaIghoS7BvDd4Afxo2RjT3r3E=" - SQLALCHEMY_DATABASE_URI = "sqlite:///development.sqlite" SQLALCHEMY_ECHO = "debug" class TestingConfig(Config): DEBUG = False TESTING = True - SECRET_KEY = "ReHdIY8zGRQUJRTgxo_zeKiv3MjIU-OYBD66GlW9ZKw=" - SQLALCHEMY_DATABASE_URI = "sqlite:///testing.sqlite" config = { From a979244874d64bb6b461e2ac03b572a76bfc0a21 Mon Sep 17 00:00:00 2001 From: Sergey Kuznetsov Date: Wed, 4 Nov 2020 00:28:17 +0300 Subject: [PATCH 028/103] Add rule in flake8 config --- .flake8 | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.flake8 b/.flake8 index 7f41022..e03c87d 100644 --- a/.flake8 +++ b/.flake8 @@ -1,5 +1,8 @@ [flake8] -ignore = F401, W504 +ignore = + F401, + W504, + E261 exclude = # Folders __pycache__ From 10a26162ef0737038612dd231aedd7b3b7f5b818 Mon Sep 17 00:00:00 2001 From: Sergey Kuznetsov Date: Wed, 4 Nov 2020 15:43:43 +0300 Subject: [PATCH 029/103] Add support for redis --- requirements.txt | 2 ++ src/app.py | 23 +++++++++++++++++++++++ src/configs/flask.py | 3 +++ 3 files changed, 28 insertions(+) diff --git a/requirements.txt b/requirements.txt index a515653..09cc1e8 100644 --- a/requirements.txt +++ b/requirements.txt @@ -14,6 +14,7 @@ Flask-SQLAlchemy==2.4.4 gevent==1.5.0 greenlet==0.4.15 gunicorn==20.0.4 +hiredis==1.1.0 idna==2.10 itsdangerous==1.1.0 Jinja2==2.11.2 @@ -33,6 +34,7 @@ python-dateutil==2.8.1 python-dotenv==0.15.0 python-editor==1.0.4 pytz==2020.1 +redis==3.5.3 requests==2.24.0 retrying==1.3.3 six==1.15.0 diff --git a/src/app.py b/src/app.py index c1f3801..87347b9 100644 --- a/src/app.py +++ b/src/app.py @@ -3,6 +3,9 @@ """ import os +from typing import ( + Union +) from flask import ( Flask, @@ -10,6 +13,7 @@ url_for, current_app ) +import redis from .configs import flask_config from .database import db, migrate @@ -22,6 +26,10 @@ ) +# `None` means Redis not enabled +redis_client: Union[redis.Redis, None] = None + + def create_app(config_name: str = None) -> Flask: """ Creates and configures the app. @@ -33,6 +41,7 @@ def create_app(config_name: str = None) -> Flask: configure_blueprints(app) configure_redirects(app) configure_error_handlers(app) + configure_redis(app) return app @@ -109,3 +118,17 @@ def not_found(error): # temporary, in case if some routes will be added in future code=302 ) + + +def configure_redis(app: Flask) -> None: + global redis_client + + redis_server_url = app.config["REDIS_URL"] + + if not redis_server_url: + return + + redis_client = redis.from_url( + redis_server_url, + decode_responses=True + ) diff --git a/src/configs/flask.py b/src/configs/flask.py index a175006..031be82 100644 --- a/src/configs/flask.py +++ b/src/configs/flask.py @@ -57,6 +57,9 @@ class Config: ) SQLALCHEMY_TRACK_MODIFICATIONS = False + # Redis + REDIS_URL = os.getenv("REDIS_URL") + # Telegram API # stop waiting for a Telegram response # after a given number of seconds From 461b9199e6bc1128b9adf92e0accb51d7e8b11e3 Mon Sep 17 00:00:00 2001 From: Sergey Kuznetsov Date: Wed, 4 Nov 2020 23:50:55 +0300 Subject: [PATCH 030/103] Refactoring of command dispatcher --- .../webhook/commands/common/decorators.py | 2 +- .../telegram_bot/webhook/dispatcher.py | 121 ++++++++++++++++++ .../webhook/telegram_interface.py | 29 ----- src/blueprints/telegram_bot/webhook/views.py | 44 +------ 4 files changed, 126 insertions(+), 70 deletions(-) create mode 100644 src/blueprints/telegram_bot/webhook/dispatcher.py diff --git a/src/blueprints/telegram_bot/webhook/commands/common/decorators.py b/src/blueprints/telegram_bot/webhook/commands/common/decorators.py index a0916e8..a7fd347 100644 --- a/src/blueprints/telegram_bot/webhook/commands/common/decorators.py +++ b/src/blueprints/telegram_bot/webhook/commands/common/decorators.py @@ -102,7 +102,7 @@ def wrapper(*args, **kwargs): (user.yandex_disk_token is None) or (not user.yandex_disk_token.have_access_token()) ): - return g.route_to(CommandsNames.YD_AUTH) + return g.direct_dispatch(CommandsNames.YD_AUTH)() return func(*args, **kwargs) diff --git a/src/blueprints/telegram_bot/webhook/dispatcher.py b/src/blueprints/telegram_bot/webhook/dispatcher.py new file mode 100644 index 0000000..c7be4c2 --- /dev/null +++ b/src/blueprints/telegram_bot/webhook/dispatcher.py @@ -0,0 +1,121 @@ +from typing import Union, Callable + +from . import commands +from .commands import CommandsNames +from .telegram_interface import ( + Message as TelegramMessage +) + + +def dispatch(message: TelegramMessage) -> Callable: + """ + Dispatch to handler of a message. + It handles message by different ways: tries to + read the content, tries to guess the command, + tries to implement stateful dialog, tries to pass + most appropriate arguments, and so on. + So, most appropriate handler will be returned + for incoming message. + + :param message: + Incoming Telegram message. + + :returns: + It is guaranteed that most appropriate callable + handler will be returned. It is a handler for + incoming message with already configured arguments, + and you should call this with no arguments + (but you can pass any if you want). + """ + command = message.get_entity_value("bot_command") + + if command is None: + command = guess_bot_command(message) + + handler = direct_dispatch(command) + + def method(*args, **kwargs): + handler(*args, **kwargs) + + return method + + +def direct_dispatch( + command: Union[CommandsNames, str], + fallback: Callable = commands.unknown_handler +) -> Callable: + """ + Direct dispatch to handler of the command. + i.e., it doesn't uses any guessing or stateful chats, + it is just direct route (command_name -> command_handler). + + :param command: + Name of command to dispatch to. + :param fallback: + Fallback handler that will be used in case + if command is unknown. + + :returns: + It is guaranteed that some callable handler will be returned. + It is handler for incoming command and you should call this. + """ + if isinstance(command, CommandsNames): + command = command.value + + routes = { + CommandsNames.START.value: commands.help_handler, + CommandsNames.HELP.value: commands.help_handler, + CommandsNames.ABOUT.value: commands.about_handler, + CommandsNames.SETTINGS.value: commands.settings_handler, + CommandsNames.YD_AUTH.value: commands.yd_auth_handler, + CommandsNames.YD_REVOKE.value: commands.yd_revoke_handler, + CommandsNames.UPLOAD_PHOTO.value: commands.upload_photo_handler, + CommandsNames.UPLOAD_FILE.value: commands.upload_file_handler, + CommandsNames.UPLOAD_AUDIO.value: commands.upload_audio_handler, + CommandsNames.UPLOAD_VIDEO.value: commands.upload_video_handler, + CommandsNames.UPLOAD_VOICE.value: commands.upload_voice_handler, + CommandsNames.UPLOAD_URL.value: commands.upload_url_handler, + CommandsNames.CREATE_FOLDER.value: commands.create_folder_handler, + CommandsNames.PUBLISH.value: commands.publish_handler, + CommandsNames.UNPUBLISH.value: commands.unpublish_handler, + CommandsNames.SPACE.value: commands.space_handler + } + handler = routes.get(command, fallback) + + def method(*args, **kwargs): + handler(*args, **kwargs) + + return method + + +def guess_bot_command( + message: TelegramMessage, + fallback: CommandsNames = CommandsNames.HELP +) -> CommandsNames: + """ + Tries to guess which bot command user + assumed based on content of a message. + + :param fallback: + Fallback command which will be returned if unable to guess. + + :returns: + Guessed bot command based on a message. + """ + command = fallback + raw_data = message.raw_data + + if ("photo" in raw_data): + command = CommandsNames.UPLOAD_PHOTO + elif ("document" in raw_data): + command = CommandsNames.UPLOAD_FILE + elif ("audio" in raw_data): + command = CommandsNames.UPLOAD_AUDIO + elif ("video" in raw_data): + command = CommandsNames.UPLOAD_VIDEO + elif ("voice" in raw_data): + command = CommandsNames.UPLOAD_VOICE + elif (message.get_entity_value("url") is not None): + command = CommandsNames.UPLOAD_URL + + return command diff --git a/src/blueprints/telegram_bot/webhook/telegram_interface.py b/src/blueprints/telegram_bot/webhook/telegram_interface.py index 3495267..88ee8c4 100644 --- a/src/blueprints/telegram_bot/webhook/telegram_interface.py +++ b/src/blueprints/telegram_bot/webhook/telegram_interface.py @@ -4,8 +4,6 @@ Any ) -from .commands import CommandsNames - class User: """ @@ -197,33 +195,6 @@ def get_entity_value( return value - def guess_bot_command(self, default=CommandsNames.HELP) -> str: - """ - Tries to guess which bot command - user assumed based on a message. - - :param default: Default command which will be - returned if unable to guess. - - :returns: Guessed bot command based on a message. - """ - command = default - - if ("photo" in self.raw_data): - command = CommandsNames.UPLOAD_PHOTO - elif ("document" in self.raw_data): - command = CommandsNames.UPLOAD_FILE - elif ("audio" in self.raw_data): - command = CommandsNames.UPLOAD_AUDIO - elif ("video" in self.raw_data): - command = CommandsNames.UPLOAD_VIDEO - elif ("voice" in self.raw_data): - command = CommandsNames.UPLOAD_VOICE - elif (self.get_entity_value("url") is not None): - command = CommandsNames.UPLOAD_URL - - return command - class Request: """ diff --git a/src/blueprints/telegram_bot/webhook/views.py b/src/blueprints/telegram_bot/webhook/views.py index 9b10389..db5d6d0 100644 --- a/src/blueprints/telegram_bot/webhook/views.py +++ b/src/blueprints/telegram_bot/webhook/views.py @@ -5,7 +5,8 @@ ) from src.blueprints.telegram_bot import telegram_bot_blueprint as bp -from . import commands, telegram_interface +from . import telegram_interface +from .dispatcher import dispatch, direct_dispatch @bp.route("/webhook", methods=["POST"]) @@ -39,50 +40,13 @@ def webhook(): g.telegram_message = message g.telegram_user = message.get_user() g.telegram_chat = message.get_chat() - g.route_to = route_command + g.direct_dispatch = direct_dispatch - command = message.get_entity_value("bot_command") - - if (command is None): - command = message.guess_bot_command() - - route_command(command) + dispatch(message)() return make_success_response() -def route_command(command: str) -> None: - """ - Routes command to specific handler. - """ - CommandNames = commands.CommandsNames - - if (isinstance(command, CommandNames)): - command = command.value - - routes = { - CommandNames.START.value: commands.help_handler, - CommandNames.HELP.value: commands.help_handler, - CommandNames.ABOUT.value: commands.about_handler, - CommandNames.SETTINGS.value: commands.settings_handler, - CommandNames.YD_AUTH.value: commands.yd_auth_handler, - CommandNames.YD_REVOKE.value: commands.yd_revoke_handler, - CommandNames.UPLOAD_PHOTO.value: commands.upload_photo_handler, - CommandNames.UPLOAD_FILE.value: commands.upload_file_handler, - CommandNames.UPLOAD_AUDIO.value: commands.upload_audio_handler, - CommandNames.UPLOAD_VIDEO.value: commands.upload_video_handler, - CommandNames.UPLOAD_VOICE.value: commands.upload_voice_handler, - CommandNames.UPLOAD_URL.value: commands.upload_url_handler, - CommandNames.CREATE_FOLDER.value: commands.create_folder_handler, - CommandNames.PUBLISH.value: commands.publish_handler, - CommandNames.UNPUBLISH.value: commands.unpublish_handler, - CommandNames.SPACE.value: commands.space_handler - } - method = routes.get(command, commands.unknown_handler) - - method() - - def make_error_response(): """ Creates error response for Telegram Webhook. From 48f0643550257d0d56b85be0b21056b4a1255aea Mon Sep 17 00:00:00 2001 From: Sergey Kuznetsov Date: Thu, 5 Nov 2020 00:00:56 +0300 Subject: [PATCH 031/103] Refactoring of enum names --- migrations/versions/c6fa89936c27_init.py | 4 +- .../telegram_bot/webhook/commands/__init__.py | 2 +- .../webhook/commands/common/decorators.py | 8 +-- .../webhook/commands/common/names.py | 4 +- .../webhook/commands/create_folder.py | 4 +- .../telegram_bot/webhook/commands/help.py | 30 +++++------ .../telegram_bot/webhook/commands/publish.py | 4 +- .../telegram_bot/webhook/commands/space.py | 2 +- .../telegram_bot/webhook/commands/unknown.py | 4 +- .../webhook/commands/unpublish.py | 4 +- .../telegram_bot/webhook/commands/yd_auth.py | 8 +-- .../webhook/commands/yd_revoke.py | 4 +- .../telegram_bot/webhook/dispatcher.py | 54 +++++++++---------- src/blueprints/telegram_bot/yd_auth/views.py | 4 +- src/database/models/user.py | 8 +-- src/localization/__init__.py | 2 +- src/localization/languages.py | 12 ++--- 17 files changed, 79 insertions(+), 79 deletions(-) diff --git a/migrations/versions/c6fa89936c27_init.py b/migrations/versions/c6fa89936c27_init.py index c676c89..cc70e63 100644 --- a/migrations/versions/c6fa89936c27_init.py +++ b/migrations/versions/c6fa89936c27_init.py @@ -1,7 +1,7 @@ """Init Revision ID: c6fa89936c27 -Revises: +Revises: Create Date: 2020-03-29 13:07:39.579009 """ @@ -24,7 +24,7 @@ def upgrade(): sa.Column('last_update_date', sa.DateTime(), server_default=sa.text('(CURRENT_TIMESTAMP)'), nullable=False), sa.Column('telegram_id', sa.Integer(), nullable=False, comment='Unique ID to identificate user in Telegram'), sa.Column('is_bot', sa.Boolean(), nullable=False, comment='User is bot in Telegram'), - sa.Column('language', sa.Enum('EN', name='supportedlanguages'), nullable=False, comment='Preferred language of user'), + sa.Column('language', sa.Enum('EN', name='supportedlanguage'), nullable=False, comment='Preferred language of user'), sa.Column('group', sa.Enum('USER', 'TESTER', 'ADMIN', name='usergroup'), nullable=False, comment='User rights group'), sa.PrimaryKeyConstraint('id'), sa.UniqueConstraint('telegram_id') diff --git a/src/blueprints/telegram_bot/webhook/commands/__init__.py b/src/blueprints/telegram_bot/webhook/commands/__init__.py index fed196a..3c1e0e6 100644 --- a/src/blueprints/telegram_bot/webhook/commands/__init__.py +++ b/src/blueprints/telegram_bot/webhook/commands/__init__.py @@ -1,4 +1,4 @@ -from .common.names import CommandsNames +from .common.names import CommandName from .unknown import handle as unknown_handler from .help import handle as help_handler from .about import handle as about_handler diff --git a/src/blueprints/telegram_bot/webhook/commands/common/decorators.py b/src/blueprints/telegram_bot/webhook/commands/common/decorators.py index a7fd347..696b9ca 100644 --- a/src/blueprints/telegram_bot/webhook/commands/common/decorators.py +++ b/src/blueprints/telegram_bot/webhook/commands/common/decorators.py @@ -12,9 +12,9 @@ from src.database.models import ( ChatType ) -from src.localization import SupportedLanguages +from src.localization import SupportedLanguage from .responses import cancel_command -from .names import CommandsNames +from .names import CommandName def register_guest(func): @@ -37,7 +37,7 @@ def wrapper(*args, **kwargs): new_user = User( telegram_id=tg_user.id, is_bot=tg_user.is_bot, - language=SupportedLanguages.get(tg_user.language_code) + language=SupportedLanguage.get(tg_user.language_code) ) Chat( telegram_id=tg_chat.id, @@ -102,7 +102,7 @@ def wrapper(*args, **kwargs): (user.yandex_disk_token is None) or (not user.yandex_disk_token.have_access_token()) ): - return g.direct_dispatch(CommandsNames.YD_AUTH)() + return g.direct_dispatch(CommandName.YD_AUTH)() return func(*args, **kwargs) diff --git a/src/blueprints/telegram_bot/webhook/commands/common/names.py b/src/blueprints/telegram_bot/webhook/commands/common/names.py index d9f3da7..6e08f53 100644 --- a/src/blueprints/telegram_bot/webhook/commands/common/names.py +++ b/src/blueprints/telegram_bot/webhook/commands/common/names.py @@ -2,9 +2,9 @@ @unique -class CommandsNames(Enum): +class CommandName(Enum): """ - Commands supported by bot. + Command supported by bot. """ START = "/start" HELP = "/help" diff --git a/src/blueprints/telegram_bot/webhook/commands/create_folder.py b/src/blueprints/telegram_bot/webhook/commands/create_folder.py index a980f2f..f3d8ecc 100644 --- a/src/blueprints/telegram_bot/webhook/commands/create_folder.py +++ b/src/blueprints/telegram_bot/webhook/commands/create_folder.py @@ -15,7 +15,7 @@ YandexAPICreateFolderError, YandexAPIRequestError ) -from . import CommandsNames +from . import CommandName @yd_access_token_required @@ -29,7 +29,7 @@ def handle(): chat = g.db_chat message_text = message.get_text() folder_name = message_text.replace( - CommandsNames.CREATE_FOLDER.value, + CommandName.CREATE_FOLDER.value, "" ).strip() diff --git a/src/blueprints/telegram_bot/webhook/commands/help.py b/src/blueprints/telegram_bot/webhook/commands/help.py index 66ee466..9e71517 100644 --- a/src/blueprints/telegram_bot/webhook/commands/help.py +++ b/src/blueprints/telegram_bot/webhook/commands/help.py @@ -3,7 +3,7 @@ from flask import g, current_app from src.api import telegram -from . import CommandsNames +from . import CommandName def handle(): @@ -26,58 +26,58 @@ def handle(): "\n" f'Maximum size of every upload (except URL) is {file_size_limit_in_mb} MB.' "\n" - f"{CommandsNames.UPLOAD_PHOTO.value} — upload a photo. " + f"{CommandName.UPLOAD_PHOTO.value} — upload a photo. " "Original name will be not saved, quality of photo will be decreased. " "You can send photo without this command." "\n" - f"{CommandsNames.UPLOAD_FILE.value} — upload a file. " + f"{CommandName.UPLOAD_FILE.value} — upload a file. " "Original name will be saved. " "For photos, original quality will be saved. " "You can send file without this command." "\n" - f"{CommandsNames.UPLOAD_AUDIO.value} — upload an audio. " + f"{CommandName.UPLOAD_AUDIO.value} — upload an audio. " "Original name will be saved, original type may be changed. " "You can send audio file without this command." "\n" - f"{CommandsNames.UPLOAD_VIDEO.value} — upload a video. " + f"{CommandName.UPLOAD_VIDEO.value} — upload a video. " "Original name will be not saved, original type may be changed. " "You can send video file without this command." "\n" - f"{CommandsNames.UPLOAD_VOICE.value} — upload a voice. " + f"{CommandName.UPLOAD_VOICE.value} — upload a voice. " "You can send voice file without this command." "\n" - f"{CommandsNames.UPLOAD_URL.value} — upload a file using direct URL. " + f"{CommandName.UPLOAD_URL.value} — upload a file using direct URL. " "Original name will be saved. " "You can send direct URL to a file without this command." "\n" - f"{CommandsNames.PUBLISH.value} — publish a file or folder that already exists. " + f"{CommandName.PUBLISH.value} — publish a file or folder that already exists. " "Send full name of item to publish with this command. " f'Example: {to_code(f"/{yd_upload_default_folder}/files/photo.jpeg")}' "\n" - f"{CommandsNames.UNPUBLISH.value} — unpublish a file or folder that already exists. " + f"{CommandName.UNPUBLISH.value} — unpublish a file or folder that already exists. " "Send full name of item to unpublish with this command. " f'Example: {to_code(f"/{yd_upload_default_folder}/files/photo.jpeg")}' "\n" - f"{CommandsNames.CREATE_FOLDER.value} — create a folder. " + f"{CommandName.CREATE_FOLDER.value} — create a folder. " "Send folder name to create with this command. " "Folder name should starts from root, " f'nested folders should be separated with "{to_code("/")}" character.' "\n" - f"{CommandsNames.SPACE.value} — get information about remaining Yandex.Disk space. " + f"{CommandName.SPACE.value} — get information about remaining Yandex.Disk space. " "\n\n" "Yandex.Disk Access" "\n" - f"{CommandsNames.YD_AUTH.value} — grant me access to your Yandex.Disk" + f"{CommandName.YD_AUTH.value} — grant me access to your Yandex.Disk" "\n" - f"{CommandsNames.YD_REVOKE.value} — revoke my access to your Yandex.Disk" + f"{CommandName.YD_REVOKE.value} — revoke my access to your Yandex.Disk" "\n\n" "Settings" "\n" - f"{CommandsNames.SETTINGS.value} — edit your settings" + f"{CommandName.SETTINGS.value} — edit your settings" "\n\n" "Information" "\n" - f"{CommandsNames.ABOUT.value} — read about me" + f"{CommandName.ABOUT.value} — read about me" ) telegram.send_message( diff --git a/src/blueprints/telegram_bot/webhook/commands/publish.py b/src/blueprints/telegram_bot/webhook/commands/publish.py index 1245e52..b6ee1e5 100644 --- a/src/blueprints/telegram_bot/webhook/commands/publish.py +++ b/src/blueprints/telegram_bot/webhook/commands/publish.py @@ -15,7 +15,7 @@ YandexAPIPublishItemError, YandexAPIRequestError ) -from . import CommandsNames +from . import CommandName @yd_access_token_required @@ -29,7 +29,7 @@ def handle(): chat = g.db_chat message_text = message.get_text() path = message_text.replace( - CommandsNames.PUBLISH.value, + CommandName.PUBLISH.value, "" ).strip() diff --git a/src/blueprints/telegram_bot/webhook/commands/space.py b/src/blueprints/telegram_bot/webhook/commands/space.py index 97b56ca..9b377c3 100644 --- a/src/blueprints/telegram_bot/webhook/commands/space.py +++ b/src/blueprints/telegram_bot/webhook/commands/space.py @@ -19,7 +19,7 @@ get_disk_info, YandexAPIRequestError ) -from . import CommandsNames +from . import CommandName @yd_access_token_required diff --git a/src/blueprints/telegram_bot/webhook/commands/unknown.py b/src/blueprints/telegram_bot/webhook/commands/unknown.py index f1d70b0..2478804 100644 --- a/src/blueprints/telegram_bot/webhook/commands/unknown.py +++ b/src/blueprints/telegram_bot/webhook/commands/unknown.py @@ -1,7 +1,7 @@ from flask import g from src.api import telegram -from . import CommandsNames +from . import CommandName def handle(): @@ -12,6 +12,6 @@ def handle(): chat_id=g.telegram_chat.id, text=( "I don't know this command. " - f"See commands list or type {CommandsNames.HELP.value}" + f"See commands list or type {CommandName.HELP.value}" ) ) diff --git a/src/blueprints/telegram_bot/webhook/commands/unpublish.py b/src/blueprints/telegram_bot/webhook/commands/unpublish.py index 80457d3..32afc57 100644 --- a/src/blueprints/telegram_bot/webhook/commands/unpublish.py +++ b/src/blueprints/telegram_bot/webhook/commands/unpublish.py @@ -15,7 +15,7 @@ YandexAPIUnpublishItemError, YandexAPIRequestError ) -from . import CommandsNames +from . import CommandName @yd_access_token_required @@ -29,7 +29,7 @@ def handle(): chat = g.db_chat message_text = message.get_text() path = message_text.replace( - CommandsNames.UNPUBLISH.value, + CommandName.UNPUBLISH.value, "" ).strip() diff --git a/src/blueprints/telegram_bot/webhook/commands/yd_auth.py b/src/blueprints/telegram_bot/webhook/commands/yd_auth.py index bcefc4d..2d16440 100644 --- a/src/blueprints/telegram_bot/webhook/commands/yd_auth.py +++ b/src/blueprints/telegram_bot/webhook/commands/yd_auth.py @@ -23,7 +23,7 @@ request_private_chat, cancel_command ) -from . import CommandsNames +from . import CommandName @register_guest @@ -61,7 +61,7 @@ def handle(): "You already grant me access to your Yandex.Disk." "\n" "You can revoke my access with " - f"{CommandsNames.YD_REVOKE.value}" + f"{CommandName.YD_REVOKE.value}" ) ) @@ -91,7 +91,7 @@ def handle(): f"on {date} at {time} {timezone}." "\n\n" "If it wasn't you, you can detach this access with " - f"{CommandsNames.YD_REVOKE.value}" + f"{CommandName.YD_REVOKE.value}" ) ) @@ -157,7 +157,7 @@ def handle(): "\n" "Yes! I'm getting access only to your Yandex.Disk, " "not to your account. You can revoke my access at any time with " - f"{CommandsNames.YD_REVOKE.value} or in your " + f"{CommandName.YD_REVOKE.value} or in your " 'Yandex Profile. ' "By the way, i'm " f'open-source ' # noqa diff --git a/src/blueprints/telegram_bot/webhook/commands/yd_revoke.py b/src/blueprints/telegram_bot/webhook/commands/yd_revoke.py index ff113b0..d209311 100644 --- a/src/blueprints/telegram_bot/webhook/commands/yd_revoke.py +++ b/src/blueprints/telegram_bot/webhook/commands/yd_revoke.py @@ -5,7 +5,7 @@ from src.blueprints.utils import get_current_datetime from .common.decorators import get_db_data from .common.responses import request_private_chat -from . import CommandsNames +from . import CommandName @get_db_data @@ -32,7 +32,7 @@ def handle(): text=( "You don't granted me access to your Yandex.Disk." "\n" - f"You can do that with {CommandsNames.YD_AUTH.value}" + f"You can do that with {CommandName.YD_AUTH.value}" ) ) diff --git a/src/blueprints/telegram_bot/webhook/dispatcher.py b/src/blueprints/telegram_bot/webhook/dispatcher.py index c7be4c2..c2fb319 100644 --- a/src/blueprints/telegram_bot/webhook/dispatcher.py +++ b/src/blueprints/telegram_bot/webhook/dispatcher.py @@ -1,7 +1,7 @@ from typing import Union, Callable from . import commands -from .commands import CommandsNames +from .commands import CommandName from .telegram_interface import ( Message as TelegramMessage ) @@ -41,7 +41,7 @@ def method(*args, **kwargs): def direct_dispatch( - command: Union[CommandsNames, str], + command: Union[CommandName, str], fallback: Callable = commands.unknown_handler ) -> Callable: """ @@ -59,26 +59,26 @@ def direct_dispatch( It is guaranteed that some callable handler will be returned. It is handler for incoming command and you should call this. """ - if isinstance(command, CommandsNames): + if isinstance(command, CommandName): command = command.value routes = { - CommandsNames.START.value: commands.help_handler, - CommandsNames.HELP.value: commands.help_handler, - CommandsNames.ABOUT.value: commands.about_handler, - CommandsNames.SETTINGS.value: commands.settings_handler, - CommandsNames.YD_AUTH.value: commands.yd_auth_handler, - CommandsNames.YD_REVOKE.value: commands.yd_revoke_handler, - CommandsNames.UPLOAD_PHOTO.value: commands.upload_photo_handler, - CommandsNames.UPLOAD_FILE.value: commands.upload_file_handler, - CommandsNames.UPLOAD_AUDIO.value: commands.upload_audio_handler, - CommandsNames.UPLOAD_VIDEO.value: commands.upload_video_handler, - CommandsNames.UPLOAD_VOICE.value: commands.upload_voice_handler, - CommandsNames.UPLOAD_URL.value: commands.upload_url_handler, - CommandsNames.CREATE_FOLDER.value: commands.create_folder_handler, - CommandsNames.PUBLISH.value: commands.publish_handler, - CommandsNames.UNPUBLISH.value: commands.unpublish_handler, - CommandsNames.SPACE.value: commands.space_handler + CommandName.START.value: commands.help_handler, + CommandName.HELP.value: commands.help_handler, + CommandName.ABOUT.value: commands.about_handler, + CommandName.SETTINGS.value: commands.settings_handler, + CommandName.YD_AUTH.value: commands.yd_auth_handler, + CommandName.YD_REVOKE.value: commands.yd_revoke_handler, + CommandName.UPLOAD_PHOTO.value: commands.upload_photo_handler, + CommandName.UPLOAD_FILE.value: commands.upload_file_handler, + CommandName.UPLOAD_AUDIO.value: commands.upload_audio_handler, + CommandName.UPLOAD_VIDEO.value: commands.upload_video_handler, + CommandName.UPLOAD_VOICE.value: commands.upload_voice_handler, + CommandName.UPLOAD_URL.value: commands.upload_url_handler, + CommandName.CREATE_FOLDER.value: commands.create_folder_handler, + CommandName.PUBLISH.value: commands.publish_handler, + CommandName.UNPUBLISH.value: commands.unpublish_handler, + CommandName.SPACE.value: commands.space_handler } handler = routes.get(command, fallback) @@ -90,8 +90,8 @@ def method(*args, **kwargs): def guess_bot_command( message: TelegramMessage, - fallback: CommandsNames = CommandsNames.HELP -) -> CommandsNames: + fallback: CommandName = CommandName.HELP +) -> CommandName: """ Tries to guess which bot command user assumed based on content of a message. @@ -106,16 +106,16 @@ def guess_bot_command( raw_data = message.raw_data if ("photo" in raw_data): - command = CommandsNames.UPLOAD_PHOTO + command = CommandName.UPLOAD_PHOTO elif ("document" in raw_data): - command = CommandsNames.UPLOAD_FILE + command = CommandName.UPLOAD_FILE elif ("audio" in raw_data): - command = CommandsNames.UPLOAD_AUDIO + command = CommandName.UPLOAD_AUDIO elif ("video" in raw_data): - command = CommandsNames.UPLOAD_VIDEO + command = CommandName.UPLOAD_VIDEO elif ("voice" in raw_data): - command = CommandsNames.UPLOAD_VOICE + command = CommandName.UPLOAD_VOICE elif (message.get_entity_value("url") is not None): - command = CommandsNames.UPLOAD_URL + command = CommandName.UPLOAD_URL return command diff --git a/src/blueprints/telegram_bot/yd_auth/views.py b/src/blueprints/telegram_bot/yd_auth/views.py index a5803c5..aa33e37 100644 --- a/src/blueprints/telegram_bot/yd_auth/views.py +++ b/src/blueprints/telegram_bot/yd_auth/views.py @@ -18,7 +18,7 @@ from src.blueprints.utils import ( get_current_datetime ) -from src.blueprints.telegram_bot.webhook.commands import CommandsNames +from src.blueprints.telegram_bot.webhook.commands import CommandName from .exceptions import ( InvalidCredentials, LinkExpired, @@ -152,7 +152,7 @@ def handle_success(): f"on {date} at {time} {timezone}." "\n\n" "If it wasn't you, then detach this access with " - f"{CommandsNames.YD_REVOKE.value}" + f"{CommandName.YD_REVOKE.value}" ) ) diff --git a/src/database/models/user.py b/src/database/models/user.py index e1f7e8f..742b1ee 100644 --- a/src/database/models/user.py +++ b/src/database/models/user.py @@ -3,7 +3,7 @@ from sqlalchemy.sql import func from src.database import db -from src.localization import SupportedLanguages +from src.localization import SupportedLanguage @unique @@ -55,8 +55,8 @@ class User(db.Model): comment="User is bot in Telegram" ) language = db.Column( - db.Enum(SupportedLanguages), - default=SupportedLanguages.EN, + db.Enum(SupportedLanguage), + default=SupportedLanguage.EN, nullable=False, comment="Preferred language of user" ) @@ -112,7 +112,7 @@ def create_fake(): step=1 ) result.is_bot = (fake.pyint() % 121 == 0) - result.language = fake.random_element(list(SupportedLanguages)) + result.language = fake.random_element(list(SupportedLanguage)) result.group = ( fake.random_element(list(UserGroup)) if ( random_number % 20 == 0 diff --git a/src/localization/__init__.py b/src/localization/__init__.py index 309ef19..c8230fb 100644 --- a/src/localization/__init__.py +++ b/src/localization/__init__.py @@ -1 +1 @@ -from .languages import SupportedLanguages +from .languages import SupportedLanguage diff --git a/src/localization/languages.py b/src/localization/languages.py index d7f3bd9..3b3ed06 100644 --- a/src/localization/languages.py +++ b/src/localization/languages.py @@ -2,9 +2,9 @@ @unique -class SupportedLanguages(IntEnum): +class SupportedLanguage(IntEnum): """ - Languages supported by app. + Language supported by the app. """ EN = 1 @@ -18,9 +18,9 @@ def get(ietf_tag: str) -> int: """ ietf_tag = ietf_tag.lower() languages = { - "en": SupportedLanguages.EN, - "en-us": SupportedLanguages.EN, - "en-gb": SupportedLanguages.EN + "en": SupportedLanguage.EN, + "en-us": SupportedLanguage.EN, + "en-gb": SupportedLanguage.EN } - return languages.get(ietf_tag, SupportedLanguages.EN) + return languages.get(ietf_tag, SupportedLanguage.EN) From 2d3a720299164fe212d77da3a91df94c6a9d33f2 Mon Sep 17 00:00:00 2001 From: Sergey Kuznetsov Date: Thu, 5 Nov 2020 12:26:00 +0300 Subject: [PATCH 032/103] Refactoring of configuration of flask extensions --- manage.py | 2 +- src/app.py | 43 ++++------- .../webhook/commands/common/decorators.py | 2 +- .../telegram_bot/webhook/commands/yd_auth.py | 2 +- .../webhook/commands/yd_revoke.py | 2 +- src/blueprints/telegram_bot/yd_auth/views.py | 2 +- src/database/__init__.py | 2 - src/database/database.py | 13 ---- src/database/migrate.py | 10 --- src/database/models/chat.py | 2 +- src/database/models/user.py | 2 +- src/database/models/yandex_disk_token.py | 2 +- src/extensions.py | 77 +++++++++++++++++++ 13 files changed, 101 insertions(+), 60 deletions(-) delete mode 100644 src/database/database.py delete mode 100644 src/database/migrate.py create mode 100644 src/extensions.py diff --git a/manage.py b/manage.py index 79e5a96..b085ba3 100644 --- a/manage.py +++ b/manage.py @@ -4,8 +4,8 @@ from sqlalchemy.exc import IntegrityError from src.app import create_app +from src.extensions import db from src.database import ( - db, User, Chat, YandexDiskToken, diff --git a/src/app.py b/src/app.py index 87347b9..d9b1446 100644 --- a/src/app.py +++ b/src/app.py @@ -3,9 +3,6 @@ """ import os -from typing import ( - Union -) from flask import ( Flask, @@ -13,10 +10,13 @@ url_for, current_app ) -import redis from .configs import flask_config -from .database import db, migrate +from .extensions import ( + db, + migrate, + redis_client +) from .blueprints import ( telegram_bot_blueprint, legal_blueprint @@ -24,10 +24,8 @@ from .blueprints.utils import ( absolute_url_for ) - - -# `None` means Redis not enabled -redis_client: Union[redis.Redis, None] = None +# we need to import every model in order Migrate knows them +from .database.models import * # noqa: F403 def create_app(config_name: str = None) -> Flask: @@ -37,11 +35,10 @@ def create_app(config_name: str = None) -> Flask: app = Flask(__name__) configure_app(app, config_name) - configure_db(app) + configure_extensions(app) configure_blueprints(app) configure_redirects(app) configure_error_handlers(app) - configure_redis(app) return app @@ -58,13 +55,19 @@ def configure_app(app: Flask, config_name: str = None) -> None: app.config.from_object(config) -def configure_db(app: Flask) -> None: +def configure_extensions(app: Flask) -> None: """ - Configures database. + Configures Flask extensions. """ + # Database db.init_app(app) + + # Migration migrate.init_app(app, db) + # Redis + redis_client.init_app(app) + def configure_blueprints(app: Flask) -> None: """ @@ -118,17 +121,3 @@ def not_found(error): # temporary, in case if some routes will be added in future code=302 ) - - -def configure_redis(app: Flask) -> None: - global redis_client - - redis_server_url = app.config["REDIS_URL"] - - if not redis_server_url: - return - - redis_client = redis.from_url( - redis_server_url, - decode_responses=True - ) diff --git a/src/blueprints/telegram_bot/webhook/commands/common/decorators.py b/src/blueprints/telegram_bot/webhook/commands/common/decorators.py index 696b9ca..4c80324 100644 --- a/src/blueprints/telegram_bot/webhook/commands/common/decorators.py +++ b/src/blueprints/telegram_bot/webhook/commands/common/decorators.py @@ -2,8 +2,8 @@ from flask import g +from src.extensions import db from src.database import ( - db, User, UserQuery, Chat, diff --git a/src/blueprints/telegram_bot/webhook/commands/yd_auth.py b/src/blueprints/telegram_bot/webhook/commands/yd_auth.py index 2d16440..cd1c7b5 100644 --- a/src/blueprints/telegram_bot/webhook/commands/yd_auth.py +++ b/src/blueprints/telegram_bot/webhook/commands/yd_auth.py @@ -6,8 +6,8 @@ ) import jwt +from src.extensions import db from src.database import ( - db, YandexDiskToken ) from src.api import telegram, yandex diff --git a/src/blueprints/telegram_bot/webhook/commands/yd_revoke.py b/src/blueprints/telegram_bot/webhook/commands/yd_revoke.py index d209311..e55b0bc 100644 --- a/src/blueprints/telegram_bot/webhook/commands/yd_revoke.py +++ b/src/blueprints/telegram_bot/webhook/commands/yd_revoke.py @@ -1,6 +1,6 @@ from flask import g -from src.database import db +from src.extensions import db from src.api import telegram from src.blueprints.utils import get_current_datetime from .common.decorators import get_db_data diff --git a/src/blueprints/telegram_bot/yd_auth/views.py b/src/blueprints/telegram_bot/yd_auth/views.py index aa33e37..0ddcead 100644 --- a/src/blueprints/telegram_bot/yd_auth/views.py +++ b/src/blueprints/telegram_bot/yd_auth/views.py @@ -8,8 +8,8 @@ ) import jwt +from src.extensions import db from src.database import ( - db, UserQuery, ChatQuery ) diff --git a/src/database/__init__.py b/src/database/__init__.py index ee22204..54e85fb 100644 --- a/src/database/__init__.py +++ b/src/database/__init__.py @@ -1,5 +1,3 @@ -from .database import db -from .migrate import migrate from .models import ( User, Chat, diff --git a/src/database/database.py b/src/database/database.py deleted file mode 100644 index 6d947de..0000000 --- a/src/database/database.py +++ /dev/null @@ -1,13 +0,0 @@ -from sqlalchemy.pool import NullPool - -from flask_sqlalchemy import SQLAlchemy - - -db = SQLAlchemy( - engine_options={ - # pooling will be disabled until that - # https://stackoverflow.com/questions/61197228/flask-gunicorn-gevent-sqlalchemy-postgresql-too-many-connections - # will be resolved - "poolclass": NullPool - } -) diff --git a/src/database/migrate.py b/src/database/migrate.py deleted file mode 100644 index 111e719..0000000 --- a/src/database/migrate.py +++ /dev/null @@ -1,10 +0,0 @@ -from flask_migrate import Migrate - -# we need to import every model in order Migrate knows them -from .models import * # noqa - - -migrate = Migrate( - compare_type=True, - render_as_batch=True -) diff --git a/src/database/models/chat.py b/src/database/models/chat.py index cc8350c..0f3f72d 100644 --- a/src/database/models/chat.py +++ b/src/database/models/chat.py @@ -1,6 +1,6 @@ from enum import IntEnum, unique -from src.database import db +from src.extensions import db @unique diff --git a/src/database/models/user.py b/src/database/models/user.py index 742b1ee..44c46c8 100644 --- a/src/database/models/user.py +++ b/src/database/models/user.py @@ -2,7 +2,7 @@ from sqlalchemy.sql import func -from src.database import db +from src.extensions import db from src.localization import SupportedLanguage diff --git a/src/database/models/yandex_disk_token.py b/src/database/models/yandex_disk_token.py index 8f4cc43..2f2652e 100644 --- a/src/database/models/yandex_disk_token.py +++ b/src/database/models/yandex_disk_token.py @@ -8,7 +8,7 @@ InvalidToken as InvalidTokenFernetError ) -from src.database import db +from src.extensions import db class YandexDiskToken(db.Model): diff --git a/src/extensions.py b/src/extensions.py new file mode 100644 index 0000000..49a4ed3 --- /dev/null +++ b/src/extensions.py @@ -0,0 +1,77 @@ +""" +Flask extensions that used by the app. +They already preconfigured and should be just +initialized by the app. Optionally, at initialization +stage you can provide additional configuration. +These extensions extracted in separate module in order +to avoid circular imports. +""" + +from typing import ( + Union +) + +from flask import Flask +from flask_sqlalchemy import SQLAlchemy +from flask_migrate import Migrate +from sqlalchemy.pool import NullPool +import redis + + +# Database + +db = SQLAlchemy( + engine_options={ + # pooling will be disabled until that + # https://stackoverflow.com/questions/61197228/flask-gunicorn-gevent-sqlalchemy-postgresql-too-many-connections + # will be resolved + "poolclass": NullPool + } +) + + +# Migration + +migrate = Migrate( + compare_type=True, + render_as_batch=True +) + + +# Redis + +class FlaskRedis: + def __init__(self): + self._redis_client = None + + def __getattr__(self, name): + return getattr(self._redis_client, name) + + def __getitem__(self, name): + return self._redis_client[name] + + def __setitem__(self, name, value): + self._redis_client[name] = value + + def __delitem__(self, name): + del self._redis_client[name] + + @property + def is_enabled(self) -> bool: + return (self._redis_client is not None) + + def init_app(self, app: Flask, **kwargs) -> None: + self._redis_client = None + redis_server_url = app.config.get("REDIS_URL") + + if not redis_server_url: + return + + self._redis_client = redis.from_url( + redis_server_url, + decode_responses=True, + **kwargs + ) + + +redis_client: Union[redis.Redis, FlaskRedis] = FlaskRedis() From 7df483e66fc46ad87586124fa2f18e4ec0d80ba5 Mon Sep 17 00:00:00 2001 From: Sergey Kuznetsov Date: Wed, 11 Nov 2020 18:21:57 +0300 Subject: [PATCH 033/103] Add debug config for VSCode --- .vscode/launch.json | 26 ++++++++++++++++++++++++++ CHANGELOG.md | 1 + 2 files changed, 27 insertions(+) create mode 100644 .vscode/launch.json diff --git a/.vscode/launch.json b/.vscode/launch.json new file mode 100644 index 0000000..c048bea --- /dev/null +++ b/.vscode/launch.json @@ -0,0 +1,26 @@ +{ + "version": "0.2.0", + "configurations": [ + { + "name": "Python: Flask", + "type": "python", + "request": "launch", + "module": "flask", + "env": { + "FLASK_APP": "wsgi.py", + "FLASK_ENV": "development", + "FLASK_DEBUG": "0", + "CONFIG_NAME": "development" + }, + "args": [ + "run", + "--no-debugger", + "--no-reload", + "--port", + "8000" + ], + "jinja": true, + "console": "integratedTerminal" + } + ] +} diff --git a/CHANGELOG.md b/CHANGELOG.md index 3cc38bd..08175ff 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -20,6 +20,7 @@ - Support for different env-type files (based on current environment). Initially it was only for production. - Web Site: 302 (redirect to Telegram) will be returned instead of 404 (not found page), but only in production mode. +- Debug configuration for VSCode. ### Changed From ae1e52d45d47da236fffac32ae3a8a63f082ebbc Mon Sep 17 00:00:00 2001 From: Sergey Kuznetsov Date: Thu, 12 Nov 2020 16:12:48 +0300 Subject: [PATCH 034/103] Add stateful chat module and support for it --- CHANGELOG.md | 1 + .../telegram_bot/webhook/commands/__init__.py | 24 + .../telegram_bot/webhook/commands/about.py | 2 +- .../webhook/commands/create_folder.py | 2 +- .../telegram_bot/webhook/commands/help.py | 2 +- .../telegram_bot/webhook/commands/publish.py | 2 +- .../telegram_bot/webhook/commands/settings.py | 2 +- .../telegram_bot/webhook/commands/space.py | 2 +- .../telegram_bot/webhook/commands/unknown.py | 2 +- .../webhook/commands/unpublish.py | 2 +- .../telegram_bot/webhook/commands/upload.py | 14 +- .../telegram_bot/webhook/commands/yd_auth.py | 2 +- .../webhook/commands/yd_revoke.py | 2 +- .../telegram_bot/webhook/dispatcher.py | 210 +++++++- .../telegram_bot/webhook/dispatcher_events.py | 67 +++ .../telegram_bot/webhook/stateful_chat.py | 507 ++++++++++++++++++ .../webhook/telegram_interface.py | 61 +++ src/blueprints/telegram_bot/webhook/views.py | 4 +- 18 files changed, 867 insertions(+), 41 deletions(-) create mode 100644 src/blueprints/telegram_bot/webhook/dispatcher_events.py create mode 100644 src/blueprints/telegram_bot/webhook/stateful_chat.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 08175ff..abea2bf 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -18,6 +18,7 @@ ### Added +- Stateful chat support. Now bot can store custom user data (in different namespaces: user, chat, user in chat); determine Telegram message types; register single use handler (call once for message) with optional timeout for types of message; subscribe handlers with optional timeout for types of messages. - Support for different env-type files (based on current environment). Initially it was only for production. - Web Site: 302 (redirect to Telegram) will be returned instead of 404 (not found page), but only in production mode. - Debug configuration for VSCode. diff --git a/src/blueprints/telegram_bot/webhook/commands/__init__.py b/src/blueprints/telegram_bot/webhook/commands/__init__.py index 3c1e0e6..ab4f391 100644 --- a/src/blueprints/telegram_bot/webhook/commands/__init__.py +++ b/src/blueprints/telegram_bot/webhook/commands/__init__.py @@ -1,3 +1,27 @@ +""" +How to create handler for the dispatcher: +- you should provide one function. This function +will be called to handle an incoming Telegram message +- dispatcher handles any handler exceptions, so, if +any unexpected error occurs you can raise the exception. +Sure, you can raise your own exception for nice debug +- edit dispatcher module in order to register your handler +- handler should accept only `(*args, **kwargs)` arguments +- dispatcher passes in `**kwargs`: `route_source: RouteSource`, +`message_events: Set[str]` (`str` it is `DispatcherEvent` values), +`user_id: int`, `chat_id: int`, `message: TelegramMessage`. +These values can have `None` value, so, check for it before using. +For another values in `*args` and `**kwargs` see documentation of +functions from call stack of dispatcher function. + +How to create stateful chat: +- use `set_disposable_handler`, `subscribe_handler`, +`unsubcribe_handler`, `set/get/delete_user/user_chat/chat_data` +- if you want to provide Enum, then provide value of that enum, +not object directly +""" + + from .common.names import CommandName from .unknown import handle as unknown_handler from .help import handle as help_handler diff --git a/src/blueprints/telegram_bot/webhook/commands/about.py b/src/blueprints/telegram_bot/webhook/commands/about.py index ab7e8b4..ad9d87c 100644 --- a/src/blueprints/telegram_bot/webhook/commands/about.py +++ b/src/blueprints/telegram_bot/webhook/commands/about.py @@ -4,7 +4,7 @@ from src.blueprints.utils import absolute_url_for -def handle(): +def handle(*args, **kwargs): """ Handles `/about` command. """ diff --git a/src/blueprints/telegram_bot/webhook/commands/create_folder.py b/src/blueprints/telegram_bot/webhook/commands/create_folder.py index f3d8ecc..c8ba075 100644 --- a/src/blueprints/telegram_bot/webhook/commands/create_folder.py +++ b/src/blueprints/telegram_bot/webhook/commands/create_folder.py @@ -20,7 +20,7 @@ @yd_access_token_required @get_db_data -def handle(): +def handle(*args, **kwargs): """ Handles `/create_folder` command. """ diff --git a/src/blueprints/telegram_bot/webhook/commands/help.py b/src/blueprints/telegram_bot/webhook/commands/help.py index 9e71517..3897bd1 100644 --- a/src/blueprints/telegram_bot/webhook/commands/help.py +++ b/src/blueprints/telegram_bot/webhook/commands/help.py @@ -6,7 +6,7 @@ from . import CommandName -def handle(): +def handle(*args, **kwargs): """ Handles `/help` command. """ diff --git a/src/blueprints/telegram_bot/webhook/commands/publish.py b/src/blueprints/telegram_bot/webhook/commands/publish.py index b6ee1e5..b86b62e 100644 --- a/src/blueprints/telegram_bot/webhook/commands/publish.py +++ b/src/blueprints/telegram_bot/webhook/commands/publish.py @@ -20,7 +20,7 @@ @yd_access_token_required @get_db_data -def handle(): +def handle(*args, **kwargs): """ Handles `/publish` command. """ diff --git a/src/blueprints/telegram_bot/webhook/commands/settings.py b/src/blueprints/telegram_bot/webhook/commands/settings.py index a9d0f66..705757a 100644 --- a/src/blueprints/telegram_bot/webhook/commands/settings.py +++ b/src/blueprints/telegram_bot/webhook/commands/settings.py @@ -12,7 +12,7 @@ @register_guest @get_db_data -def handle(): +def handle(*args, **kwargs): """ Handles `/settings` command. """ diff --git a/src/blueprints/telegram_bot/webhook/commands/space.py b/src/blueprints/telegram_bot/webhook/commands/space.py index 9b377c3..874a650 100644 --- a/src/blueprints/telegram_bot/webhook/commands/space.py +++ b/src/blueprints/telegram_bot/webhook/commands/space.py @@ -24,7 +24,7 @@ @yd_access_token_required @get_db_data -def handle(): +def handle(*args, **kwargs): """ Handles `/publish` command. """ diff --git a/src/blueprints/telegram_bot/webhook/commands/unknown.py b/src/blueprints/telegram_bot/webhook/commands/unknown.py index 2478804..8e22365 100644 --- a/src/blueprints/telegram_bot/webhook/commands/unknown.py +++ b/src/blueprints/telegram_bot/webhook/commands/unknown.py @@ -4,7 +4,7 @@ from . import CommandName -def handle(): +def handle(*args, **kwargs): """ Handles unknown command. """ diff --git a/src/blueprints/telegram_bot/webhook/commands/unpublish.py b/src/blueprints/telegram_bot/webhook/commands/unpublish.py index 32afc57..bbdd44f 100644 --- a/src/blueprints/telegram_bot/webhook/commands/unpublish.py +++ b/src/blueprints/telegram_bot/webhook/commands/unpublish.py @@ -20,7 +20,7 @@ @yd_access_token_required @get_db_data -def handle(): +def handle(*args, **kwargs): """ Handles `/unpublish` command. """ diff --git a/src/blueprints/telegram_bot/webhook/commands/upload.py b/src/blueprints/telegram_bot/webhook/commands/upload.py index 746dc73..bc70231 100644 --- a/src/blueprints/telegram_bot/webhook/commands/upload.py +++ b/src/blueprints/telegram_bot/webhook/commands/upload.py @@ -60,7 +60,7 @@ def __init__(self) -> None: @staticmethod @abstractmethod - def handle() -> None: + def handle(*args, **kwargs) -> None: """ Starts uploading process. """ @@ -337,7 +337,7 @@ class PhotoHandler(AttachmentHandler): Handles uploading of photo. """ @staticmethod - def handle(): + def handle(*args, **kwargs): handler = PhotoHandler() handler.upload() @@ -377,7 +377,7 @@ class FileHandler(AttachmentHandler): Handles uploading of file. """ @staticmethod - def handle(): + def handle(*args, **kwargs): handler = FileHandler() handler.upload() @@ -403,7 +403,7 @@ class AudioHandler(AttachmentHandler): Handles uploading of audio. """ @staticmethod - def handle(): + def handle(*args, **kwargs): handler = AudioHandler() handler.upload() @@ -441,7 +441,7 @@ class VideoHandler(AttachmentHandler): Handles uploading of video. """ @staticmethod - def handle(): + def handle(*args, **kwargs): handler = VideoHandler() handler.upload() @@ -463,7 +463,7 @@ class VoiceHandler(AttachmentHandler): Handles uploading of voice. """ @staticmethod - def handle(): + def handle(*args, **kwargs): handler = VoiceHandler() handler.upload() @@ -485,7 +485,7 @@ class URLHandler(AttachmentHandler): Handles uploading of direct URL to file. """ @staticmethod - def handle(): + def handle(*args, **kwargs): handler = URLHandler() handler.upload() diff --git a/src/blueprints/telegram_bot/webhook/commands/yd_auth.py b/src/blueprints/telegram_bot/webhook/commands/yd_auth.py index cd1c7b5..5849a4b 100644 --- a/src/blueprints/telegram_bot/webhook/commands/yd_auth.py +++ b/src/blueprints/telegram_bot/webhook/commands/yd_auth.py @@ -28,7 +28,7 @@ @register_guest @get_db_data -def handle(): +def handle(*args, **kwargs): """ Handles `/yandex_disk_authorization` command. diff --git a/src/blueprints/telegram_bot/webhook/commands/yd_revoke.py b/src/blueprints/telegram_bot/webhook/commands/yd_revoke.py index e55b0bc..fd26bf2 100644 --- a/src/blueprints/telegram_bot/webhook/commands/yd_revoke.py +++ b/src/blueprints/telegram_bot/webhook/commands/yd_revoke.py @@ -9,7 +9,7 @@ @get_db_data -def handle(): +def handle(*args, **kwargs): """ Handles `/yandex_disk_revoke` command. diff --git a/src/blueprints/telegram_bot/webhook/dispatcher.py b/src/blueprints/telegram_bot/webhook/dispatcher.py index c2fb319..dd35e3f 100644 --- a/src/blueprints/telegram_bot/webhook/dispatcher.py +++ b/src/blueprints/telegram_bot/webhook/dispatcher.py @@ -1,41 +1,135 @@ -from typing import Union, Callable +from typing import ( + Union, + Callable, + Set +) +from collections import deque +from src.extensions import redis_client from . import commands from .commands import CommandName +from .dispatcher_events import ( + DispatcherEvent, + RouteSource +) from .telegram_interface import ( Message as TelegramMessage ) +from .stateful_chat import ( + get_disposable_handler, + delete_disposable_handler, + get_subscribed_handlers +) -def dispatch(message: TelegramMessage) -> Callable: +def intellectual_dispatch( + message: TelegramMessage +) -> Callable: """ - Dispatch to handler of a message. - It handles message by different ways: tries to - read the content, tries to guess the command, - tries to implement stateful dialog, tries to pass - most appropriate arguments, and so on. - So, most appropriate handler will be returned - for incoming message. + Intellectual dispatch to handlers of a message. + Provides support for stateful chat (if Redis is enabled). + + Priority of handlers: + 1) if disposable handler exists and message events matches, + then only that handler will be called and after removed + 2) if subscribed handlers exists, then only ones with events + matched to message events will be called. If nothing is matched, + then forwarding to № 3 + 3) attempt to get first `bot_command` entity from message. + If nothing found, then forwarding to № 4 + 4) guessing of command that user assumed based on + content of message + + Events matching: + - if at least one event matched, then that handler will be + marked as "matched". + + If stateful chat not enabled, then № 1 and № 2 will be skipped. + + Note: there can be multiple handlers picked for single message. + Order of execution not determined. :param message: Incoming Telegram message. :returns: - It is guaranteed that most appropriate callable - handler will be returned. It is a handler for - incoming message with already configured arguments, - and you should call this with no arguments - (but you can pass any if you want). + It is guaranteed that most appropriate callable handler that + not raises an error will be returned. Function arguments already + configured, but you can also provided your own through `*args` + and `**kwargs`. You should call this function in order to run + handlers (there can be multiple handlers in one return function). """ - command = message.get_entity_value("bot_command") + user_id = message.get_user().id + chat_id = message.get_chat().id + stateful_chat_is_enabled = redis_client.is_enabled + disposable_handler = None + subscribed_handlers = None + + if stateful_chat_is_enabled: + disposable_handler = get_disposable_handler(user_id, chat_id) + subscribed_handlers = get_subscribed_handlers(user_id, chat_id) + + message_events = ( + detect_events(message) if ( + disposable_handler or + subscribed_handlers + ) else None + ) + handler_names = deque() + route_source = None - if command is None: - command = guess_bot_command(message) + if disposable_handler: + match = match_events( + message_events, + disposable_handler["events"] + ) - handler = direct_dispatch(command) + if match: + route_source = RouteSource.DISPOSABLE_HANDLER + handler_names.append(disposable_handler["name"]) + delete_disposable_handler(user_id, chat_id) + + if ( + subscribed_handlers and + not handler_names + ): + for handler in subscribed_handlers: + match = match_events( + message_events, + handler["events"] + ) + + if match: + route_source = RouteSource.SUBSCRIBED_HANDLER + handler_names.append(handler["name"]) + + if not handler_names: + command = message.get_entity_value("bot_command") + + if command: + route_source = RouteSource.DIRECT_COMMAND + handler_names.append(command) + + if not handler_names: + route_source = RouteSource.GUESSED_COMMAND + handler_names.append(guess_bot_command(message)) def method(*args, **kwargs): - handler(*args, **kwargs) + for handler_name in handler_names: + handler_method = direct_dispatch(handler_name) + + try: + handler_method( + *args, + **kwargs, + user_id=user_id, + chat_id=chat_id, + message=message, + route_source=route_source, + message_events=message_events + ) + except Exception as error: + print(handler_name, error) return method @@ -91,7 +185,7 @@ def method(*args, **kwargs): def guess_bot_command( message: TelegramMessage, fallback: CommandName = CommandName.HELP -) -> CommandName: +) -> str: """ Tries to guess which bot command user assumed based on content of a message. @@ -100,7 +194,7 @@ def guess_bot_command( Fallback command which will be returned if unable to guess. :returns: - Guessed bot command based on a message. + Guessed bot command name based on a message. """ command = fallback raw_data = message.raw_data @@ -118,4 +212,76 @@ def guess_bot_command( elif (message.get_entity_value("url") is not None): command = CommandName.UPLOAD_URL - return command + return command.value + + +def detect_events( + message: TelegramMessage +) -> Set[str]: + """ + :returns: + Detected dispatcher events. + See `DispatcherEvent` documentation for more. + Note: it is strings values, because these values + should be compared with Redis values, which is + also strings. + """ + events = set() + entities = message.get_entities() + photo, document, audio, video, voice = map( + lambda x: x in message.raw_data, + ("photo", "document", "audio", "video", "voice") + ) + url, hashtag, email, bot_command = map( + lambda x: any(e.type == x for e in entities), + ("url", "hashtag", "email", "bot_command") + ) + plain_text = message.get_plain_text() + + if photo: + events.add(DispatcherEvent.PHOTO.value) + + if document: + events.add(DispatcherEvent.FILE.value) + + if audio: + events.add(DispatcherEvent.AUDIO.value) + + if video: + events.add(DispatcherEvent.VIDEO.value) + + if voice: + events.add(DispatcherEvent.VOICE.value) + + if url: + events.add(DispatcherEvent.URL.value) + + if hashtag: + events.add(DispatcherEvent.HASHTAG.value) + + if email: + events.add(DispatcherEvent.EMAIL.value) + + if bot_command: + events.add(DispatcherEvent.BOT_COMMAND.value) + + if plain_text: + events.add(DispatcherEvent.PLAIN_TEXT.value) + + if not len(events): + events.add(DispatcherEvent.NONE.value) + + return events + + +def match_events( + a: Set[str], + b: Set[str] +) -> bool: + """ + Checks if two groups of events are matched. + + :returns: + `True` - match found, `False` otherwise. + """ + return any(x in b for x in a) diff --git a/src/blueprints/telegram_bot/webhook/dispatcher_events.py b/src/blueprints/telegram_bot/webhook/dispatcher_events.py new file mode 100644 index 0000000..421fe0a --- /dev/null +++ b/src/blueprints/telegram_bot/webhook/dispatcher_events.py @@ -0,0 +1,67 @@ +""" +This code directly related to dispatcher (`dispatcher.py`). +However, it was extracted into separate file to +avoid circular imports. + +Many handlers using `DispatcherEvent`. Howerver, dispatcher +itself imports these handlers. So, circular import occurs. +Handlers may import entire dispatcher module, not only enum +(`import dispatcher`). But i don't like this approach, so, +dispatcher events were extracted into separate file. +Same logic for another enums. +""" + + +from enum import Enum, auto + + +class StringAutoName(Enum): + """ + `auto()` will return strings, not ints. + """ + @staticmethod + def _generate_next_value_(name, start, count, last_values): + return str(count) + + +class DispatcherEvent(StringAutoName): + """ + An event that was detected and fired by dispatcher. + + These events is primarily for Telegram messages. + Note: there can be multiple events in one message, + so, you always should operate with list of events, + not single event (instead you can use list with one element). + + Note: values of that enums are strings, not ints. + It is because it is expected that these values will be + compared with Redis values in future. Redis by default + returns strings. + """ + # Nothing of known events is detected + NONE = auto() + # Message contains plain text (non empty) + PLAIN_TEXT = auto() + PHOTO = auto() + VIDEO = auto() + FILE = auto() + AUDIO = auto() + VOICE = auto() + BOT_COMMAND = auto() + URL = auto() + HASHTAG = auto() + EMAIL = auto() + + +class RouteSource(Enum): + """ + Who initiated the route to a handler. + + For example, `DISPOSABLE_HANDLER` means a handler was + called because of `set_disposable_handler()` from + stateful chat module. + """ + DISPOSABLE_HANDLER = auto() + SUBSCRIBED_HANDLER = auto() + DIRECT_COMMAND = auto() + GUESSED_COMMAND = auto() diff --git a/src/blueprints/telegram_bot/webhook/stateful_chat.py b/src/blueprints/telegram_bot/webhook/stateful_chat.py new file mode 100644 index 0000000..d6351ad --- /dev/null +++ b/src/blueprints/telegram_bot/webhook/stateful_chat.py @@ -0,0 +1,507 @@ +""" +Used for implementing of stateful dialog between the bot and user +on Telegram chats. It is just manages the state and data, so, this +module should work in pair with dispatcher that will route messages +based on current state, data and some custom conditions. In short, +this module manages the state and dispatcher manages the behavior, +dispatcher should be implemented independently. + +- requires Redis to be enabled +- you shouldn't import things that starts with `_`, +because they are intended for internal usage only + +Documentation for this entire module will be provided here, not in every +function documentation. This documentation tells the dispatcher how this +should be implemented, however, dispatcher can change final realization, +so, also take a look at dispatcher documentation for how to use this module. + +- each function can raise an error if it occurs + +"Custom Data" functions. + +Using set/get/delete_ you can manage your own data in +specific namespace. Instead of `update` operation use `set` operation. +For example, `set_user_data` sets/updates your custom data in user namespace, +that potentially can be used to store data about specific user across +multiple chats. `user` argument provides unique user identificator, as +id or name. `expire` argument tells if this data should be deletde after +given number of seconds, `0` is used for permanent storing. +Documentation for different namespaces (user, user in chat, chat, etc.) +is similar. + +"Event Handling" functions. + +"Disposable handler" means that only one handler will exists +at a time, and that handler should handle an event only one time. +For example, you can set handler for `TEXT` event, and when message with +text will arrive, this handler will be called for that message. Next time, +when text messages arrives, this handler will be not called, because +it already was called for previous message. +There can be only one handler, so, calling `set_disposable_handler` at +first in function № 1, then in function № 2, and after in function № 3 +will set only handler from function № 3, because last call from № 3 will +override handler from № 2, and call from № 2 will override handler from № 1. +You should specify set of events, not single event. Dispatcher should +implement the logic for how events of both handler and message are compared. +Dispatcher also should implement deleting of handler when it about to call. +It is recommended to stick with documeneted logic. +`user`, `chat`, `handler` - unique identifiers of these objects, such as +id's or names. +`expire` - means handler will be automatically deleted after given number +of seconds. `0` means it is permanent handler that will wait for it call. +If you call register function again for same handler, then old timeout +will be removed, and new one with this value will be setted. +`events` - iterable of unique events for that dispatcher. +Note: if you want to use `Enum`, then pass values of that enum, not +objects itself. Redis will return back strings even if you will pass +int values, so, be aware of it when comparing these values. +Note: return result is an unordered. So, you shouldn't rely on order. + +"Subscribed handlers" means you can register any amount of handlers for +any events, these handlers will be called every time until you remove them. +For example, you can register handler № 1 for TEXT and URL event, and +register handler № 2 for TEXT and PHOTO event. Realization of routing +is up to dispatcher, but recommend way is to route message (TEXT) to both +handler № 1 and handler № 2, route message (TEXT, PHOTO) to both +handler № 1 and handler № 2, and route message (URL) to handler № 1. +Documentation of function arguments is same as for "disposable handler". +Note: these handlers stored in a set, so, you can safely call register +function which registers same handler from different functions, and result +will be one registered handler. +""" + + +from collections import deque +from typing import Union, Set + +from src.extensions import redis_client + + +# region Common + + +# Namespaces +_SEPARATOR = ":" +_NAMESPACE_KEY = "stateful_chat" +_USER_KEY = "user" +_CHAT_KEY = "chat" +_DATA_KEY = "custom_data" +_DISPOSABLE_HANDLER_KEY = "disposable_handler" +_NAME_KEY = "name" +_EVENTS_KEY = "events" +_SUBSCRIBED_HANDLERS_KEY = "subscribed_handlers" + + +def _create_key(*args) -> str: + return _SEPARATOR.join(map(str, args)) + + +# endregion + + +# region Custom Data + + +def _set_data( + key: str, + field: str, + value: str, + expire: int +) -> None: + key = _create_key(_NAMESPACE_KEY, key, _DATA_KEY, field) + pipeline = redis_client.pipeline() + + pipeline.set(key, value) + + if (expire > 0): + pipeline.expire(key, expire) + + pipeline.execute(raise_on_error=True) + + +def _get_data( + key: str, + field: str +) -> Union[str, None]: + return redis_client.get( + _create_key(_NAMESPACE_KEY, key, _DATA_KEY, field) + ) + + +def _delete_data( + key: str, + field: str +) -> None: + redis_client.delete( + _create_key(_NAMESPACE_KEY, key, _DATA_KEY, field) + ) + + +def set_user_data( + user: str, + key: str, + value: str, + expire: int = 0 +) -> None: + _set_data( + _create_key(_USER_KEY, user), + key, + value, + expire + ) + + +def get_user_data( + user: str, + key: str +): + return _get_data( + _create_key(_USER_KEY, user), + key + ) + + +def delete_user_data( + user: str, + key: str +) -> None: + _delete_data( + _create_key(_USER_KEY, user), + key + ) + + +def set_user_chat_data( + user: str, + chat: str, + key: str, + value: str, + expire: int = 0 +) -> None: + _set_data( + _create_key(_USER_KEY, user, _CHAT_KEY, chat), + key, + value, + expire + ) + + +def get_user_chat_data( + user: str, + chat: str, + key: str +): + return _get_data( + _create_key(_USER_KEY, user, _CHAT_KEY, chat), + key + ) + + +def delete_user_chat_data( + user: str, + chat: str, + key: str +) -> None: + _delete_data( + _create_key(_USER_KEY, user, _CHAT_KEY, chat), + key + ) + + +def set_chat_data( + chat: str, + key: str, + value: str, + expire: int = 0 +) -> None: + _set_data( + _create_key(_CHAT_KEY, chat), + key, + value, + expire + ) + + +def get_chat_data( + chat: str, + key: str +): + return _get_data( + _create_key(_CHAT_KEY, chat), + key + ) + + +def delete_chat_data( + chat: str, + key: str +) -> None: + _delete_data( + _create_key(_CHAT_KEY, chat), + key + ) + + +# endregion + + +# region Event Handling + + +def set_disposable_handler( + user: str, + chat: str, + handler: str, + events: Set[str], + expire: int = 0 +) -> None: + name_key = _create_key( + _NAMESPACE_KEY, + _USER_KEY, + user, + _CHAT_KEY, + chat, + _DISPOSABLE_HANDLER_KEY, + _NAME_KEY + ) + events_key = _create_key( + _NAMESPACE_KEY, + _USER_KEY, + user, + _CHAT_KEY, + chat, + _DISPOSABLE_HANDLER_KEY, + _EVENTS_KEY + ) + pipeline = redis_client.pipeline() + + pipeline.set(name_key, handler) + # in case of update (same name for already + # existing handler) we need to delete old events + # in order to not merge them with new ones. + # also, `sadd` don't clears expire, but + # `delete` does + pipeline.delete(events_key) + pipeline.sadd(events_key, *events) + + if (expire > 0): + pipeline.expire(name_key, expire) + pipeline.expire(events_key, expire) + + pipeline.execute(raise_on_error=True) + + +def get_disposable_handler( + user: str, + chat: str +) -> Union[dict, None]: + name_key = _create_key( + _NAMESPACE_KEY, + _USER_KEY, + user, + _CHAT_KEY, + chat, + _DISPOSABLE_HANDLER_KEY, + _NAME_KEY + ) + events_key = _create_key( + _NAMESPACE_KEY, + _USER_KEY, + user, + _CHAT_KEY, + chat, + _DISPOSABLE_HANDLER_KEY, + _EVENTS_KEY + ) + pipeline = redis_client.pipeline() + + pipeline.get(name_key) + pipeline.smembers(events_key) + + name, events = pipeline.execute(raise_on_error=True) + result = None + + if name: + result = { + "name": name, + "events": events + } + + return result + + +def delete_disposable_handler( + user: str, + chat: str +) -> None: + name_key = _create_key( + _NAMESPACE_KEY, + _USER_KEY, + user, + _CHAT_KEY, + chat, + _DISPOSABLE_HANDLER_KEY, + _NAME_KEY + ) + events_key = _create_key( + _NAMESPACE_KEY, + _USER_KEY, + user, + _CHAT_KEY, + chat, + _DISPOSABLE_HANDLER_KEY, + _EVENTS_KEY + ) + pipeline = redis_client.pipeline() + + pipeline.delete(name_key) + pipeline.delete(events_key) + + pipeline.execute(raise_on_error=True) + + +def subscribe_handler( + user: str, + chat: str, + handler: str, + events: Set[str], + expire: int = 0 +) -> None: + # In this set stored all handler names + # that were registered. It is not indicator + # if handler is registered at the moment of check. + subscribed_handlers_key = _create_key( + _NAMESPACE_KEY, + _USER_KEY, + user, + _CHAT_KEY, + chat, + _SUBSCRIBED_HANDLERS_KEY + ) + # Example - `...::events`. + # In this set stored all events for this handler. + # If `events_key` doesn't exists or empty, then it is + # indicates that handler not registered anymore and + # should be removed from `subscribed_handlers_key`. + events_key = _create_key( + _NAMESPACE_KEY, + _USER_KEY, + user, + _CHAT_KEY, + chat, + _SUBSCRIBED_HANDLERS_KEY, + handler, + _EVENTS_KEY + ) + pipeline = redis_client.pipeline() + + pipeline.sadd(subscribed_handlers_key, handler) + # in case of update (same name for already + # existing handler) we need to delete old events + # in order to not merge them with new ones. + # also, `sadd` don't clears expire, but + # `delete` does + pipeline.delete(events_key) + pipeline.sadd(events_key, *events) + + if (expire > 0): + pipeline.expire(events_key) + + pipeline.execute(raise_on_error=True) + + +def unsubcribe_handler( + user: str, + chat: str, + handler: str +) -> None: + subscribed_handlers_key = _create_key( + _NAMESPACE_KEY, + _USER_KEY, + user, + _CHAT_KEY, + chat, + _SUBSCRIBED_HANDLERS_KEY + ) + events_key = _create_key( + _NAMESPACE_KEY, + _USER_KEY, + user, + _CHAT_KEY, + chat, + _SUBSCRIBED_HANDLERS_KEY, + handler, + _EVENTS_KEY + ) + pipeline = redis_client.pipeline() + + pipeline.srem(subscribed_handlers_key, handler) + pipeline.delete(events_key) + + pipeline.execute(raise_on_error=True) + + +def get_subscribed_handlers( + user: str, + chat: str +) -> deque: + subscribed_handlers_key = _create_key( + _NAMESPACE_KEY, + _USER_KEY, + user, + _CHAT_KEY, + chat, + _SUBSCRIBED_HANDLERS_KEY + ) + possible_handlers = redis_client.smembers( + subscribed_handlers_key + ) + pipeline = redis_client.pipeline() + + for possible_handler in possible_handlers: + pipeline.smembers( + _create_key( + _NAMESPACE_KEY, + _USER_KEY, + user, + _CHAT_KEY, + chat, + _SUBSCRIBED_HANDLERS_KEY, + possible_handler, + _EVENTS_KEY + ) + ) + + events = pipeline.execute(raise_on_error=True) + subscribed_handlers = deque() + i = 0 + + # `possible_handlers` is a set. + # Set it is an unordered structure, so, order + # of iteration not guaranted from time to time + # (for example, from first script execution to second + # script execution). + # However, in Python order of set iteration in + # single run is a same (if set wasn't modified), + # so, we can safely iterate this set one more time + # and associate it values with another values + # through `i` counter (`events` is an array). + # See for more: https://stackoverflow.com/q/3848091/8445442 + for handler_name in possible_handlers: + handler_events = events[i] + i += 1 + + # see `subscribe_handler` documentation for + # why this check works so + if handler_events: + subscribed_handlers.append({ + "name": handler_name, + "events": handler_events + }) + else: + unsubcribe_handler(user, chat, handler_name) + + return subscribed_handlers + + +# endregion diff --git a/src/blueprints/telegram_bot/webhook/telegram_interface.py b/src/blueprints/telegram_bot/webhook/telegram_interface.py index 88ee8c4..4fce4a1 100644 --- a/src/blueprints/telegram_bot/webhook/telegram_interface.py +++ b/src/blueprints/telegram_bot/webhook/telegram_interface.py @@ -88,6 +88,7 @@ class Message: def __init__(self, raw_data: dict) -> None: self.raw_data = raw_data self.entities: Union[List[Entity], None] = None + self.plain_text: Union[str, None] = None @property def message_id(self) -> int: @@ -130,6 +131,66 @@ def get_text(self) -> str: "" ) + def get_text_without_entities( + self, + without: List[str] + ) -> str: + """ + :param without: + Types of entities which should be removed + from message text. See + https://core.telegram.org/bots/api#messageentity + + :returns: + Text of message without specified entities. + Empty string in case if there are no initial + text or nothing left after removing. + """ + original_text = self.get_text() + result_text = original_text + entities = self.get_entities() + + if not original_text: + return "" + + for entity in entities: + if entity.type not in without: + continue + + offset = entity.offset + length = entity.length + value = original_text[offset:offset + length] + + result_text = result_text.replace(value, "") + + return result_text.strip() + + def get_plain_text(self) -> str: + """ + :returns: + `get_text_without_entities([mention, hashtag, + cashtag, bot_command, url, email, phone_number, + code, pre, text_link, text_mention]` + """ + if self.plain_text is not None: + return self.plain_text + + self.plain_text = self.get_text_without_entities([ + "mention", + "hashtag", + "cashtag", + "bot_command", + "url", + "email", + "phone_number", + "code", + "pre", + "text_link", + "text_mention" + ]) + + return self.plain_text + def get_entities(self) -> List[Entity]: """ :returns: Entities from a message. diff --git a/src/blueprints/telegram_bot/webhook/views.py b/src/blueprints/telegram_bot/webhook/views.py index db5d6d0..8fe876f 100644 --- a/src/blueprints/telegram_bot/webhook/views.py +++ b/src/blueprints/telegram_bot/webhook/views.py @@ -6,7 +6,7 @@ from src.blueprints.telegram_bot import telegram_bot_blueprint as bp from . import telegram_interface -from .dispatcher import dispatch, direct_dispatch +from .dispatcher import intellectual_dispatch, direct_dispatch @bp.route("/webhook", methods=["POST"]) @@ -42,7 +42,7 @@ def webhook(): g.telegram_chat = message.get_chat() g.direct_dispatch = direct_dispatch - dispatch(message)() + intellectual_dispatch(message)() return make_success_response() From 9aa54e35e0e33c5df1a1d270a1ac43e87714ce8a Mon Sep 17 00:00:00 2001 From: Sergey Kuznetsov Date: Mon, 16 Nov 2020 16:45:25 +0300 Subject: [PATCH 035/103] Rewrite Yandex.OAuth modules --- CHANGELOG.md | 1 + src/api/yandex/requests.py | 6 +- .../telegram_bot/_common/__init__.py | 0 .../telegram_bot/_common/yandex_oauth.py | 494 ++++++++++++++++++ .../telegram_bot/webhook/commands/yd_auth.py | 305 +++++------ .../telegram_bot/yd_auth/exceptions.py | 20 - src/blueprints/telegram_bot/yd_auth/views.py | 258 +++------ src/configs/flask.py | 39 +- 8 files changed, 728 insertions(+), 395 deletions(-) create mode 100644 src/blueprints/telegram_bot/_common/__init__.py create mode 100644 src/blueprints/telegram_bot/_common/yandex_oauth.py delete mode 100644 src/blueprints/telegram_bot/yd_auth/exceptions.py diff --git a/CHANGELOG.md b/CHANGELOG.md index abea2bf..bde0dd3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -15,6 +15,7 @@ - Upgrade `python` to 3.8.5. - All requirements upgraded to latest version. +- Big refactoring. ### Added diff --git a/src/api/yandex/requests.py b/src/api/yandex/requests.py index e5de073..d5c25c2 100644 --- a/src/api/yandex/requests.py +++ b/src/api/yandex/requests.py @@ -1,5 +1,4 @@ from os import environ -import base64 from requests.auth import HTTPBasicAuth from flask import current_app @@ -29,12 +28,9 @@ def create_user_oauth_url(state: str) -> str: - https://yandex.ru/dev/oauth/doc/dg/concepts/about-docpage/ - :param state: `state` parameter. Will be encoded with base64. + :param state: urlsafe `state` parameter. """ client_id = environ["YANDEX_OAUTH_API_APP_ID"] - state = base64.urlsafe_b64encode( - state.encode() - ).decode() return ( "https://oauth.yandex.ru/authorize?" diff --git a/src/blueprints/telegram_bot/_common/__init__.py b/src/blueprints/telegram_bot/_common/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/blueprints/telegram_bot/_common/yandex_oauth.py b/src/blueprints/telegram_bot/_common/yandex_oauth.py new file mode 100644 index 0000000..19536cf --- /dev/null +++ b/src/blueprints/telegram_bot/_common/yandex_oauth.py @@ -0,0 +1,494 @@ +import base64 +import secrets +from enum import Enum, auto +from typing import Union + +from flask import current_app +import jwt + +from src.api import yandex +from src.extensions import db +from src.database import ( + User, + YandexDiskToken, + UserQuery, + ChatQuery +) + + +class InvalidState(Exception): + """ + Provided state is invalid + (invalid Base64, missing data, wrong data, etc.). + For security reasons there is no exact reason. + """ + pass + + +class ExpiredInsertToken(Exception): + """ + Provided insert token is expired. + """ + pass + + +class InvalidInsertToken(Exception): + """ + Provided insert token (extracted from state) is invalid. + Most probably new state was requested and old one + was passed for handling. + """ + + +class YandexRequestError(Exception): + """ + Unexpected error occurred during Yandex.OAuth HTTP request. + """ + pass + + +class MissingData(Exception): + """ + Requested data is missing. + """ + pass + + +class YandexOAuthClient: + """ + Base class for all Yandex.OAuth clients. + """ + def encode_state(self, user_id: int, insert_token: str) -> str: + """ + :returns: + JWT which should be used as a value for `state` + Yandex.OAuth key. It is urlsafe base64 string. + """ + return base64.urlsafe_b64encode( + jwt.encode( + { + "user_id": user_id, + "insert_token": insert_token + }, + current_app.secret_key.encode(), + algorithm="HS256" + ) + ).decode() + + def decode_state(self, state: str) -> dict: + """ + :param state: + A state from `create_state()`. + + :returns: + A dict of arguments that were passed into `create_state()`. + + :raises: + `InvalidState`. + """ + encoded_state = None + + try: + encoded_state = base64.urlsafe_b64decode( + state.encode() + ).decode() + except Exception: + raise InvalidState() + + decoded_state = None + + try: + decoded_state = jwt.decode( + encoded_state, + current_app.secret_key.encode(), + algorithm="HS256" + ) + except Exception: + raise InvalidState() + + user_id = decoded_state.get("user_id") + insert_token = decoded_state.get("insert_token") + + if not any((user_id, insert_token)): + raise InvalidState() + + return { + "user_id": user_id, + "insert_token": insert_token + } + + def get_user(self, user_id: int, insert_token: str) -> User: + """ + :param user_id: + DB id of needed user. + :param insert_token: + User will be returned only in case when provided + insert token matchs with one from DB. This means + you are allowed to modify this DB user. + Insert token of that user can be modified in futher by + some another operation, so, you should call this function + once and reuse returned result. + + :returns: + DB user. + + :raises: + `MissingData`, `ExpiredInsertToken`, `InvalidInsertToken`. + """ + user = UserQuery.get_user_by_id(user_id) + + if ( + user is None or + # for some reason `yandex_disk_token` not created, + # it is not intended behavior. + user.yandex_disk_token is None + ): + raise MissingData() + + db_insert_token = None + + try: + db_insert_token = user.yandex_disk_token.get_insert_token() + except Exception: + raise ExpiredInsertToken() + + if (insert_token != db_insert_token): + raise InvalidInsertToken() + + return user + + def request_access_token(self, code="", refresh_token="") -> dict: + """ + Makes HTTP request to Yandex.OAuth API to get access token. + + - you should specify only one parameter: + `code` or `refresh_token`. If specified both, then `code` + will be selected. If nothing is specified, then an error + will be thrown. + + :returns: + `ok` indicates status of operation. + If `ok = False`, then `error` will contain + `title` and optional `description`. + if `ok = True`, then `access_token`, `token_type`, + `expires_in`, `refresh_token` will be provided. + + :raises: + `YandexRequestError`. + """ + response = None + kwargs = None + + if code: + kwargs["grant_type"] = "authorization_code" + kwargs["code"] = code + elif refresh_token: + kwargs["grant_type"] = "refresh_token" + kwargs["refresh_token"] = refresh_token + else: + raise Exception("Invalid arguments") + + try: + response = yandex.get_access_token( + **kwargs + )["content"] + except Exception as error: + raise YandexRequestError(str(error)) + + if "error" in response: + return { + "ok": False, + "error": { + "title": response["error"], + "description": response.get("error_description") + } + } + + return { + "ok": True, + "access_token": response["access_token"], + "token_type": response["token_type"], + "expires_in": response["expires_in"], + "refresh_token": response["refresh_token"], + } + + def set_access_token(self, user: User, code: str) -> dict: + """ + Makes request to Yandex.OAuth server, gets access + token and saves it. + + - on success clears insert token. + - perform a DB commit in order to save changes! + + :param user: + DB user. + :param code: + Code from Yandex which was given to user. + + :returns: + `ok` which contains status of operation. + `error` from Yandex in case of `ok = False`, + `error` contains `title` and optional `description`. + + :raises: + `YandexRequestError`. + """ + response = self.request_access_token(code=code) + + if not response["ok"]: + return response + + user.yandex_disk_token.clear_insert_token() + user.yandex_disk_token.set_access_token( + response["access_token"] + ) + user.yandex_disk_token.access_token_type = ( + response["token_type"] + ) + user.yandex_disk_token.access_token_expires_in = ( + response["expires_in"] + ) + user.yandex_disk_token.set_refresh_token( + response["refresh_token"] + ) + + return { + "ok": True + } + + def refresh_access_token(self, user: User) -> dict: + """ + Similar to `set_access_token()`, but uses user + refresh token from DB. + + - perform DB commit in order to save changes! + - `error` not always presented in case of `ok = False`. + + :raises: + `YandexRequestError`. + """ + refresh_token = user.yandex_disk_token.get_refresh_token() + + if refresh_token is None: + return { + "ok": False + } + + response = self.request_access_token(refresh_token=refresh_token) + + if not response["ok"]: + return response + + user.yandex_disk_token.clear_insert_token() + user.yandex_disk_token.set_access_token( + response["access_token"] + ) + user.yandex_disk_token.access_token_type = ( + response["token_type"] + ) + user.yandex_disk_token.access_token_expires_in = ( + response["expires_in"] + ) + user.yandex_disk_token.set_refresh_token( + response["refresh_token"] + ) + + return { + "ok": True + } + + def have_valid_access_token(self, user: User) -> bool: + """ + :returns: + User have valid (not expired) access token. + """ + token = user.yandex_disk_token + + if not token: + return False + + if not token.have_access_token(): + return False + + try: + # if no errors, then `access_token` is valid + token.get_access_token() + + return True + except Exception: + pass + + return False + + def create_insert_token(self, user: User) -> str: + """ + Creates insert token (used to insert new data). + + WARNING: it clears all previous data + (access token, refresh token, etc)! + + - perform DB commit in order to save changes! + + :returns: + Created insert token. + + :raises: + `MissingData` (DB data is corrupted or problems with DB). + """ + user.yandex_disk_token.clear_all_tokens() + user.yandex_disk_token.set_insert_token( + secrets.token_hex( + current_app.config[ + "YANDEX_OAUTH_API_INSERT_TOKEN_BYTES" + ] + ) + ) + user.yandex_disk_token.insert_token_expires_in = ( + current_app.config[ + "YANDEX_OAUTH_API_INSERT_TOKEN_LIFETIME" + ] + ) + + # it is necessary to check if we able to get + # valid token after inseting + insert_token = user.yandex_disk_token.get_insert_token() + + if insert_token is None: + raise MissingData("Insert token is NULL") + + return insert_token + + +class YandexOAuthAutoCodeClient(YandexOAuthClient): + """ + Implements https://yandex.ru/dev/oauth/doc/dg/reference/auto-code-client.html # noqa + """ + def before_user_interaction(self, user: User) -> dict: + """ + This function should be executed before user interation. + + :returns: + `status` that contains operation status. See `OperationStatus` + documentation for more. In case of `status = CONTINUE_TO_URL` + there will be both `url` and `lifetime`. User should open this + url, after `lifetime` seconds this url will be expired. + In case of any other `status` further user actions not needed + because this user already have valid access token. + + :raises: + `YandexRequestError`, `MissingData`. + """ + # it can be not created if it is a new user + if not user.yandex_disk_token: + db.session.add( + YandexDiskToken(user=user) + ) + elif self.have_valid_access_token(user): + return { + "status": OperationStatus.HAVE_ACCESS_TOKEN + } + + refresh_result = self.refresh_access_token(user) + + # if `ok = False`, then there can be useful error message + # from Yandex with some description. At the moment + # we will do nothing with it and just continue + # with need of user interaction + if refresh_result["ok"]: + db.session.commit() + + return { + "status": OperationStatus.ACCESS_TOKEN_REFRESHED + } + + insert_token = self.create_insert_token(user) + state = self.encode_state(user.id, insert_token) + url = yandex.create_user_oauth_url(state) + lifetime_in_seconds = ( + user.yandex_disk_token.insert_token_expires_in + ) + + db.session.commit() + + return { + "status": OperationStatus.CONTINUE_TO_URL, + "url": url, + "lifetime": lifetime_in_seconds + } + + def after_success_redirect(self, state: str, code: str) -> dict: + """ + Should be called after Yandex successful redirect + (when there is both `code` and `state` parameters). + Performs needed operations to end user authorization. + + :returns: + `ok` which contains status of setting of access token. + `error` from Yandex in case of `ok = False`, + `error` contains `title` and optional `description`. + If `ok = False`, you should notify user about occured error + and user should request new authorization link because old + one will become invalid. + `user` is DB user. + + :raises: + - `InvalidState`, `ExpiredInsertToken`, `MissingData`, + `InvalidInsertToken`, `YandexRequestError`. + - Other errors (`Exception`) should be considered as + internal server error. + """ + data = self.decode_state(state) + user = self.get_user( + data["user_id"], + data["insert_token"] + ) + result = self.set_access_token(user, code) + result["user"] = user + + if not result["ok"]: + user.yandex_disk_token.clear_insert_token() + + db.session.commit() + + return result + + def after_error_redirect(self, state: str) -> None: + """ + Should be called after Yandex error redirect + (when there is both `error` and `state` parameters). + + - if function successfully ends, then old user authorization + link will become invalid. + + :raises: + `InvalidState`, `ExpiredInsertToken`, + `InvalidInsertToken`, `MissingData`. + """ + data = self.decode_state(state) + user = self.get_user( + data["user_id"], + data["insert_token"] + ) + + user.yandex_disk_token.clear_insert_token() + db.session.commit() + + +class YandexOAuthConsoleClient(YandexOAuthClient): + pass + + +class OperationStatus(Enum): + """ + Status of requested operation. + """ + # User already have valid access token. + # No further actions is required + HAVE_ACCESS_TOKEN = auto() + + # Access token was successfully refreshed. + # No further actions is required + ACCESS_TOKEN_REFRESHED = auto() + + # User should manually open an URL + CONTINUE_TO_URL = auto() diff --git a/src/blueprints/telegram_bot/webhook/commands/yd_auth.py b/src/blueprints/telegram_bot/webhook/commands/yd_auth.py index 5849a4b..104e6c6 100644 --- a/src/blueprints/telegram_bot/webhook/commands/yd_auth.py +++ b/src/blueprints/telegram_bot/webhook/commands/yd_auth.py @@ -1,20 +1,12 @@ -import secrets +from flask import g, current_app -from flask import ( - g, - current_app -) -import jwt - -from src.extensions import db -from src.database import ( - YandexDiskToken -) -from src.api import telegram, yandex +from src.api import telegram +from src.configs.flask import YandexOAuthAPIMethod from src.blueprints.utils import ( absolute_url_for, get_current_datetime ) +from src.blueprints.telegram_bot._common import yandex_oauth from .common.decorators import ( register_guest, get_db_data @@ -30,116 +22,118 @@ @get_db_data def handle(*args, **kwargs): """ - Handles `/yandex_disk_authorization` command. + Handles `/grant_access` command. - Authorization of bot in user Yandex.Disk. + Allowing to bot to use user Yandex.Disk. """ - user = g.db_user - incoming_chat = g.db_chat private_chat = g.db_private_chat - yd_token = user.yandex_disk_token - - if (private_chat is None): - return request_private_chat(incoming_chat.telegram_id) - - if (yd_token is None): - try: - yd_token = create_empty_yd_token(user) - except Exception as error: - print(error) - return cancel_command(private_chat.telegram_id) - - refresh_needed = False - - if (yd_token.have_access_token()): - try: - yd_token.get_access_token() - - telegram.send_message( - chat_id=private_chat.telegram_id, - text=( - "You already grant me access to your Yandex.Disk." - "\n" - "You can revoke my access with " - f"{CommandName.YD_REVOKE.value}" - ) - ) - - # `access_token` is valid - return - except Exception: - # `access_token` is expired (most probably) or - # data in DB is invalid - refresh_needed = True - - if (refresh_needed): - success = refresh_access_token(yd_token) - - if (success): - current_datetime = get_current_datetime() - date = current_datetime["date"] - time = current_datetime["time"] - timezone = current_datetime["timezone"] - - telegram.send_message( - chat_id=private_chat.telegram_id, - parse_mode="HTML", - text=( - "Access to Yandex.Disk Refreshed" - "\n\n" - "Your granted access was refreshed automatically by me " - f"on {date} at {time} {timezone}." - "\n\n" - "If it wasn't you, you can detach this access with " - f"{CommandName.YD_REVOKE.value}" - ) - ) - - return - - yd_token.clear_all_tokens() - yd_token.set_insert_token( - secrets.token_hex( - current_app.config[ - "YANDEX_DISK_API_INSERT_TOKEN_BYTES" - ] + + # we allow to use this command only in + # private chats for security reasons + if private_chat is None: + incoming_chat_id = kwargs.get( + "chat_id", + g.db_chat.telegram_id ) - ) - yd_token.insert_token_expires_in = ( - current_app.config[ - "YANDEX_DISK_API_INSERT_TOKEN_LIFETIME" - ] - ) - db.session.commit() - insert_token = None + return request_private_chat(incoming_chat_id) + + user = g.db_user + chat_id = private_chat.telegram_id + oauth_method = current_app.config.get("YANDEX_OAUTH_API_METHOD") + + if (oauth_method == YandexOAuthAPIMethod.AUTO_CODE_CLIENT): + run_auto_code_client(user, chat_id) + elif (oauth_method == YandexOAuthAPIMethod.CONSOLE_CLIENT): + run_console_client(user, chat_id) + else: + cancel_command(chat_id) + raise Exception("Unknown Yandex.OAuth method") + + +def run_auto_code_client(db_user, chat_id: int) -> None: + client = yandex_oauth.YandexOAuthAutoCodeClient() + result = None try: - insert_token = yd_token.get_insert_token() + result = client.before_user_interaction(db_user) except Exception as error: - print(error) - return cancel_command(private_chat.telegram_id) - - if (insert_token is None): - print("Error: Insert Token is NULL") - return cancel_command(private_chat.telegram_id) - - state = jwt.encode( - { - "user_id": user.id, - "insert_token": insert_token - }, - current_app.secret_key.encode(), - algorithm="HS256" - ).decode() - yandex_oauth_url = yandex.create_user_oauth_url(state) - insert_token_lifetime = int( - yd_token.insert_token_expires_in / 60 + cancel_command(chat_id) + raise error + + status = result["status"] + + if (status == yandex_oauth.OperationStatus.HAVE_ACCESS_TOKEN): + message_have_access_token(chat_id) + elif (status == yandex_oauth.OperationStatus.ACCESS_TOKEN_REFRESHED): + message_access_token_refreshed(chat_id) + elif (status == yandex_oauth.OperationStatus.CONTINUE_TO_URL): + # only in that case further user actions is needed + message_grant_access_redirect( + chat_id, + result["url"], + result["lifetime"] + ) + else: + cancel_command(chat_id) + raise Exception("Unknown operation status") + + +def run_console_client(db_user, chat_id: int) -> None: + pass + + +# region Messages + + +def message_have_access_token(chat_id: int) -> None: + telegram.send_message( + chat_id=chat_id, + text=( + "You already grant me access to your Yandex.Disk." + "\n" + "You can revoke my access with " + f"{CommandName.YD_REVOKE.value}" + ) ) + + +def message_access_token_refreshed(chat_id: int) -> None: + now = get_current_datetime() + date = now["date"] + time = now["time"] + timezone = now["timezone"] + + telegram.send_message( + chat_id=chat_id, + parse_mode="HTML", + text=( + "Access to Yandex.Disk Refreshed" + "\n\n" + "Your granted access was refreshed automatically by me " + f"on {date} at {time} {timezone}." + "\n\n" + "If it wasn't you, you can detach this access with " + f"{CommandName.YD_REVOKE.value}" + ) + ) + + +def message_grant_access_redirect( + chat_id: int, + url: str, + lifetime_in_seconds: int +) -> None: open_link_button_text = "Grant access" + revoke_command = CommandName.YD_REVOKE.value + yandex_passport_url = "https://passport.yandex.ru/profile" + source_code_url = current_app.config["PROJECT_URL_FOR_CODE"] + privacy_policy_url = absolute_url_for("legal.privacy_policy") + terms_url = absolute_url_for("legal.terms_and_conditions") + lifetime_in_minutes = int(lifetime_in_seconds / 60) telegram.send_message( - chat_id=private_chat.telegram_id, + chat_id=chat_id, parse_mode="HTML", disable_web_page_preview=True, text=( @@ -149,90 +143,37 @@ def handle(*args, **kwargs): "IMPORTANT: don't give this link to anyone, " "because it contains your secret information." "\n\n" - f"This link will expire in {insert_token_lifetime} minutes." + f"This link will expire in {lifetime_in_minutes} minutes." "\n" "This link leads to Yandex page and redirects to bot page." "\n\n" "It is safe to give the access?" "\n" "Yes! I'm getting access only to your Yandex.Disk, " - "not to your account. You can revoke my access at any time with " - f"{CommandName.YD_REVOKE.value} or in your " - 'Yandex Profile. ' + "not to your account. You can revoke my access at any time " + f"with {revoke_command} or in your " + f'Yandex profile. ' "By the way, i'm " - f'open-source ' # noqa + f'open-source ' "and you can make sure that your data will be safe. " - "You can even create your own bot with my functionality if using " - "me makes you feel uncomfortable (:" + "You can even create your own bot with my functionality " + "if using me makes you feel uncomfortable (:" "\n\n" "By using me, you accept " - f'Privacy Policy and ' # noqa - f'Terms of service. ' # noqa + f'Privacy Policy and ' + f'Terms of service. ' ), - reply_markup={"inline_keyboard": [ - [ - { - "text": open_link_button_text, - "url": yandex_oauth_url - } + reply_markup={ + "inline_keyboard": [ + [ + { + "text": open_link_button_text, + "url": url + } + ] ] - ]} + } ) -def create_empty_yd_token(user) -> YandexDiskToken: - """ - Creates empty Yandex.Disk token and binds - this to provided user. - """ - new_yd_token = YandexDiskToken(user=user) - - db.session.add(new_yd_token) - db.session.commit() - - return new_yd_token - - -def refresh_access_token(yd_token: YandexDiskToken) -> bool: - """ - Tries to refresh user access token by using refresh token. - - :returns: `True` in case of success else `False`. - """ - refresh_token = yd_token.get_refresh_token() - - if (refresh_token is None): - return False - - result = None - - try: - result = yandex.get_access_token( - grant_type="refresh_token", - refresh_token=refresh_token - ) - except Exception as error: - print(error) - return False - - yandex_response = result["content"] - - if ("error" in yandex_response): - return False - - yd_token.clear_insert_token() - yd_token.set_access_token( - yandex_response["access_token"] - ) - yd_token.access_token_type = ( - yandex_response["token_type"] - ) - yd_token.access_token_expires_in = ( - yandex_response["expires_in"] - ) - yd_token.set_refresh_token( - yandex_response["refresh_token"] - ) - db.session.commit() - - return True +# endregion diff --git a/src/blueprints/telegram_bot/yd_auth/exceptions.py b/src/blueprints/telegram_bot/yd_auth/exceptions.py deleted file mode 100644 index a9eda46..0000000 --- a/src/blueprints/telegram_bot/yd_auth/exceptions.py +++ /dev/null @@ -1,20 +0,0 @@ -class InvalidCredentials(Exception): - """ - Provided credentials is not valid. - """ - pass - - -class LinkExpired(Exception): - """ - Link is expired and not valid anymore. - """ - pass - - -class InvalidInsertToken(Exception): - """ - Provided `insert_token` is not valid with - `insert_token` from DB. - """ - pass diff --git a/src/blueprints/telegram_bot/yd_auth/views.py b/src/blueprints/telegram_bot/yd_auth/views.py index 0ddcead..047304c 100644 --- a/src/blueprints/telegram_bot/yd_auth/views.py +++ b/src/blueprints/telegram_bot/yd_auth/views.py @@ -1,146 +1,95 @@ -import base64 - from flask import ( request, abort, - render_template, - current_app + render_template ) -import jwt -from src.extensions import db -from src.database import ( - UserQuery, - ChatQuery -) -from src.api import yandex, telegram +from src.api import telegram +from src.database import ChatQuery +from src.blueprints.utils import get_current_datetime from src.blueprints.telegram_bot import telegram_bot_blueprint as bp -from src.blueprints.utils import ( - get_current_datetime -) +from src.blueprints.telegram_bot._common import yandex_oauth from src.blueprints.telegram_bot.webhook.commands import CommandName -from .exceptions import ( - InvalidCredentials, - LinkExpired, - InvalidInsertToken -) @bp.route("/yandex_disk_authorization") def yd_auth(): """ - Handles user redirect from Yandex OAuth page. + Handles user redirect from Yandex.OAuth page + (Auto Code method). """ - if (is_error_request()): - return handle_error() - elif (is_success_request()): - return handle_success() + if is_success_request(): + handle_success() + elif is_error_request(): + handle_error() else: abort(400) -def is_error_request() -> bool: +def is_success_request() -> bool: """ - :returns: Incoming request is a failed user authorization. + :returns: + Incoming request is a successful user authorization. """ state = request.args.get("state", "") - error = request.args.get("error", "") + code = request.args.get("code", "") return ( len(state) > 0 and - len(error) > 0 + len(code) > 0 ) -def is_success_request() -> bool: +def is_error_request() -> bool: """ - :returns: Incoming request is a successful user authorization. + :returns: + Incoming request is a failed user authorization. """ state = request.args.get("state", "") - code = request.args.get("code", "") + error = request.args.get("error", "") return ( len(state) > 0 and - len(code) > 0 + len(error) > 0 ) -def handle_error(): - """ - Handles failed user authorization. - """ - try: - db_user = get_db_user() - db_user.yandex_disk_token.clear_insert_token() - db.session.commit() - except Exception: - pass - - return create_error_response() - - def handle_success(): """ Handles success user authorization. """ - db_user = None + state = request.args["state"] + code = request.args["code"] + client = yandex_oauth.YandexOAuthAutoCodeClient() + result = None try: - db_user = get_db_user() - except InvalidCredentials: + result = client.after_success_redirect(state, code) + except yandex_oauth.InvalidState: return create_error_response("invalid_credentials") - except LinkExpired: + except yandex_oauth.ExpiredInsertToken: return create_error_response("link_expired") - except InvalidInsertToken: + except yandex_oauth.InvalidInsertToken: return create_error_response("invalid_insert_token") except Exception as error: print(error) return create_error_response("internal_server_error") - code = request.args["code"] - yandex_response = None - - try: - yandex_response = yandex.get_access_token( - grant_type="authorization_code", - code=code - )["content"] - except Exception as error: - print(error) - return create_error_response("internal_server_error") - - if ("error" in yandex_response): - db_user.yandex_disk_token.clear_all_tokens() - db.session.commit() - + if not result["ok"]: return create_error_response( error_code="internal_server_error", - raw_error_title=yandex_response["error"], - raw_error_description=yandex_response.get("error_description") + raw_error_title=result["error"], + raw_error_description=result.get("error_description") ) - db_user.yandex_disk_token.clear_insert_token() - db_user.yandex_disk_token.set_access_token( - yandex_response["access_token"] - ) - db_user.yandex_disk_token.access_token_type = ( - yandex_response["token_type"] - ) - db_user.yandex_disk_token.access_token_expires_in = ( - yandex_response["expires_in"] - ) - db_user.yandex_disk_token.set_refresh_token( - yandex_response["refresh_token"] - ) - db.session.commit() - - private_chat = ChatQuery.get_private_chat(db_user.id) + user = result["user"] + private_chat = ChatQuery.get_private_chat(user.id) - if (private_chat): - current_datetime = get_current_datetime() - date = current_datetime["date"] - time = current_datetime["time"] - timezone = current_datetime["timezone"] + if private_chat: + now = get_current_datetime() + date = now["date"] + time = now["time"] + timezone = now["timezone"] telegram.send_message( chat_id=private_chat.telegram_id, @@ -159,23 +108,53 @@ def handle_success(): return create_success_response() +def handle_error(): + """ + Handles failed user authorization. + """ + state = request.args["state"] + client = yandex_oauth.YandexOAuthAutoCodeClient() + + try: + client.after_error_redirect(state) + except Exception: + # we do not care about any errors at that stage + pass + + return create_error_response() + + +def create_success_response(): + """ + :returns: + Rendered template for success page. + """ + return render_template( + "telegram_bot/yd_auth/success.html" + ) + + def create_error_response( error_code: str = None, raw_error_title: str = None, raw_error_description: str = None ): """ - :param error_code: Name of error for user friendly - information. If not specified, then defaults to + :param error_code: + Name of error for user friendly information. + If not specified, then defaults to `error` argument from request. - :param raw_error_title: Raw error title for - debugging purposes. If not specified, then defaults to + :param raw_error_title: + Raw error title for debugging purposes. + If not specified, then defaults to `error_code` argument. - :param raw_error_description: Raw error description - for debugging purposes. If not specified, then defaults to + :param raw_error_description: + Raw error description for debugging purposes. + If not specified, then defaults to `error_description` argument from request. - :returns: Rendered template for error page. + :returns: + Rendered template for error page. """ possible_errors = { "access_denied": { @@ -188,7 +167,7 @@ def create_error_response( "unauthorized_client": { "title": "Application is unavailable", "description": ( - "There is a problems with the me. " + "There is a problems with me. " "Try later please." ) }, @@ -223,7 +202,7 @@ def create_error_response( error = request.args.get("error") error_description = request.args.get("error_description") - if (error_code is None): + if error_code is None: error_code = error error_info = possible_errors.get(error_code, {}) @@ -237,84 +216,3 @@ def create_error_response( raw_error_description=(raw_error_description or error_description), raw_state=state ) - - -def create_success_response(): - """ - :returns: Rendered template for success page. - """ - return render_template( - "telegram_bot/yd_auth/success.html" - ) - - -def get_db_user(): - """ - - `insert_token` will be checked. If it is not valid, - an error will be thrown. You shouldn't clear any tokens - in case of error, because provided tokens is not known - to attacker (potential). - - you shouldn't try to avoid checking logic! It is really - unsafe to access DB user without `insert_token`. - - :returns: User from DB based on incoming `state` from request. - This user have `yandex_disk_token` property which is - not `None`. - - :raises InvalidCredentials: If `state` have invalid - data or user not found in DB. - :raises LinkExpired: Requested link is expired and - not valid anymore. - :raises InvalidInsertToken: Provided `insert_token` - is not valid. - """ - base64_state = request.args["state"] - encoded_state = None - decoded_state = None - - try: - encoded_state = base64.urlsafe_b64decode( - base64_state.encode() - ).decode() - except Exception: - raise InvalidCredentials() - - try: - decoded_state = jwt.decode( - encoded_state, - current_app.secret_key.encode(), - algorithm="HS256" - ) - except Exception: - raise InvalidCredentials() - - incoming_user_id = decoded_state.get("user_id") - incoming_insert_token = decoded_state.get("insert_token") - - if ( - incoming_user_id is None or - incoming_insert_token is None - ): - raise InvalidCredentials() - - db_user = UserQuery.get_user_by_id(int(incoming_user_id)) - - if ( - db_user is None or - # for some reason `yandex_disk_token` not created, - # it is not intended behavior. - db_user.yandex_disk_token is None - ): - raise InvalidCredentials() - - db_insert_token = None - - try: - db_insert_token = db_user.yandex_disk_token.get_insert_token() - except Exception: - raise LinkExpired() - - if (incoming_insert_token != db_insert_token): - raise InvalidInsertToken() - - return db_user diff --git a/src/configs/flask.py b/src/configs/flask.py index 031be82..678cff3 100644 --- a/src/configs/flask.py +++ b/src/configs/flask.py @@ -3,6 +3,7 @@ """ import os +from enum import Enum, auto from dotenv import load_dotenv @@ -28,6 +29,25 @@ def load_env(): load_env() +class YandexOAuthAPIMethod(Enum): + """ + Which method to use for OAuth. + """ + # When user give access, he will be redirected + # to the app site, and app will extract code + # automatically. + # https://yandex.ru/dev/oauth/doc/dg/reference/auto-code-client.html + AUTO_CODE_CLIENT = auto() + # When user give access, he will see code, and + # that code user should manually send to the Telegram bot. + # This method intended for cases when you don't have + # permanent domain name (for example, when testing with `ngrok`) + # or when you want to hide it. + # `AUTO_CODE_CLIENT` provides better UX. + # https://yandex.ru/dev/oauth/doc/dg/reference/console-client.html/ + CONSOLE_CLIENT = auto() + + class Config: """ Notes: @@ -78,19 +98,21 @@ class Config: # stop waiting for a Yandex response # after a given number of seconds YANDEX_OAUTH_API_TIMEOUT = 15 - - # Yandex.Disk API - # stop waiting for a Yandex response - # after a given number of seconds - YANDEX_DISK_API_TIMEOUT = 5 + # see `YandexOAuthAPIMethod` for more + YANDEX_OAUTH_API_METHOD = YandexOAuthAPIMethod.AUTO_CODE_CLIENT # `insert_token` (controls `INSERT` operation) - # will contain n random bytes. Each byte will be + # will contain N random bytes. Each byte will be # converted to two hex digits - YANDEX_DISK_API_INSERT_TOKEN_BYTES = 8 + YANDEX_OAUTH_API_INSERT_TOKEN_BYTES = 8 # lifetime of `insert_token` in seconds starting # from date of issue. It is better to find # best combination between `bytes` and `lifetime` - YANDEX_DISK_API_INSERT_TOKEN_LIFETIME = 60 * 10 + YANDEX_OAUTH_API_INSERT_TOKEN_LIFETIME = 60 * 10 + + # Yandex.Disk API + # stop waiting for a Yandex response + # after a given number of seconds + YANDEX_DISK_API_TIMEOUT = 5 # maximum number of checks of operation status # (for example, if file is downloaded by Yandex.Disk). # It is blocks request until check ending! @@ -120,6 +142,7 @@ class DevelopmentConfig(Config): DEBUG = True TESTING = False SQLALCHEMY_ECHO = "debug" + YANDEX_OAUTH_API_METHOD = YandexOAuthAPIMethod.CONSOLE_CLIENT class TestingConfig(Config): From 8342d71833f6d43d277d6a455dd100324fba0ab6 Mon Sep 17 00:00:00 2001 From: Sergey Kuznetsov Date: Tue, 17 Nov 2020 16:35:18 +0300 Subject: [PATCH 036/103] In dispatcher add traceback logging on handler error --- src/blueprints/telegram_bot/webhook/dispatcher.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/src/blueprints/telegram_bot/webhook/dispatcher.py b/src/blueprints/telegram_bot/webhook/dispatcher.py index dd35e3f..6454139 100644 --- a/src/blueprints/telegram_bot/webhook/dispatcher.py +++ b/src/blueprints/telegram_bot/webhook/dispatcher.py @@ -4,6 +4,7 @@ Set ) from collections import deque +import traceback from src.extensions import redis_client from . import commands @@ -129,7 +130,11 @@ def method(*args, **kwargs): message_events=message_events ) except Exception as error: - print(handler_name, error) + print( + f"{handler_name}: {error}", + "\n", + traceback.format_exc() + ) return method From a3b99731feb8f1d108e376a4de076260bbc2dc06 Mon Sep 17 00:00:00 2001 From: Sergey Kuznetsov Date: Tue, 17 Nov 2020 18:40:08 +0300 Subject: [PATCH 037/103] Fixes for 9aa54e35e0e33c5df1a1d270a1ac43e87714ce8a --- src/blueprints/telegram_bot/_common/yandex_oauth.py | 11 +++++------ src/blueprints/telegram_bot/yd_auth/views.py | 4 ++-- 2 files changed, 7 insertions(+), 8 deletions(-) diff --git a/src/blueprints/telegram_bot/_common/yandex_oauth.py b/src/blueprints/telegram_bot/_common/yandex_oauth.py index 19536cf..0780d2e 100644 --- a/src/blueprints/telegram_bot/_common/yandex_oauth.py +++ b/src/blueprints/telegram_bot/_common/yandex_oauth.py @@ -177,7 +177,7 @@ def request_access_token(self, code="", refresh_token="") -> dict: `YandexRequestError`. """ response = None - kwargs = None + kwargs = {} if code: kwargs["grant_type"] = "authorization_code" @@ -311,14 +311,13 @@ def have_valid_access_token(self, user: User) -> bool: return False try: - # if no errors, then `access_token` is valid + # there will be errors in case of + # expired or invalid token token.get_access_token() - - return True except Exception: - pass + return False - return False + return True def create_insert_token(self, user: User) -> str: """ diff --git a/src/blueprints/telegram_bot/yd_auth/views.py b/src/blueprints/telegram_bot/yd_auth/views.py index 047304c..0f51398 100644 --- a/src/blueprints/telegram_bot/yd_auth/views.py +++ b/src/blueprints/telegram_bot/yd_auth/views.py @@ -19,9 +19,9 @@ def yd_auth(): (Auto Code method). """ if is_success_request(): - handle_success() + return handle_success() elif is_error_request(): - handle_error() + return handle_error() else: abort(400) From 31f7641e982f7f09efb418f754586873cc93d6d6 Mon Sep 17 00:00:00 2001 From: Sergey Kuznetsov Date: Tue, 17 Nov 2020 18:41:22 +0300 Subject: [PATCH 038/103] Add Yandex.OAuth Console Client --- CHANGELOG.md | 1 + src/api/yandex/requests.py | 1 + .../telegram_bot/_common/yandex_oauth.py | 21 +- .../telegram_bot/webhook/commands/yd_auth.py | 245 +++++++++++++++++- src/configs/flask.py | 1 + 5 files changed, 263 insertions(+), 6 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index bde0dd3..7f5e1a0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -20,6 +20,7 @@ ### Added - Stateful chat support. Now bot can store custom user data (in different namespaces: user, chat, user in chat); determine Telegram message types; register single use handler (call once for message) with optional timeout for types of message; subscribe handlers with optional timeout for types of messages. +- [Console Client](https://yandex.ru/dev/oauth/doc/dg/reference/console-client.html) Yandex.OAuth method. By default it is disabled, and default one is [Auto Code Client](https://yandex.ru/dev/oauth/doc/dg/reference/auto-code-client.html/). - Support for different env-type files (based on current environment). Initially it was only for production. - Web Site: 302 (redirect to Telegram) will be returned instead of 404 (not found page), but only in production mode. - Debug configuration for VSCode. diff --git a/src/api/yandex/requests.py b/src/api/yandex/requests.py index d5c25c2..ac1de4b 100644 --- a/src/api/yandex/requests.py +++ b/src/api/yandex/requests.py @@ -67,6 +67,7 @@ def make_oauth_request(method_name: str, data: dict): password = environ["YANDEX_OAUTH_API_APP_PASSWORD"] return request( + raise_for_status=False, content_type="json", method="POST", url=url, diff --git a/src/blueprints/telegram_bot/_common/yandex_oauth.py b/src/blueprints/telegram_bot/_common/yandex_oauth.py index 0780d2e..dcfac71 100644 --- a/src/blueprints/telegram_bot/_common/yandex_oauth.py +++ b/src/blueprints/telegram_bot/_common/yandex_oauth.py @@ -371,6 +371,8 @@ def before_user_interaction(self, user: User) -> dict: documentation for more. In case of `status = CONTINUE_TO_URL` there will be both `url` and `lifetime`. User should open this url, after `lifetime` seconds this url will be expired. + `state` is used to avoid handling of url, but you should + already have valid code from Yandex. In case of any other `status` further user actions not needed because this user already have valid access token. @@ -412,7 +414,8 @@ def before_user_interaction(self, user: User) -> dict: return { "status": OperationStatus.CONTINUE_TO_URL, "url": url, - "lifetime": lifetime_in_seconds + "lifetime": lifetime_in_seconds, + "state": state } def after_success_redirect(self, state: str, code: str) -> dict: @@ -473,8 +476,20 @@ def after_error_redirect(self, state: str) -> None: db.session.commit() -class YandexOAuthConsoleClient(YandexOAuthClient): - pass +# It inherits `YandexOAuthAutoCodeClient`, not base `YandexOAuthClient`, +# because we will use it exactly like `YandexOAuthAutoCodeClient`. +# We doing so because `YandexOAuthConsoleClient` intended mostly +# for usage at development process, so, UX not the key. +# However, it is better to write pure code for that module later +class YandexOAuthConsoleClient(YandexOAuthAutoCodeClient): + """ + Implements https://yandex.ru/dev/oauth/doc/dg/reference/console-client.html # noqa + """ + def handle_code(self, state: str, code: str) -> dict: + """ + See `YandexOAuthAutoCodeClient.after_success_redirect` documentation. + """ + return self.after_success_redirect(state, code) class OperationStatus(Enum): diff --git a/src/blueprints/telegram_bot/webhook/commands/yd_auth.py b/src/blueprints/telegram_bot/webhook/commands/yd_auth.py index 104e6c6..b5920b8 100644 --- a/src/blueprints/telegram_bot/webhook/commands/yd_auth.py +++ b/src/blueprints/telegram_bot/webhook/commands/yd_auth.py @@ -1,12 +1,23 @@ from flask import g, current_app from src.api import telegram +from src.extensions import redis_client from src.configs.flask import YandexOAuthAPIMethod from src.blueprints.utils import ( absolute_url_for, get_current_datetime ) from src.blueprints.telegram_bot._common import yandex_oauth +from src.blueprints.telegram_bot.webhook.stateful_chat import ( + set_disposable_handler, + set_user_chat_data, + get_user_chat_data, + delete_user_chat_data +) +from src.blueprints.telegram_bot.webhook.dispatcher_events import ( + DispatcherEvent, + RouteSource +) from .common.decorators import ( register_guest, get_db_data @@ -40,12 +51,22 @@ def handle(*args, **kwargs): user = g.db_user chat_id = private_chat.telegram_id + route_source = kwargs.get("route_source") + + # console client is waiting for user code + if (route_source == RouteSource.DISPOSABLE_HANDLER): + return end_console_client( + user, + chat_id, + kwargs["message"].get_plain_text() + ) + oauth_method = current_app.config.get("YANDEX_OAUTH_API_METHOD") if (oauth_method == YandexOAuthAPIMethod.AUTO_CODE_CLIENT): run_auto_code_client(user, chat_id) elif (oauth_method == YandexOAuthAPIMethod.CONSOLE_CLIENT): - run_console_client(user, chat_id) + start_console_client(user, chat_id) else: cancel_command(chat_id) raise Exception("Unknown Yandex.OAuth method") @@ -79,8 +100,133 @@ def run_auto_code_client(db_user, chat_id: int) -> None: raise Exception("Unknown operation status") -def run_console_client(db_user, chat_id: int) -> None: - pass +def start_console_client(db_user, chat_id: int) -> None: + if not redis_client.is_enabled: + cancel_command(chat_id) + raise Exception("Redis is required") + + client = yandex_oauth.YandexOAuthConsoleClient() + result = None + + try: + result = client.before_user_interaction(db_user) + except Exception as error: + cancel_command(chat_id) + raise error + + status = result["status"] + + if (status == yandex_oauth.OperationStatus.HAVE_ACCESS_TOKEN): + message_have_access_token(chat_id) + elif (status == yandex_oauth.OperationStatus.ACCESS_TOKEN_REFRESHED): + message_access_token_refreshed(chat_id) + elif (status == yandex_oauth.OperationStatus.CONTINUE_TO_URL): + # only in that case further user actions is needed + message_grant_access_code( + chat_id, + result["url"], + result["lifetime"] + ) + # we will pass this state later + set_user_chat_data( + db_user.telegram_id, + chat_id, + "yandex_oauth_console_client_state", + result["state"], + result["lifetime"] + ) + # on next plain text message we will handle provided code + set_disposable_handler( + db_user.telegram_id, + chat_id, + CommandName.YD_AUTH.value, + [DispatcherEvent.PLAIN_TEXT.value], + result["lifetime"] + ) + else: + cancel_command(chat_id) + raise Exception("Unknown operation status") + + +def end_console_client(db_user, chat_id: int, code: str) -> None: + state = get_user_chat_data( + db_user.telegram_id, + chat_id, + "yandex_oauth_console_client_state" + ) + delete_user_chat_data( + db_user.telegram_id, + chat_id, + "yandex_oauth_console_client_state" + ) + + if not state: + return telegram.send_message( + chat_id=chat_id, + text=( + "You waited too long. " + f"Send {CommandName.YD_AUTH} again." + ) + ) + + client = yandex_oauth.YandexOAuthConsoleClient() + result = None + message = None + + try: + result = client.handle_code(state, code) + except yandex_oauth.InvalidState: + message = ( + "Your credentials is not valid. " + f"Try send {CommandName.YD_AUTH} again." + ) + except yandex_oauth.ExpiredInsertToken: + message = ( + "You waited too long. " + f"Send {CommandName.YD_AUTH} again." + ) + except yandex_oauth.InvalidInsertToken: + message = ( + "You have too many authorization sessions. " + f"Send {CommandName.YD_AUTH} again and use only last link." + ) + except Exception as error: + cancel_command(chat_id) + raise error + + if message: + return telegram.send_message( + chat_id=chat_id, + text=message + ) + + if not result["ok"]: + error = result["error"] + title = error.get( + "title", + "Unknown error" + ) + description = error.get( + "description", + "Can't tell you more" + ) + + return telegram.send_message( + chat_id=chat_id, + parse_mode="HTML", + text=( + "Yandex.OAuth Error" + "\n\n" + f"Error: {title}" + "\n" + f"Description: {description}" + "\n\n" + "I still don't have access. " + f"Start new session using {CommandName.YD_AUTH.value}" + ) + ) + + message_access_token_granted(chat_id) # region Messages @@ -119,6 +265,27 @@ def message_access_token_refreshed(chat_id: int) -> None: ) +def message_access_token_granted(chat_id: int) -> None: + now = get_current_datetime() + date = now["date"] + time = now["time"] + timezone = now["timezone"] + + telegram.send_message( + chat_id=chat_id, + parse_mode="HTML", + text=( + "Access to Yandex.Disk Granted" + "\n\n" + "My access was attached to your Telegram account " + f"on {date} at {time} {timezone}." + "\n\n" + "If it wasn't you, then detach this access with " + f"{CommandName.YD_REVOKE.value}" + ) + ) + + def message_grant_access_redirect( chat_id: int, url: str, @@ -176,4 +343,76 @@ def message_grant_access_redirect( ) +def message_grant_access_code( + chat_id: int, + url: str, + lifetime_in_seconds: int +) -> None: + open_link_button_text = "Grant access" + revoke_command = CommandName.YD_REVOKE.value + yandex_passport_url = "https://passport.yandex.ru/profile" + source_code_url = current_app.config["PROJECT_URL_FOR_CODE"] + privacy_policy_url = absolute_url_for("legal.privacy_policy") + terms_url = absolute_url_for("legal.terms_and_conditions") + lifetime_in_minutes = int(lifetime_in_seconds / 60) + + telegram.send_message( + chat_id=chat_id, + parse_mode="HTML", + disable_web_page_preview=True, + text=( + f'Open special link by pressing on "{open_link_button_text}" ' + "button and grant me access to your Yandex.Disk." + "\n\n" + "IMPORTANT: don't give this link to anyone, " + "because it contains your secret information." + "\n\n" + f"This link will expire in {lifetime_in_minutes} minutes." + "\n" + "This link leads to Yandex page. After granting access, " + "you will need to send me the issued code." + "\n\n" + "It is safe to give the access?" + "\n" + "Yes! I'm getting access only to your Yandex.Disk, " + "not to your account. You can revoke my access at any time " + f"with {revoke_command} or in your " + f'Yandex profile. ' + "By the way, i'm " + f'open-source ' + "and you can make sure that your data will be safe. " + "You can even create your own bot with my functionality " + "if using me makes you feel uncomfortable (:" + "\n\n" + "By using me, you accept " + f'Privacy Policy and ' + f'Terms of service. ' + ), + reply_markup={ + "inline_keyboard": [ + [ + { + "text": open_link_button_text, + "url": url + } + ] + ] + } + ) + # TODO: + # React on press of inline keyboard button + # (https://core.telegram.org/bots/api#callbackquery), + # not send separate message immediately. + # But it requires refactoring of dispatcher and others. + # At the moment let it be implemented as it is, + # because "Console Client" is mostly for developers, not users. + telegram.send_message( + chat_id=chat_id, + text=( + "Open this link, grant me an access " + "and then send me a code" + ) + ) + + # endregion diff --git a/src/configs/flask.py b/src/configs/flask.py index 678cff3..c23a185 100644 --- a/src/configs/flask.py +++ b/src/configs/flask.py @@ -38,6 +38,7 @@ class YandexOAuthAPIMethod(Enum): # automatically. # https://yandex.ru/dev/oauth/doc/dg/reference/auto-code-client.html AUTO_CODE_CLIENT = auto() + # When user give access, he will see code, and # that code user should manually send to the Telegram bot. # This method intended for cases when you don't have From 416ab5da999e736e966c222d8c962b1da6a6c94e Mon Sep 17 00:00:00 2001 From: Sergey Kuznetsov Date: Tue, 17 Nov 2020 19:40:06 +0300 Subject: [PATCH 039/103] Refactoring of /revoke_access --- CHANGELOG.md | 8 ++ .../telegram_bot/_common/yandex_oauth.py | 9 ++ .../webhook/commands/yd_revoke.py | 87 +++++++++++++------ 3 files changed, 79 insertions(+), 25 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 7f5e1a0..980f55e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,10 @@ ## Telegram Bot +### Improved + +- Text of some bot responses. + ### Added - `/publish`: publishing of files or folders. @@ -30,6 +34,10 @@ - Redirect to favicon will be handled by nginx. - Biggest photo (from single photo file) will be selected based on total pixels count, not based on height. +### Fixed + +- A bug when new user (didn't use any command before) used `/revoke_access` command and it led to request crash (500). + # 1.1.0 (May 9, 2020) diff --git a/src/blueprints/telegram_bot/_common/yandex_oauth.py b/src/blueprints/telegram_bot/_common/yandex_oauth.py index dcfac71..c116ad5 100644 --- a/src/blueprints/telegram_bot/_common/yandex_oauth.py +++ b/src/blueprints/telegram_bot/_common/yandex_oauth.py @@ -297,6 +297,15 @@ def refresh_access_token(self, user: User) -> dict: "ok": True } + def clear_access_token(self, user: User) -> None: + """ + Clears access token. + + - perform DB commit in order to save changes! + """ + user.yandex_disk_token.clear_access_token() + user.yandex_disk_token.clear_refresh_token() + def have_valid_access_token(self, user: User) -> bool: """ :returns: diff --git a/src/blueprints/telegram_bot/webhook/commands/yd_revoke.py b/src/blueprints/telegram_bot/webhook/commands/yd_revoke.py index fd26bf2..7c9f9cb 100644 --- a/src/blueprints/telegram_bot/webhook/commands/yd_revoke.py +++ b/src/blueprints/telegram_bot/webhook/commands/yd_revoke.py @@ -3,51 +3,83 @@ from src.extensions import db from src.api import telegram from src.blueprints.utils import get_current_datetime -from .common.decorators import get_db_data -from .common.responses import request_private_chat +from src.blueprints.telegram_bot._common.yandex_oauth import YandexOAuthClient +from .common.decorators import ( + get_db_data, + register_guest +) +from .common.responses import ( + request_private_chat, + cancel_command +) from . import CommandName +class YandexOAuthRemoveClient(YandexOAuthClient): + def clear_access_token(self, db_user) -> None: + super().clear_access_token(db_user) + db.session.commit() + + +@register_guest @get_db_data def handle(*args, **kwargs): """ - Handles `/yandex_disk_revoke` command. + Handles `/revoke_access` command. Revokes bot access to user Yandex.Disk. """ - user = g.db_user - incoming_chat = g.db_chat private_chat = g.db_private_chat - if (private_chat is None): - return request_private_chat(incoming_chat.telegram_id) + if private_chat is None: + incoming_chat_id = kwargs.get( + "chat_id", + g.db_chat.telegram_id + ) + + return request_private_chat(incoming_chat_id) + + user = g.db_user + chat_id = private_chat.telegram_id + client = YandexOAuthRemoveClient() if ( (user is None) or - (user.yandex_disk_token is None) or - (not user.yandex_disk_token.have_access_token()) + not client.have_valid_access_token(user) ): - telegram.send_message( - chat_id=private_chat.telegram_id, - text=( - "You don't granted me access to your Yandex.Disk." - "\n" - f"You can do that with {CommandName.YD_AUTH.value}" - ) - ) + return message_dont_have_access_token(chat_id) + + try: + client.clear_access_token(user) + except Exception as error: + cancel_command(chat_id) + raise error + + message_access_token_removed(chat_id) - return - user.yandex_disk_token.clear_all_tokens() - db.session.commit() +# region Messages - current_datetime = get_current_datetime() - date = current_datetime["date"] - time = current_datetime["time"] - timezone = current_datetime["timezone"] +def message_dont_have_access_token(chat_id: int) -> None: telegram.send_message( - chat_id=private_chat.telegram_id, + chat_id=chat_id, + text=( + "You don't granted me access to your Yandex.Disk." + "\n" + f"You can do that with {CommandName.YD_AUTH.value}" + ) + ) + + +def message_access_token_removed(chat_id: int) -> None: + now = get_current_datetime() + date = now["date"] + time = now["time"] + timezone = now["timezone"] + + telegram.send_message( + chat_id=chat_id, parse_mode="HTML", disable_web_page_preview=True, text=( @@ -58,5 +90,10 @@ def handle(*args, **kwargs): "\n\n" "Don't forget to do that in your " 'Yandex Profile.' + "\n" + f"To grant access again use {CommandName.YD_AUTH.value}" ) ) + + +# endregion From 07be1a4dbba919c02a0058a4d9bf012791da5755 Mon Sep 17 00:00:00 2001 From: Sergey Kuznetsov Date: Wed, 18 Nov 2020 18:00:49 +0300 Subject: [PATCH 040/103] Refactoring of folders structure --- src/app.py | 2 +- src/blueprints/{ => _common}/utils.py | 0 src/blueprints/legal/views.py | 2 +- .../names.py => _common/command_names.py} | 0 .../{webhook => _common}/stateful_chat.py | 0 .../telegram_interface.py | 0 .../yandex_api.py => _common/yandex_disk.py} | 0 .../telegram_bot/webhook/commands/__init__.py | 1 - .../commands/{common => _common}/__init__.py | 0 .../{common => _common}/decorators.py | 2 +- .../commands/{common => _common}/responses.py | 0 .../telegram_bot/webhook/commands/about.py | 2 +- .../webhook/commands/create_folder.py | 16 +++++++------- .../telegram_bot/webhook/commands/help.py | 2 +- .../telegram_bot/webhook/commands/publish.py | 16 +++++++------- .../telegram_bot/webhook/commands/settings.py | 4 ++-- .../telegram_bot/webhook/commands/space.py | 14 ++++++------- .../telegram_bot/webhook/commands/unknown.py | 2 +- .../webhook/commands/unpublish.py | 16 +++++++------- .../telegram_bot/webhook/commands/upload.py | 21 +++++++++---------- .../telegram_bot/webhook/commands/yd_auth.py | 10 ++++----- .../webhook/commands/yd_revoke.py | 8 +++---- .../telegram_bot/webhook/dispatcher.py | 18 ++++++++-------- src/blueprints/telegram_bot/webhook/views.py | 2 +- src/blueprints/telegram_bot/yd_auth/views.py | 4 ++-- 25 files changed, 70 insertions(+), 72 deletions(-) rename src/blueprints/{ => _common}/utils.py (100%) rename src/blueprints/telegram_bot/{webhook/commands/common/names.py => _common/command_names.py} (100%) rename src/blueprints/telegram_bot/{webhook => _common}/stateful_chat.py (100%) rename src/blueprints/telegram_bot/{webhook => _common}/telegram_interface.py (100%) rename src/blueprints/telegram_bot/{webhook/commands/common/yandex_api.py => _common/yandex_disk.py} (100%) rename src/blueprints/telegram_bot/webhook/commands/{common => _common}/__init__.py (100%) rename src/blueprints/telegram_bot/webhook/commands/{common => _common}/decorators.py (97%) rename src/blueprints/telegram_bot/webhook/commands/{common => _common}/responses.py (100%) diff --git a/src/app.py b/src/app.py index d9b1446..beaf55c 100644 --- a/src/app.py +++ b/src/app.py @@ -21,7 +21,7 @@ telegram_bot_blueprint, legal_blueprint ) -from .blueprints.utils import ( +from .blueprints._common.utils import ( absolute_url_for ) # we need to import every model in order Migrate knows them diff --git a/src/blueprints/utils.py b/src/blueprints/_common/utils.py similarity index 100% rename from src/blueprints/utils.py rename to src/blueprints/_common/utils.py diff --git a/src/blueprints/legal/views.py b/src/blueprints/legal/views.py index 3e04e0d..67245ea 100644 --- a/src/blueprints/legal/views.py +++ b/src/blueprints/legal/views.py @@ -1,6 +1,6 @@ from flask import redirect, url_for -from src.blueprints.utils import absolute_url_for +from src.blueprints._common.utils import absolute_url_for from src.blueprints.legal import legal_blueprint as bp diff --git a/src/blueprints/telegram_bot/webhook/commands/common/names.py b/src/blueprints/telegram_bot/_common/command_names.py similarity index 100% rename from src/blueprints/telegram_bot/webhook/commands/common/names.py rename to src/blueprints/telegram_bot/_common/command_names.py diff --git a/src/blueprints/telegram_bot/webhook/stateful_chat.py b/src/blueprints/telegram_bot/_common/stateful_chat.py similarity index 100% rename from src/blueprints/telegram_bot/webhook/stateful_chat.py rename to src/blueprints/telegram_bot/_common/stateful_chat.py diff --git a/src/blueprints/telegram_bot/webhook/telegram_interface.py b/src/blueprints/telegram_bot/_common/telegram_interface.py similarity index 100% rename from src/blueprints/telegram_bot/webhook/telegram_interface.py rename to src/blueprints/telegram_bot/_common/telegram_interface.py diff --git a/src/blueprints/telegram_bot/webhook/commands/common/yandex_api.py b/src/blueprints/telegram_bot/_common/yandex_disk.py similarity index 100% rename from src/blueprints/telegram_bot/webhook/commands/common/yandex_api.py rename to src/blueprints/telegram_bot/_common/yandex_disk.py diff --git a/src/blueprints/telegram_bot/webhook/commands/__init__.py b/src/blueprints/telegram_bot/webhook/commands/__init__.py index ab4f391..429500c 100644 --- a/src/blueprints/telegram_bot/webhook/commands/__init__.py +++ b/src/blueprints/telegram_bot/webhook/commands/__init__.py @@ -22,7 +22,6 @@ """ -from .common.names import CommandName from .unknown import handle as unknown_handler from .help import handle as help_handler from .about import handle as about_handler diff --git a/src/blueprints/telegram_bot/webhook/commands/common/__init__.py b/src/blueprints/telegram_bot/webhook/commands/_common/__init__.py similarity index 100% rename from src/blueprints/telegram_bot/webhook/commands/common/__init__.py rename to src/blueprints/telegram_bot/webhook/commands/_common/__init__.py diff --git a/src/blueprints/telegram_bot/webhook/commands/common/decorators.py b/src/blueprints/telegram_bot/webhook/commands/_common/decorators.py similarity index 97% rename from src/blueprints/telegram_bot/webhook/commands/common/decorators.py rename to src/blueprints/telegram_bot/webhook/commands/_common/decorators.py index 4c80324..a18fc3a 100644 --- a/src/blueprints/telegram_bot/webhook/commands/common/decorators.py +++ b/src/blueprints/telegram_bot/webhook/commands/_common/decorators.py @@ -13,8 +13,8 @@ ChatType ) from src.localization import SupportedLanguage +from src.blueprints.telegram_bot._common.command_names import CommandName from .responses import cancel_command -from .names import CommandName def register_guest(func): diff --git a/src/blueprints/telegram_bot/webhook/commands/common/responses.py b/src/blueprints/telegram_bot/webhook/commands/_common/responses.py similarity index 100% rename from src/blueprints/telegram_bot/webhook/commands/common/responses.py rename to src/blueprints/telegram_bot/webhook/commands/_common/responses.py diff --git a/src/blueprints/telegram_bot/webhook/commands/about.py b/src/blueprints/telegram_bot/webhook/commands/about.py index ad9d87c..4d5f2e5 100644 --- a/src/blueprints/telegram_bot/webhook/commands/about.py +++ b/src/blueprints/telegram_bot/webhook/commands/about.py @@ -1,7 +1,7 @@ from flask import g, current_app from src.api import telegram -from src.blueprints.utils import absolute_url_for +from src.blueprints._common.utils import absolute_url_for def handle(*args, **kwargs): diff --git a/src/blueprints/telegram_bot/webhook/commands/create_folder.py b/src/blueprints/telegram_bot/webhook/commands/create_folder.py index c8ba075..7a0a3b0 100644 --- a/src/blueprints/telegram_bot/webhook/commands/create_folder.py +++ b/src/blueprints/telegram_bot/webhook/commands/create_folder.py @@ -1,21 +1,21 @@ from flask import g from src.api import telegram -from .common.responses import ( +from src.blueprints.telegram_bot._common.yandex_disk import ( + create_folder, + YandexAPICreateFolderError, + YandexAPIRequestError +) +from ._common.responses import ( cancel_command, abort_command, AbortReason ) -from .common.decorators import ( +from ._common.decorators import ( yd_access_token_required, get_db_data ) -from .common.yandex_api import ( - create_folder, - YandexAPICreateFolderError, - YandexAPIRequestError -) -from . import CommandName +from src.blueprints.telegram_bot._common.command_names import CommandName @yd_access_token_required diff --git a/src/blueprints/telegram_bot/webhook/commands/help.py b/src/blueprints/telegram_bot/webhook/commands/help.py index 3897bd1..e188270 100644 --- a/src/blueprints/telegram_bot/webhook/commands/help.py +++ b/src/blueprints/telegram_bot/webhook/commands/help.py @@ -3,7 +3,7 @@ from flask import g, current_app from src.api import telegram -from . import CommandName +from src.blueprints.telegram_bot._common.command_names import CommandName def handle(*args, **kwargs): diff --git a/src/blueprints/telegram_bot/webhook/commands/publish.py b/src/blueprints/telegram_bot/webhook/commands/publish.py index b86b62e..8636b60 100644 --- a/src/blueprints/telegram_bot/webhook/commands/publish.py +++ b/src/blueprints/telegram_bot/webhook/commands/publish.py @@ -1,21 +1,21 @@ from flask import g from src.api import telegram -from .common.responses import ( +from src.blueprints.telegram_bot._common.yandex_disk import ( + publish_item, + YandexAPIPublishItemError, + YandexAPIRequestError +) +from ._common.responses import ( cancel_command, abort_command, AbortReason ) -from .common.decorators import ( +from ._common.decorators import ( yd_access_token_required, get_db_data ) -from .common.yandex_api import ( - publish_item, - YandexAPIPublishItemError, - YandexAPIRequestError -) -from . import CommandName +from src.blueprints.telegram_bot._common.command_names import CommandName @yd_access_token_required diff --git a/src/blueprints/telegram_bot/webhook/commands/settings.py b/src/blueprints/telegram_bot/webhook/commands/settings.py index 705757a..0c6d871 100644 --- a/src/blueprints/telegram_bot/webhook/commands/settings.py +++ b/src/blueprints/telegram_bot/webhook/commands/settings.py @@ -1,11 +1,11 @@ from flask import g from src.api import telegram -from .common.decorators import ( +from ._common.decorators import ( register_guest, get_db_data ) -from .common.responses import ( +from ._common.responses import ( request_private_chat ) diff --git a/src/blueprints/telegram_bot/webhook/commands/space.py b/src/blueprints/telegram_bot/webhook/commands/space.py index 874a650..8e8db8d 100644 --- a/src/blueprints/telegram_bot/webhook/commands/space.py +++ b/src/blueprints/telegram_bot/webhook/commands/space.py @@ -7,19 +7,19 @@ from plotly.io import to_image from src.api import telegram -from .common.responses import ( +from src.blueprints.telegram_bot._common.yandex_disk import ( + get_disk_info, + YandexAPIRequestError +) +from ._common.responses import ( cancel_command, AbortReason ) -from .common.decorators import ( +from ._common.decorators import ( yd_access_token_required, get_db_data ) -from .common.yandex_api import ( - get_disk_info, - YandexAPIRequestError -) -from . import CommandName +from src.blueprints.telegram_bot._common.command_names import CommandName @yd_access_token_required diff --git a/src/blueprints/telegram_bot/webhook/commands/unknown.py b/src/blueprints/telegram_bot/webhook/commands/unknown.py index 8e22365..dcb760f 100644 --- a/src/blueprints/telegram_bot/webhook/commands/unknown.py +++ b/src/blueprints/telegram_bot/webhook/commands/unknown.py @@ -1,7 +1,7 @@ from flask import g from src.api import telegram -from . import CommandName +from src.blueprints.telegram_bot._common.command_names import CommandName def handle(*args, **kwargs): diff --git a/src/blueprints/telegram_bot/webhook/commands/unpublish.py b/src/blueprints/telegram_bot/webhook/commands/unpublish.py index bbdd44f..d85c936 100644 --- a/src/blueprints/telegram_bot/webhook/commands/unpublish.py +++ b/src/blueprints/telegram_bot/webhook/commands/unpublish.py @@ -1,21 +1,21 @@ from flask import g from src.api import telegram -from .common.responses import ( +from src.blueprints.telegram_bot._common.yandex_disk import ( + unpublish_item, + YandexAPIUnpublishItemError, + YandexAPIRequestError +) +from ._common.responses import ( cancel_command, abort_command, AbortReason ) -from .common.decorators import ( +from ._common.decorators import ( yd_access_token_required, get_db_data ) -from .common.yandex_api import ( - unpublish_item, - YandexAPIUnpublishItemError, - YandexAPIRequestError -) -from . import CommandName +from src.blueprints.telegram_bot._common.command_names import CommandName @yd_access_token_required diff --git a/src/blueprints/telegram_bot/webhook/commands/upload.py b/src/blueprints/telegram_bot/webhook/commands/upload.py index bc70231..7b6f833 100644 --- a/src/blueprints/telegram_bot/webhook/commands/upload.py +++ b/src/blueprints/telegram_bot/webhook/commands/upload.py @@ -4,24 +4,23 @@ from flask import g, current_app from src.api import telegram -from src.blueprints.telegram_bot.webhook import telegram_interface -from .common.decorators import ( +from src.blueprints.telegram_bot._common import telegram_interface +from src.blueprints.telegram_bot._common.yandex_disk import ( + upload_file_with_url, + YandexAPIRequestError, + YandexAPICreateFolderError, + YandexAPIUploadFileError, + YandexAPIExceededNumberOfStatusChecksError +) +from ._common.decorators import ( yd_access_token_required, get_db_data ) -from .common.responses import ( +from ._common.responses import ( abort_command, cancel_command, AbortReason ) -from .common.yandex_api import ( - upload_file_with_url, - YandexAPIRequestError, - YandexAPICreateFolderError, - YandexAPIUploadFileError, - YandexAPIExceededNumberOfStatusChecksError -) - class MessageHealth: """ diff --git a/src/blueprints/telegram_bot/webhook/commands/yd_auth.py b/src/blueprints/telegram_bot/webhook/commands/yd_auth.py index b5920b8..067d9b9 100644 --- a/src/blueprints/telegram_bot/webhook/commands/yd_auth.py +++ b/src/blueprints/telegram_bot/webhook/commands/yd_auth.py @@ -3,12 +3,12 @@ from src.api import telegram from src.extensions import redis_client from src.configs.flask import YandexOAuthAPIMethod -from src.blueprints.utils import ( +from src.blueprints._common.utils import ( absolute_url_for, get_current_datetime ) from src.blueprints.telegram_bot._common import yandex_oauth -from src.blueprints.telegram_bot.webhook.stateful_chat import ( +from src.blueprints.telegram_bot._common.stateful_chat import ( set_disposable_handler, set_user_chat_data, get_user_chat_data, @@ -18,15 +18,15 @@ DispatcherEvent, RouteSource ) -from .common.decorators import ( +from ._common.decorators import ( register_guest, get_db_data ) -from .common.responses import ( +from ._common.responses import ( request_private_chat, cancel_command ) -from . import CommandName +from src.blueprints.telegram_bot._common.command_names import CommandName @register_guest diff --git a/src/blueprints/telegram_bot/webhook/commands/yd_revoke.py b/src/blueprints/telegram_bot/webhook/commands/yd_revoke.py index 7c9f9cb..20ad565 100644 --- a/src/blueprints/telegram_bot/webhook/commands/yd_revoke.py +++ b/src/blueprints/telegram_bot/webhook/commands/yd_revoke.py @@ -2,17 +2,17 @@ from src.extensions import db from src.api import telegram -from src.blueprints.utils import get_current_datetime +from src.blueprints._common.utils import get_current_datetime from src.blueprints.telegram_bot._common.yandex_oauth import YandexOAuthClient -from .common.decorators import ( +from ._common.decorators import ( get_db_data, register_guest ) -from .common.responses import ( +from ._common.responses import ( request_private_chat, cancel_command ) -from . import CommandName +from src.blueprints.telegram_bot._common.command_names import CommandName class YandexOAuthRemoveClient(YandexOAuthClient): diff --git a/src/blueprints/telegram_bot/webhook/dispatcher.py b/src/blueprints/telegram_bot/webhook/dispatcher.py index 6454139..5ed1574 100644 --- a/src/blueprints/telegram_bot/webhook/dispatcher.py +++ b/src/blueprints/telegram_bot/webhook/dispatcher.py @@ -7,20 +7,20 @@ import traceback from src.extensions import redis_client +from src.blueprints.telegram_bot._common.stateful_chat import ( + get_disposable_handler, + delete_disposable_handler, + get_subscribed_handlers +) +from src.blueprints.telegram_bot._common.telegram_interface import ( + Message as TelegramMessage +) +from src.blueprints.telegram_bot._common.command_names import CommandName from . import commands -from .commands import CommandName from .dispatcher_events import ( DispatcherEvent, RouteSource ) -from .telegram_interface import ( - Message as TelegramMessage -) -from .stateful_chat import ( - get_disposable_handler, - delete_disposable_handler, - get_subscribed_handlers -) def intellectual_dispatch( diff --git a/src/blueprints/telegram_bot/webhook/views.py b/src/blueprints/telegram_bot/webhook/views.py index 8fe876f..c1219dd 100644 --- a/src/blueprints/telegram_bot/webhook/views.py +++ b/src/blueprints/telegram_bot/webhook/views.py @@ -5,7 +5,7 @@ ) from src.blueprints.telegram_bot import telegram_bot_blueprint as bp -from . import telegram_interface +from src.blueprints.telegram_bot._common import telegram_interface from .dispatcher import intellectual_dispatch, direct_dispatch diff --git a/src/blueprints/telegram_bot/yd_auth/views.py b/src/blueprints/telegram_bot/yd_auth/views.py index 0f51398..49121ca 100644 --- a/src/blueprints/telegram_bot/yd_auth/views.py +++ b/src/blueprints/telegram_bot/yd_auth/views.py @@ -6,10 +6,10 @@ from src.api import telegram from src.database import ChatQuery -from src.blueprints.utils import get_current_datetime +from src.blueprints._common.utils import get_current_datetime from src.blueprints.telegram_bot import telegram_bot_blueprint as bp from src.blueprints.telegram_bot._common import yandex_oauth -from src.blueprints.telegram_bot.webhook.commands import CommandName +from src.blueprints.telegram_bot._common.command_names import CommandName @bp.route("/yandex_disk_authorization") From 0f39b218d0abb03defc3694638e3a8500e5c18bf Mon Sep 17 00:00:00 2001 From: Sergey Kuznetsov Date: Wed, 18 Nov 2020 19:02:32 +0300 Subject: [PATCH 041/103] Fix a typo --- src/api/telegram/requests.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/api/telegram/requests.py b/src/api/telegram/requests.py index 761836e..4f19571 100644 --- a/src/api/telegram/requests.py +++ b/src/api/telegram/requests.py @@ -90,7 +90,7 @@ def make_request( if not ok: raise RequestFailed( create_error_text( - result.content + result["content"] ) ) From f171b71c5f79dd595ecf815ccd9a37baa59df654 Mon Sep 17 00:00:00 2001 From: Sergey Kuznetsov Date: Wed, 18 Nov 2020 19:09:46 +0300 Subject: [PATCH 042/103] Handle cases when Telegram returns 4xx or 5xx response --- CHANGELOG.md | 1 + src/api/request.py | 2 +- src/api/telegram/requests.py | 1 + src/blueprints/telegram_bot/webhook/views.py | 8 ++++++++ 4 files changed, 11 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 980f55e..0399bdd 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -37,6 +37,7 @@ ### Fixed - A bug when new user (didn't use any command before) used `/revoke_access` command and it led to request crash (500). +- Situation: Telegram send an update, the server sent back a 500; Telegram will send same update again and again until it get 200 from a server, but server always returns 500. Such sitations can occur, for example, when user initiated a command and blocked the bot - bot can't send message to user in this case (it gets 403 from Telegram API, so, server raises error because it is an unexpected error and should be logged). Now it is fixed and the bot always send back 200, even for such error situations. # 1.1.0 (May 9, 2020) diff --git a/src/api/request.py b/src/api/request.py index a8ee217..9768244 100644 --- a/src/api/request.py +++ b/src/api/request.py @@ -23,7 +23,7 @@ class RequestResult(typing.TypedDict): def request( - raise_for_status=True, + raise_for_status=False, content_type: CONTENT_TYPE = "none", **kwargs ) -> RequestResult: diff --git a/src/api/telegram/requests.py b/src/api/telegram/requests.py index 4f19571..e05ecd8 100644 --- a/src/api/telegram/requests.py +++ b/src/api/telegram/requests.py @@ -87,6 +87,7 @@ def make_request( ) ok = result["content"]["ok"] + # 4xx or 5xx if not ok: raise RequestFailed( create_error_text( diff --git a/src/blueprints/telegram_bot/webhook/views.py b/src/blueprints/telegram_bot/webhook/views.py index c1219dd..449e473 100644 --- a/src/blueprints/telegram_bot/webhook/views.py +++ b/src/blueprints/telegram_bot/webhook/views.py @@ -42,6 +42,14 @@ def webhook(): g.telegram_chat = message.get_chat() g.direct_dispatch = direct_dispatch + # We call this handler and do not handle any errors. + # We assume that all errors already was handeld by + # handlers, loggers, etc. + # WARNING: in case of any exceptions there will be + # 500 from a server. Telegram will send user message + # again and again until it get 200 from a server. + # So, it is important to always return 200 or return + # 500 and expect same message again intellectual_dispatch(message)() return make_success_response() From 6ac118265063018ef42d4a4550d20715bbf941a7 Mon Sep 17 00:00:00 2001 From: Sergey Kuznetsov Date: Wed, 18 Nov 2020 19:12:03 +0300 Subject: [PATCH 043/103] Fix flake8 errors --- src/blueprints/telegram_bot/webhook/commands/upload.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/blueprints/telegram_bot/webhook/commands/upload.py b/src/blueprints/telegram_bot/webhook/commands/upload.py index 7b6f833..2534389 100644 --- a/src/blueprints/telegram_bot/webhook/commands/upload.py +++ b/src/blueprints/telegram_bot/webhook/commands/upload.py @@ -22,6 +22,7 @@ AbortReason ) + class MessageHealth: """ Health status of Telegram message. From 0edee5002e5d746408b3849e8c7503c2695fc8bf Mon Sep 17 00:00:00 2001 From: Sergey Kuznetsov Date: Thu, 19 Nov 2020 13:25:12 +0300 Subject: [PATCH 044/103] Add runtime configuration --- src/configs/flask.py | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/src/configs/flask.py b/src/configs/flask.py index c23a185..7cd34e8 100644 --- a/src/configs/flask.py +++ b/src/configs/flask.py @@ -52,6 +52,10 @@ class YandexOAuthAPIMethod(Enum): class Config: """ Notes: + - don't remove any key from configuration, because code logic + may depend on this. Instead set disable value (if code logic + supports it); or set empty value and edit code logic to handle + such values. - keep in mind that Heroku have 30 seconds request timeout. So, if your configuration value can exceed 30 seconds, then request will be terminated by Heroku. @@ -66,6 +70,19 @@ class Config: PROJECT_URL_FOR_QUESTION = "https://github.com/Amaimersion/yandex-disk-telegram-bot/issues/new?template=question.md" # noqa PROJECT_URL_FOR_BOT = "https://t.me/Ya_Disk_Bot" + # Runtime (interaction of bot with user, behavior of bot) + # Default value (in seconds) when setted but unused + # disposable handler should expire and be removed. + # Example: user send `/create_folder` but didn't send + # any data for that command; bot will handle next message + # as needed data for that command; if user don't send any + # data in 10 minutes, then this handler will be removed from queue. + # Keep in mind that it is only recommended default value, + # specific handler can use it own expiration time and ignore + # this value at all. + # Set to 0 to disable expiration + RUNTIME_DISPOSABLE_HANDLER_EXPIRE = 60 * 10 + # Flask DEBUG = False TESTING = False From 7db87989ad0fe094074ba3d0d3a5fa5e88e50ef5 Mon Sep 17 00:00:00 2001 From: Sergey Kuznetsov Date: Thu, 19 Nov 2020 13:27:18 +0300 Subject: [PATCH 045/103] Refactoring of /create_folder --- CHANGELOG.md | 9 ++ .../telegram_bot/_common/yandex_disk.py | 2 +- .../webhook/commands/create_folder.py | 131 +++++++++++++----- 3 files changed, 106 insertions(+), 36 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 0399bdd..4e134a1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -13,6 +13,15 @@ - `/space`: getting of information about remaining Yandex.Disk space. - Handling of files that exceed file size limit in 20 MB. At the moment the bot will asnwer with warning message, not trying to upload that file. [#3](https://github.com/Amaimersion/yandex-disk-telegram-bot/issues/3) +### Changed + +- `/create_folder`: now it will wait for folder name if you send empty command, not deny operation. + +### Fixed + +- `/create_folder`: fixed a bug when bot could remove `/create_folder` occurrences from folder name. +- `/create_folder`: fixed a bug when bot don't send any responses on invalid folder name. + ## Project ### Improved diff --git a/src/blueprints/telegram_bot/_common/yandex_disk.py b/src/blueprints/telegram_bot/_common/yandex_disk.py index 60183ff..7dd9d7f 100644 --- a/src/blueprints/telegram_bot/_common/yandex_disk.py +++ b/src/blueprints/telegram_bot/_common/yandex_disk.py @@ -117,7 +117,7 @@ def create_folder( continue raise YandexAPICreateFolderError( - create_yandex_error_text(result) + create_yandex_error_text(response) ) return last_status_code diff --git a/src/blueprints/telegram_bot/webhook/commands/create_folder.py b/src/blueprints/telegram_bot/webhook/commands/create_folder.py index 7a0a3b0..104cd00 100644 --- a/src/blueprints/telegram_bot/webhook/commands/create_folder.py +++ b/src/blueprints/telegram_bot/webhook/commands/create_folder.py @@ -1,4 +1,4 @@ -from flask import g +from flask import g, current_app from src.api import telegram from src.blueprints.telegram_bot._common.yandex_disk import ( @@ -6,39 +6,61 @@ YandexAPICreateFolderError, YandexAPIRequestError ) -from ._common.responses import ( - cancel_command, - abort_command, - AbortReason +from src.blueprints.telegram_bot._common.command_names import ( + CommandName ) +from src.blueprints.telegram_bot._common.stateful_chat import ( + set_disposable_handler +) +from src.blueprints.telegram_bot.webhook.dispatcher_events import ( + DispatcherEvent, + RouteSource +) +from ._common.responses import cancel_command from ._common.decorators import ( yd_access_token_required, get_db_data ) -from src.blueprints.telegram_bot._common.command_names import CommandName @yd_access_token_required @get_db_data def handle(*args, **kwargs): - """ - Handles `/create_folder` command. - """ - message = g.telegram_message - user = g.db_user - chat = g.db_chat - message_text = message.get_text() - folder_name = message_text.replace( - CommandName.CREATE_FOLDER.value, - "" - ).strip() - - if not (folder_name): - return abort_command( - chat.telegram_id, - AbortReason.NO_SUITABLE_DATA + message = kwargs.get( + "message", + g.telegram_message + ) + user_id = kwargs.get( + "user_id", + g.telegram_user.id + ) + chat_id = kwargs.get( + "chat_id", + g.telegram_chat.id + ) + folder_name = get_folder_name( + message, + kwargs.get("route_source") + ) + + if not folder_name: + set_disposable_handler( + user_id, + chat_id, + CommandName.CREATE_FOLDER.value, + [ + DispatcherEvent.PLAIN_TEXT.value, + DispatcherEvent.BOT_COMMAND.value, + DispatcherEvent.EMAIL.value, + DispatcherEvent.HASHTAG.value, + DispatcherEvent.URL.value + ], + current_app.config["RUNTIME_DISPOSABLE_HANDLER_EXPIRE"] ) + return message_wait_for_text(chat_id) + + user = g.db_user access_token = user.yandex_disk_token.get_access_token() last_status_code = None @@ -48,19 +70,13 @@ def handle(*args, **kwargs): folder_name=folder_name ) except YandexAPIRequestError as error: - print(error) - return cancel_command(chat.telegram_id) + cancel_command(chat_id) + raise error except YandexAPICreateFolderError as error: - error_text = ( - str(error) or - "Unknown Yandex.Disk error" - ) - - telegram.send_message( - chat_id=chat.telegram_id, - text=error_text - ) + message_yandex_error(chat_id, str(error)) + # it is expected error and should be + # logged only to user return text = None @@ -70,9 +86,54 @@ def handle(*args, **kwargs): elif (last_status_code == 409): text = "Already exists" else: - text = f"Unknown status code: {last_status_code}" + text = f"Unknown operation status: {last_status_code}" telegram.send_message( - chat_id=chat.telegram_id, + chat_id=chat_id, text=text ) + + +def get_folder_name( + telegram_message, + route_source: RouteSource +) -> str: + folder_name = telegram_message.get_text() + + # On "Disposable handler" route we expect pure text, + # in other cases we expect bot command as start of a message + if (route_source != RouteSource.DISPOSABLE_HANDLER): + folder_name = folder_name.replace( + CommandName.CREATE_FOLDER.value, + "", + 1 + ).strip() + + return folder_name + + +def message_wait_for_text(chat_id: int) -> None: + telegram.send_message( + chat_id=chat_id, + parse_mode="HTML", + text=( + "Send a folder name to create." + "\n\n" + "It should starts from root directory, " + "nested folders should be separated with " + '"/" character. ' + "In short, i expect a full path." + "\n\n" + "Example: Telegram Bot/kittens and raccoons" + ) + ) + + +def message_yandex_error(chat_id: int, error_text: str) -> None: + telegram.send_message( + chat_id=chat_id, + text=( + error_text or + "Unknown Yandex.Disk error" + ) + ) From 811503157bd5bc5b874e586c4deac4f744b07c35 Mon Sep 17 00:00:00 2001 From: Sergey Kuznetsov Date: Thu, 19 Nov 2020 13:29:04 +0300 Subject: [PATCH 046/103] Refactoring of /about --- src/blueprints/telegram_bot/webhook/commands/about.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/blueprints/telegram_bot/webhook/commands/about.py b/src/blueprints/telegram_bot/webhook/commands/about.py index 4d5f2e5..e904cd0 100644 --- a/src/blueprints/telegram_bot/webhook/commands/about.py +++ b/src/blueprints/telegram_bot/webhook/commands/about.py @@ -9,7 +9,10 @@ def handle(*args, **kwargs): Handles `/about` command. """ telegram.send_message( - chat_id=g.telegram_chat.id, + chat_id=kwargs.get( + "chat_id", + g.telegram_chat.id + ), disable_web_page_preview=True, text=( "I'm free and open-source bot that allows " From e2e548579e915be89693d23812da1c9e107f8d68 Mon Sep 17 00:00:00 2001 From: Sergey Kuznetsov Date: Thu, 19 Nov 2020 13:32:21 +0300 Subject: [PATCH 047/103] Refactoring of help message --- src/blueprints/telegram_bot/webhook/commands/help.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/src/blueprints/telegram_bot/webhook/commands/help.py b/src/blueprints/telegram_bot/webhook/commands/help.py index e188270..917e407 100644 --- a/src/blueprints/telegram_bot/webhook/commands/help.py +++ b/src/blueprints/telegram_bot/webhook/commands/help.py @@ -63,7 +63,7 @@ def handle(*args, **kwargs): "Folder name should starts from root, " f'nested folders should be separated with "{to_code("/")}" character.' "\n" - f"{CommandName.SPACE.value} — get information about remaining Yandex.Disk space. " + f"{CommandName.SPACE.value} — get information about remaining space. " "\n\n" "Yandex.Disk Access" "\n" @@ -80,8 +80,12 @@ def handle(*args, **kwargs): f"{CommandName.ABOUT.value} — read about me" ) + chat_id = kwargs.get( + "chat_id", + g.telegram_chat.id + ) telegram.send_message( - chat_id=g.telegram_chat.id, + chat_id=chat_id, parse_mode="HTML", text=text ) From 628ef33a401ca3c83501f1486140e7eca2cfb6b3 Mon Sep 17 00:00:00 2001 From: Sergey Kuznetsov Date: Thu, 19 Nov 2020 18:03:46 +0300 Subject: [PATCH 048/103] Refactoring of command files --- .../telegram_bot/_common/yandex_disk.py | 10 +-- .../webhook/commands/_common/responses.py | 66 +++++++++++++++++++ .../webhook/commands/_common/utils.py | 44 +++++++++++++ .../webhook/commands/create_folder.py | 62 ++++------------- 4 files changed, 126 insertions(+), 56 deletions(-) create mode 100644 src/blueprints/telegram_bot/webhook/commands/_common/utils.py diff --git a/src/blueprints/telegram_bot/_common/yandex_disk.py b/src/blueprints/telegram_bot/_common/yandex_disk.py index 7dd9d7f..c422ac7 100644 --- a/src/blueprints/telegram_bot/_common/yandex_disk.py +++ b/src/blueprints/telegram_bot/_common/yandex_disk.py @@ -319,17 +319,17 @@ def create_yandex_error_text(data: dict) -> str: """ :returns: Human error message from Yandex error response. """ - error_name = data["error"] + error_name = data.get( + "error", + "?" + ) error_description = ( data.get("message") or data.get("description") or "?" ) - return ( - "Yandex.Disk Error: " - f"{error_name} ({error_description})" - ) + return (f"{error_name}: {error_description}") def yandex_operation_is_completed(data: dict) -> bool: diff --git a/src/blueprints/telegram_bot/webhook/commands/_common/responses.py b/src/blueprints/telegram_bot/webhook/commands/_common/responses.py index 7f88d93..bf4ab08 100644 --- a/src/blueprints/telegram_bot/webhook/commands/_common/responses.py +++ b/src/blueprints/telegram_bot/webhook/commands/_common/responses.py @@ -118,3 +118,69 @@ def request_private_chat(chat_telegram_id: int) -> None: "After that repeat your request." ) ) + + +def send_yandex_disk_error( + chat_telegram_id: int, + error_text: str +) -> None: + """ + Sends a message that indicates that Yandex.Disk threw an error. + + :param error_text: + Text of error that will be printed. + Can be empty. + """ + telegram.send_message( + chat_id=chat_telegram_id, + parse_mode="HTML", + text=( + "Yandex.Disk Error" + "\n\n" + f"{error_text or 'Unknown'}" + ) + ) + + +def request_absolute_path(chat_telegram_id: int) -> None: + """ + Sends a message that asks a user to send an + absolute path (folder or file). + """ + telegram.send_message( + chat_id=chat_telegram_id, + parse_mode="HTML", + text=( + "Send a path." + "\n\n" + "It should starts from root directory, " + "nested folders should be separated with " + '"/" character. ' + "In short, i expect an absolute path to the item." + "\n\n" + "Example: Telegram Bot/kittens and raccoons" + "\n" + "Example: /Telegram Bot/kittens and raccoons/musya.jpg" # noqa + ) + ) + + +def request_absolute_folder_name(chat_telegram_id: int) -> None: + """ + Sends a message that asks a user to send an + absolute path of folder. + """ + telegram.send_message( + chat_id=chat_telegram_id, + parse_mode="HTML", + text=( + "Send a folder name." + "\n\n" + "It should starts from root directory, " + "nested folders should be separated with " + '"/" character. ' + "In short, i expect a full path." + "\n\n" + "Example: Telegram Bot/kittens and raccoons" + ) + ) diff --git a/src/blueprints/telegram_bot/webhook/commands/_common/utils.py b/src/blueprints/telegram_bot/webhook/commands/_common/utils.py new file mode 100644 index 0000000..5c0f0a6 --- /dev/null +++ b/src/blueprints/telegram_bot/webhook/commands/_common/utils.py @@ -0,0 +1,44 @@ +from typing import Union + +from src.blueprints.telegram_bot._common.telegram_interface import ( + Message as TelegramMessage +) +from src.blueprints.telegram_bot.webhook.dispatcher_events import ( + RouteSource +) + + +def extract_absolute_path( + message: TelegramMessage, + bot_command: str, + route_source: Union[RouteSource, None], +) -> str: + """ + Extracts absolute path from Telegram message. + It supports both: folders and files. + + :param message: + Incoming Telegram message. + :param bot_command: + Bot command which will be removed from message. + :param route_source: + It is dispatcher parameter, see it documentation. + You should always pass it, even if it is `None`. + Bot command will be deleted from start of a message + when it is equal to `DISPOSABLE_HANDLER`. + + :returns: + Extracted absolute path. Can be empty. + """ + path = message.get_text() + + # On "Disposable handler" route we expect pure text, + # in other cases we expect bot command as start of a message + if (route_source != RouteSource.DISPOSABLE_HANDLER): + path = path.replace( + bot_command, + "", + 1 + ).strip() + + return path diff --git a/src/blueprints/telegram_bot/webhook/commands/create_folder.py b/src/blueprints/telegram_bot/webhook/commands/create_folder.py index 104cd00..09dbb0c 100644 --- a/src/blueprints/telegram_bot/webhook/commands/create_folder.py +++ b/src/blueprints/telegram_bot/webhook/commands/create_folder.py @@ -13,14 +13,18 @@ set_disposable_handler ) from src.blueprints.telegram_bot.webhook.dispatcher_events import ( - DispatcherEvent, - RouteSource + DispatcherEvent +) +from ._common.responses import ( + cancel_command, + send_yandex_disk_error, + request_absolute_folder_name ) -from ._common.responses import cancel_command from ._common.decorators import ( yd_access_token_required, get_db_data ) +from ._common.utils import extract_absolute_path @yd_access_token_required @@ -38,8 +42,9 @@ def handle(*args, **kwargs): "chat_id", g.telegram_chat.id ) - folder_name = get_folder_name( + folder_name = extract_absolute_path( message, + CommandName.CREATE_FOLDER.value, kwargs.get("route_source") ) @@ -58,7 +63,7 @@ def handle(*args, **kwargs): current_app.config["RUNTIME_DISPOSABLE_HANDLER_EXPIRE"] ) - return message_wait_for_text(chat_id) + return request_absolute_folder_name(chat_id) user = g.db_user access_token = user.yandex_disk_token.get_access_token() @@ -73,7 +78,7 @@ def handle(*args, **kwargs): cancel_command(chat_id) raise error except YandexAPICreateFolderError as error: - message_yandex_error(chat_id, str(error)) + send_yandex_disk_error(chat_id, str(error)) # it is expected error and should be # logged only to user @@ -92,48 +97,3 @@ def handle(*args, **kwargs): chat_id=chat_id, text=text ) - - -def get_folder_name( - telegram_message, - route_source: RouteSource -) -> str: - folder_name = telegram_message.get_text() - - # On "Disposable handler" route we expect pure text, - # in other cases we expect bot command as start of a message - if (route_source != RouteSource.DISPOSABLE_HANDLER): - folder_name = folder_name.replace( - CommandName.CREATE_FOLDER.value, - "", - 1 - ).strip() - - return folder_name - - -def message_wait_for_text(chat_id: int) -> None: - telegram.send_message( - chat_id=chat_id, - parse_mode="HTML", - text=( - "Send a folder name to create." - "\n\n" - "It should starts from root directory, " - "nested folders should be separated with " - '"/" character. ' - "In short, i expect a full path." - "\n\n" - "Example: Telegram Bot/kittens and raccoons" - ) - ) - - -def message_yandex_error(chat_id: int, error_text: str) -> None: - telegram.send_message( - chat_id=chat_id, - text=( - error_text or - "Unknown Yandex.Disk error" - ) - ) From 9159767992794fe40d64922fe7d740968ebd41be Mon Sep 17 00:00:00 2001 From: Sergey Kuznetsov Date: Thu, 19 Nov 2020 18:11:26 +0300 Subject: [PATCH 049/103] Refactoring of /publish command --- .../webhook/commands/_common/responses.py | 2 +- .../telegram_bot/webhook/commands/publish.py | 78 ++++++++++++------- 2 files changed, 52 insertions(+), 28 deletions(-) diff --git a/src/blueprints/telegram_bot/webhook/commands/_common/responses.py b/src/blueprints/telegram_bot/webhook/commands/_common/responses.py index bf4ab08..bf39d4b 100644 --- a/src/blueprints/telegram_bot/webhook/commands/_common/responses.py +++ b/src/blueprints/telegram_bot/webhook/commands/_common/responses.py @@ -151,7 +151,7 @@ def request_absolute_path(chat_telegram_id: int) -> None: chat_id=chat_telegram_id, parse_mode="HTML", text=( - "Send a path." + "Send a full path." "\n\n" "It should starts from root directory, " "nested folders should be separated with " diff --git a/src/blueprints/telegram_bot/webhook/commands/publish.py b/src/blueprints/telegram_bot/webhook/commands/publish.py index 8636b60..d02d12d 100644 --- a/src/blueprints/telegram_bot/webhook/commands/publish.py +++ b/src/blueprints/telegram_bot/webhook/commands/publish.py @@ -1,4 +1,4 @@ -from flask import g +from flask import g, current_app from src.api import telegram from src.blueprints.telegram_bot._common.yandex_disk import ( @@ -6,16 +6,25 @@ YandexAPIPublishItemError, YandexAPIRequestError ) +from src.blueprints.telegram_bot._common.stateful_chat import ( + set_disposable_handler +) +from src.blueprints.telegram_bot._common.command_names import ( + CommandName +) +from src.blueprints.telegram_bot.webhook.dispatcher_events import ( + DispatcherEvent +) from ._common.responses import ( cancel_command, - abort_command, - AbortReason + request_absolute_path, + send_yandex_disk_error ) from ._common.decorators import ( yd_access_token_required, get_db_data ) -from src.blueprints.telegram_bot._common.command_names import CommandName +from ._common.utils import extract_absolute_path @yd_access_token_required @@ -24,21 +33,42 @@ def handle(*args, **kwargs): """ Handles `/publish` command. """ - message = g.telegram_message - user = g.db_user - chat = g.db_chat - message_text = message.get_text() - path = message_text.replace( + message = kwargs.get( + "message", + g.telegram_message + ) + user_id = kwargs.get( + "user_id", + g.telegram_user.id + ) + chat_id = kwargs.get( + "chat_id", + g.telegram_chat.id + ) + path = extract_absolute_path( + message, CommandName.PUBLISH.value, - "" - ).strip() + kwargs.get("route_source") + ) - if not (path): - return abort_command( - chat.telegram_id, - AbortReason.NO_SUITABLE_DATA + if not path: + set_disposable_handler( + user_id, + chat_id, + CommandName.PUBLISH.value, + [ + DispatcherEvent.PLAIN_TEXT.value, + DispatcherEvent.BOT_COMMAND.value, + DispatcherEvent.EMAIL.value, + DispatcherEvent.HASHTAG.value, + DispatcherEvent.URL.value + ], + current_app.config["RUNTIME_DISPOSABLE_HANDLER_EXPIRE"] ) + return request_absolute_path(chat_id) + + user = g.db_user access_token = user.yandex_disk_token.get_access_token() try: @@ -47,22 +77,16 @@ def handle(*args, **kwargs): path ) except YandexAPIRequestError as error: - print(error) - return cancel_command(chat.telegram_id) + cancel_command(chat_id) + raise error except YandexAPIPublishItemError as error: - error_text = ( - str(error) or - "Unknown Yandex.Disk error" - ) - - telegram.send_message( - chat_id=chat.telegram_id, - text=error_text - ) + send_yandex_disk_error(chat_id, str(error)) + # it is expected error and should be + # logged only to user return telegram.send_message( - chat_id=chat.telegram_id, + chat_id=chat_id, text="Published" ) From a9cbb0776b5a073583a43185a031376c5ee05263 Mon Sep 17 00:00:00 2001 From: Sergey Kuznetsov Date: Thu, 19 Nov 2020 19:01:06 +0300 Subject: [PATCH 050/103] Refactoring of /settings command --- .../telegram_bot/webhook/commands/settings.py | 21 ++++++++++++------- 1 file changed, 13 insertions(+), 8 deletions(-) diff --git a/src/blueprints/telegram_bot/webhook/commands/settings.py b/src/blueprints/telegram_bot/webhook/commands/settings.py index 0c6d871..b4c6a9f 100644 --- a/src/blueprints/telegram_bot/webhook/commands/settings.py +++ b/src/blueprints/telegram_bot/webhook/commands/settings.py @@ -1,6 +1,9 @@ from flask import g from src.api import telegram +from src.blueprints.telegram_bot._common.yandex_oauth import ( + YandexOAuthClient +) from ._common.decorators import ( register_guest, get_db_data @@ -16,19 +19,21 @@ def handle(*args, **kwargs): """ Handles `/settings` command. """ - user = g.db_user - incoming_chat = g.db_chat private_chat = g.db_private_chat - if (private_chat is None): - return request_private_chat(incoming_chat.telegram_id) + if private_chat is None: + incoming_chat_id = kwargs.get( + "chat_id", + g.db_chat.telegram_id + ) + return request_private_chat(incoming_chat_id) + + user = g.db_user + yo_client = YandexOAuthClient() yd_access = False - if ( - (user.yandex_disk_token) and - (user.yandex_disk_token.have_access_token()) - ): + if yo_client.have_valid_access_token(user): yd_access = True text = ( From 48bc161baada665f65ef5fb39eebb85a6250ecb4 Mon Sep 17 00:00:00 2001 From: Sergey Kuznetsov Date: Thu, 19 Nov 2020 19:09:18 +0300 Subject: [PATCH 051/103] Refactoring of /space handler --- .../telegram_bot/webhook/commands/space.py | 21 ++++++++++++------- 1 file changed, 14 insertions(+), 7 deletions(-) diff --git a/src/blueprints/telegram_bot/webhook/commands/space.py b/src/blueprints/telegram_bot/webhook/commands/space.py index 8e8db8d..3fe874e 100644 --- a/src/blueprints/telegram_bot/webhook/commands/space.py +++ b/src/blueprints/telegram_bot/webhook/commands/space.py @@ -11,6 +11,9 @@ get_disk_info, YandexAPIRequestError ) +from src.blueprints.telegram_bot._common.command_names import ( + CommandName +) from ._common.responses import ( cancel_command, AbortReason @@ -19,7 +22,6 @@ yd_access_token_required, get_db_data ) -from src.blueprints.telegram_bot._common.command_names import CommandName @yd_access_token_required @@ -29,15 +31,18 @@ def handle(*args, **kwargs): Handles `/publish` command. """ user = g.db_user - chat = g.db_chat + chat_id = kwargs.get( + "chat_id", + g.telegram_chat.id + ) access_token = user.yandex_disk_token.get_access_token() disk_info = None try: disk_info = get_disk_info(access_token) except YandexAPIRequestError as error: - print(error) - return cancel_command(chat.telegram_id) + cancel_command(chat_id) + raise error current_date = get_current_date() jpeg_image = create_space_chart( @@ -46,15 +51,17 @@ def handle(*args, **kwargs): trash_size=disk_info["trash_size"], caption=current_date ) + filename = f"{to_filename(current_date)}.jpg" + file_caption = f"Yandex.Disk space at {current_date}" telegram.send_photo( - chat_id=chat.telegram_id, + chat_id=chat_id, photo=( - f"{to_filename(current_date)}.jpg", + filename, jpeg_image, "image/jpeg" ), - caption=f"Yandex.Disk space at {current_date}" + caption=file_caption ) From cc577a295a705a90be781b7da7cdd2305edf527e Mon Sep 17 00:00:00 2001 From: Sergey Kuznetsov Date: Thu, 19 Nov 2020 19:10:15 +0300 Subject: [PATCH 052/103] Refactoring of /unknown handler --- src/blueprints/telegram_bot/webhook/commands/unknown.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/blueprints/telegram_bot/webhook/commands/unknown.py b/src/blueprints/telegram_bot/webhook/commands/unknown.py index dcb760f..2feb829 100644 --- a/src/blueprints/telegram_bot/webhook/commands/unknown.py +++ b/src/blueprints/telegram_bot/webhook/commands/unknown.py @@ -9,7 +9,10 @@ def handle(*args, **kwargs): Handles unknown command. """ telegram.send_message( - chat_id=g.telegram_chat.id, + chat_id=kwargs.get( + "chat_id", + g.telegram_chat.id + ), text=( "I don't know this command. " f"See commands list or type {CommandName.HELP.value}" From f06f0d4358323d5f8314c07d8e9d7d7d387570f1 Mon Sep 17 00:00:00 2001 From: Sergey Kuznetsov Date: Thu, 19 Nov 2020 19:27:51 +0300 Subject: [PATCH 053/103] Refactoring of /unpublish handler --- .../webhook/commands/unpublish.py | 78 ++++++++++++------- 1 file changed, 51 insertions(+), 27 deletions(-) diff --git a/src/blueprints/telegram_bot/webhook/commands/unpublish.py b/src/blueprints/telegram_bot/webhook/commands/unpublish.py index d85c936..0250ba9 100644 --- a/src/blueprints/telegram_bot/webhook/commands/unpublish.py +++ b/src/blueprints/telegram_bot/webhook/commands/unpublish.py @@ -1,4 +1,4 @@ -from flask import g +from flask import g, current_app from src.api import telegram from src.blueprints.telegram_bot._common.yandex_disk import ( @@ -6,16 +6,25 @@ YandexAPIUnpublishItemError, YandexAPIRequestError ) +from src.blueprints.telegram_bot._common.stateful_chat import ( + set_disposable_handler +) +from src.blueprints.telegram_bot._common.command_names import ( + CommandName +) +from src.blueprints.telegram_bot.webhook.dispatcher_events import ( + DispatcherEvent +) from ._common.responses import ( cancel_command, - abort_command, - AbortReason + request_absolute_path, + send_yandex_disk_error ) from ._common.decorators import ( yd_access_token_required, get_db_data ) -from src.blueprints.telegram_bot._common.command_names import CommandName +from ._common.utils import extract_absolute_path @yd_access_token_required @@ -24,21 +33,42 @@ def handle(*args, **kwargs): """ Handles `/unpublish` command. """ - message = g.telegram_message - user = g.db_user - chat = g.db_chat - message_text = message.get_text() - path = message_text.replace( + message = kwargs.get( + "message", + g.telegram_message + ) + user_id = kwargs.get( + "user_id", + g.telegram_user.id + ) + chat_id = kwargs.get( + "chat_id", + g.telegram_chat.id + ) + path = extract_absolute_path( + message, CommandName.UNPUBLISH.value, - "" - ).strip() + kwargs.get("route_source") + ) - if not (path): - return abort_command( - chat.telegram_id, - AbortReason.NO_SUITABLE_DATA + if not path: + set_disposable_handler( + user_id, + chat_id, + CommandName.UNPUBLISH.value, + [ + DispatcherEvent.PLAIN_TEXT.value, + DispatcherEvent.BOT_COMMAND.value, + DispatcherEvent.EMAIL.value, + DispatcherEvent.HASHTAG.value, + DispatcherEvent.URL.value + ], + current_app.config["RUNTIME_DISPOSABLE_HANDLER_EXPIRE"] ) + return request_absolute_path(chat_id) + + user = g.db_user access_token = user.yandex_disk_token.get_access_token() try: @@ -47,22 +77,16 @@ def handle(*args, **kwargs): path ) except YandexAPIRequestError as error: - print(error) - return cancel_command(chat.telegram_id) + cancel_command(chat_id) + raise error except YandexAPIUnpublishItemError as error: - error_text = ( - str(error) or - "Unknown Yandex.Disk error" - ) - - telegram.send_message( - chat_id=chat.telegram_id, - text=error_text - ) + send_yandex_disk_error(chat_id, str(error)) + # it is expected error and should be + # logged only to user return telegram.send_message( - chat_id=chat.telegram_id, + chat_id=chat_id, text="Unpublished" ) From ce623aee2c9694c376007d9e4e30b905805c729b Mon Sep 17 00:00:00 2001 From: Sergey Kuznetsov Date: Fri, 20 Nov 2020 12:03:08 +0300 Subject: [PATCH 054/103] Fix typo in yandex_disk.py --- src/blueprints/telegram_bot/_common/yandex_disk.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/blueprints/telegram_bot/_common/yandex_disk.py b/src/blueprints/telegram_bot/_common/yandex_disk.py index c422ac7..3fd911a 100644 --- a/src/blueprints/telegram_bot/_common/yandex_disk.py +++ b/src/blueprints/telegram_bot/_common/yandex_disk.py @@ -34,7 +34,7 @@ class YandexAPICreateFolderError(Exception): class YandexAPIUploadFileError(Exception): """ - Unable to upload folder on Yandex.Disk. + Unable to upload file on Yandex.Disk. - may contain human-readable error message. """ From 77c216c3d6821b00ff2b5c19a519c14be1d6cc40 Mon Sep 17 00:00:00 2001 From: Sergey Kuznetsov Date: Fri, 20 Nov 2020 16:01:22 +0300 Subject: [PATCH 055/103] Add ISO datetime function in utils --- src/blueprints/_common/utils.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/src/blueprints/_common/utils.py b/src/blueprints/_common/utils.py index 60d4ca8..d93e833 100644 --- a/src/blueprints/_common/utils.py +++ b/src/blueprints/_common/utils.py @@ -39,3 +39,10 @@ def get_current_datetime() -> dict: "time": current_time, "timezone": current_timezone } + + +def get_current_iso_datetime(timespec="seconds") -> str: + """ + See https://docs.python.org/3.8/library/datetime.html#datetime.datetime.isoformat # noqa + """ + return datetime.now(timezone.utc).isoformat(timespec=timespec) From e8fea720b2527811e35a1a0047961eae592dab19 Mon Sep 17 00:00:00 2001 From: Sergey Kuznetsov Date: Fri, 20 Nov 2020 16:01:30 +0300 Subject: [PATCH 056/103] Refactoring of /upload handler --- CHANGELOG.md | 2 + .../webhook/commands/_common/responses.py | 20 +- .../telegram_bot/webhook/commands/help.py | 2 +- .../telegram_bot/webhook/commands/upload.py | 348 +++++++++++++----- 4 files changed, 267 insertions(+), 105 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 4e134a1..4a13a0f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,6 +12,7 @@ - `/unpublish`: unpublishing of files or folders. - `/space`: getting of information about remaining Yandex.Disk space. - Handling of files that exceed file size limit in 20 MB. At the moment the bot will asnwer with warning message, not trying to upload that file. [#3](https://github.com/Amaimersion/yandex-disk-telegram-bot/issues/3) +- Help messages for each upload command will be sended when there are no suitable input data. ### Changed @@ -21,6 +22,7 @@ - `/create_folder`: fixed a bug when bot could remove `/create_folder` occurrences from folder name. - `/create_folder`: fixed a bug when bot don't send any responses on invalid folder name. +- Wrong information in help message for `/upload_video`. ## Project diff --git a/src/blueprints/telegram_bot/webhook/commands/_common/responses.py b/src/blueprints/telegram_bot/webhook/commands/_common/responses.py index bf39d4b..9f98236 100644 --- a/src/blueprints/telegram_bot/webhook/commands/_common/responses.py +++ b/src/blueprints/telegram_bot/webhook/commands/_common/responses.py @@ -122,7 +122,8 @@ def request_private_chat(chat_telegram_id: int) -> None: def send_yandex_disk_error( chat_telegram_id: int, - error_text: str + error_text: str, + reply_to_message_id: int = None ) -> None: """ Sends a message that indicates that Yandex.Disk threw an error. @@ -130,16 +131,23 @@ def send_yandex_disk_error( :param error_text: Text of error that will be printed. Can be empty. + :param reply_to_message_id: + If specified, then sended message will be a reply message. """ - telegram.send_message( - chat_id=chat_telegram_id, - parse_mode="HTML", - text=( + kwargs = { + "chat_id": chat_telegram_id, + "parse_mode": "HTML", + "text": ( "Yandex.Disk Error" "\n\n" f"{error_text or 'Unknown'}" ) - ) + } + + if reply_to_message_id is not None: + kwargs["reply_to_message_id"] = reply_to_message_id + + telegram.send_message(**kwargs) def request_absolute_path(chat_telegram_id: int) -> None: diff --git a/src/blueprints/telegram_bot/webhook/commands/help.py b/src/blueprints/telegram_bot/webhook/commands/help.py index 917e407..8a52dba 100644 --- a/src/blueprints/telegram_bot/webhook/commands/help.py +++ b/src/blueprints/telegram_bot/webhook/commands/help.py @@ -40,7 +40,7 @@ def handle(*args, **kwargs): "You can send audio file without this command." "\n" f"{CommandName.UPLOAD_VIDEO.value} — upload a video. " - "Original name will be not saved, original type may be changed. " + "Original name will be saved, original type may be changed. " "You can send video file without this command." "\n" f"{CommandName.UPLOAD_VOICE.value} — upload a voice. " diff --git a/src/blueprints/telegram_bot/webhook/commands/upload.py b/src/blueprints/telegram_bot/webhook/commands/upload.py index 2534389..fa6a015 100644 --- a/src/blueprints/telegram_bot/webhook/commands/upload.py +++ b/src/blueprints/telegram_bot/webhook/commands/upload.py @@ -4,7 +4,9 @@ from flask import g, current_app from src.api import telegram -from src.blueprints.telegram_bot._common import telegram_interface +from src.blueprints.telegram_bot._common.telegram_interface import ( + Message as TelegramMessage +) from src.blueprints.telegram_bot._common.yandex_disk import ( upload_file_with_url, YandexAPIRequestError, @@ -19,6 +21,7 @@ from ._common.responses import ( abort_command, cancel_command, + send_yandex_disk_error, AbortReason ) @@ -33,8 +36,10 @@ def __init__( abort_reason: Union[AbortReason, None] = None ) -> None: """ - :param ok: Message is valid for subsequent handling. - :param abort_reason: Reason of abort. `None` if `ok = True`. + :param ok: + Message is valid for subsequent handling. + :param abort_reason: + Reason of abort. `None` if `ok = True`. """ self.ok = ok self.abort_reason = None @@ -47,22 +52,25 @@ class AttachmentHandler(metaclass=ABCMeta): - most of attachments will be treated as files. - some of the not abstract class functions are common for most attachments. If you need specific logic in some - function, then override this function. + function, then override it. """ def __init__(self) -> None: # Sended message to Telegram user. - # This message will be updated, rather - # than sending new message every time - self.sended_message: Union[ - telegram_interface.Message, - None - ] = None + # This message will be updated, instead + # than sending new message every time again + self.sended_message: Union[TelegramMessage, None] = None @staticmethod @abstractmethod def handle(*args, **kwargs) -> None: """ Starts uploading process. + + - `*args`, `**kwargs` - arguments from dispatcher. + + :raises: + Raises an error if any! So, this function should + be handled by top-function. """ pass @@ -70,7 +78,8 @@ def handle(*args, **kwargs) -> None: @abstractmethod def telegram_action(self) -> str: """ - :returns: Action type from + :returns: + Action type from https://core.telegram.org/bots/api/#sendchataction """ pass @@ -79,8 +88,9 @@ def telegram_action(self) -> str: @abstractmethod def raw_data_key(self) -> str: """ - :returns: Key in message, under this key - stored needed raw data. Example: 'audio'. + :returns: + Key in message, under this key stored needed raw data. + Example: `audio`. See https://core.telegram.org/bots/api#message """ pass @@ -89,42 +99,60 @@ def raw_data_key(self) -> str: @abstractmethod def raw_data_type(self) -> type: """ - :returns: Expected type of raw data. - Example: 'dict'. `None` never should be returned! + :returns: + Expected type of raw data. + Example: `dict`. + `None` never should be returned! + """ + pass + + @abstractmethod + def create_help_message(self) -> str: + """ + - supports HTML markup. + + :returns: + Help message that will be sended to user + in case of some triggers (for example, when + there are no suitable data for handling). """ pass def get_attachment( self, - message: telegram_interface.Message + message: TelegramMessage ) -> Union[dict, str, None]: """ - :param message: Incoming Telegram message. - - :returns: Attachment of message (photo object, - file object, audio object, etc.). If `None`, then - uploading should be aborted. If `dict`, it will - have `file_id` and `file_unique_id` properties. - If `str`, it should be assumed as direct file URL. + :param message: + Incoming Telegram message. + + :returns: + Attachment of message (photo object, file object, + audio object, etc.). If `None`, then uploading should + be aborted. If `dict`, it will have `file_id` and + `file_unique_id` properties. If `str`, it should be + assumed as direct file URL. See https://core.telegram.org/bots/api/#available-types """ return message.raw_data.get(self.raw_data_key) def check_message_health( self, - message: telegram_interface.Message + message: TelegramMessage ) -> MessageHealth: """ - :param message: Incoming Telegram message. + :param message: + Incoming Telegram message. - :returns: See `MessageHealth` documentation. + :returns: + See `MessageHealth` documentation. Message should be handled by next operators only in case of `ok = true`. """ health = MessageHealth(True) value = self.get_attachment(message) - if not (isinstance(value, self.raw_data_type)): + if not isinstance(value, self.raw_data_type): health.ok = False health.abort_reason = AbortReason.NO_SUITABLE_DATA elif ( @@ -148,34 +176,43 @@ def create_file_name( file: Union[dict, None] ) -> str: """ - :param attachment: Not `None` value from `self.get_attachment()`. - :param file: Representation of this attachment as a file on - Telegram servers. If `attachment` is `str`, then this will - be equal `None`. See https://core.telegram.org/bots/api/#file - - :returns: Name of file which will be uploaded. + :param attachment: + Not `None` value from `self.get_attachment()`. + :param file: + Representation of this attachment as a file on + Telegram servers. If `attachment` is `str`, then + this will be equal `None`. + See https://core.telegram.org/bots/api/#file + + :returns: + Name of file which will be uploaded. """ - if (isinstance(attachment, str)): + if isinstance(attachment, str): return attachment - name = attachment.get("file_name") or file["file_unique_id"] + name = ( + attachment.get("file_name") or + file["file_unique_id"] + ) extension = self.get_mime_type(attachment) - if (extension): + if extension: name = f"{name}.{extension}" return name def get_mime_type(self, attachment: dict) -> str: """ - :param attachment: `dict` result from `self.get_attachment()`. + :param attachment: + `dict` result from `self.get_attachment()`. - :returns: Empty string in case if `attachment` doesn't - have required key. Otherwise mime type of this attachment. + :returns: + Empty string in case if `attachment` doesn't have + required key. Otherwise mime type of this attachment. """ result = "" - if ("mime_type" in attachment): + if "mime_type" in attachment: types = attachment["mime_type"].split("/") result = types[1] @@ -185,57 +222,75 @@ def is_too_big_file(self, file: dict) -> bool: """ Checks if size of file exceeds limit size of upload. - :param file: `dict` value from `self.get_attachment()`. + :param file: + `dict` value from `self.get_attachment()`. - :returns: File size exceeds upload limit size. + :returns: + File size exceeds upload limit size. Always `False` if file size is unknown. """ limit = current_app.config["TELEGRAM_API_MAX_FILE_SIZE"] size = limit - if ("file_size" in file): + if "file_size" in file: size = file["file_size"] return (size > limit) @yd_access_token_required @get_db_data - def upload(self) -> None: + def upload(self, *args, **kwargs) -> None: """ Uploads an attachment. + + `*args`, `**kwargs` - arguments from dispatcher. """ - message = g.telegram_message - user = g.db_user - chat = g.db_chat + chat_id = kwargs.get( + "chat_id", + g.db_chat.telegram_id + ) + message = kwargs.get( + "message", + g.telegram_message + ) message_health = self.check_message_health(message) - if not (message_health.ok): - return abort_command( - chat.telegram_id, - message_health.abort_reason or AbortReason.UNKNOWN + if not message_health.ok: + reason = ( + message_health.abort_reason or + AbortReason.UNKNOWN ) + if (reason == AbortReason.NO_SUITABLE_DATA): + return self.send_html_message( + chat_id, + self.create_help_message() + ) + else: + return abort_command(chat_id, reason) + attachment = self.get_attachment(message) + data_is_empty = (attachment is None) - if (attachment is None): - return abort_command( - chat.telegram_id, - AbortReason.NO_SUITABLE_DATA + if data_is_empty: + return self.send_html_message( + chat_id, + self.create_help_message() ) try: telegram.send_chat_action( - chat_id=chat.telegram_id, + chat_id=chat_id, action=self.telegram_action ) except Exception as error: - print(error) - return cancel_command(chat.telegram_id) + cancel_command(chat_id) + raise error download_url = None file = None - if (isinstance(attachment, str)): + if isinstance(attachment, str): download_url = attachment else: result = None @@ -245,14 +300,15 @@ def upload(self) -> None: file_id=attachment["file_id"] ) except Exception as error: - print(error) - return cancel_command(chat.telegram_id) + cancel_command(chat_id) + raise error file = result["content"] download_url = telegram.create_file_download_url( file["file_path"] ) + user = g.db_user file_name = self.create_file_name(attachment, file) user_access_token = user.yandex_disk_token.get_access_token() folder_path = current_app.config[ @@ -267,67 +323,99 @@ def long_task(): file_name=file_name, download_url=download_url ): - self.send_message(f"Status: {status}") + self.reply_to_message( + message.message_id, + chat_id, + f"Status: {status}" + ) except YandexAPICreateFolderError as error: error_text = str(error) or ( "I can't create default upload folder " - "due to an unknown Yandex error." + "due to an unknown Yandex.Disk error." ) - return self.send_message(error_text) + return send_yandex_disk_error( + chat_id, + error_text, + message.message_id + ) except YandexAPIUploadFileError as error: error_text = str(error) or ( "I can't upload this due " - "to an unknown Yandex error." + "to an unknown Yandex.Disk error." ) - return self.send_message(error_text) + return send_yandex_disk_error( + chat_id, + error_text, + message.message_id + ) except YandexAPIExceededNumberOfStatusChecksError: error_text = ( "I can't track operation status of " "this anymore. Perform manual checking." ) - return self.send_message(error_text) - except (YandexAPIRequestError, Exception) as error: - print(error) - - if (self.sended_message is None): - return cancel_command( - chat.telegram_id, + return self.reply_to_message( + message.message_id, + chat_id, + error_text + ) + except Exception as error: + if self.sended_message is None: + cancel_command( + chat_id, reply_to_message=message.message_id ) else: - return cancel_command( - chat.telegram_id, + cancel_command( + chat_id, edit_message=self.sended_message.message_id ) + raise error + long_task() - def send_message(self, text: str) -> None: + def send_html_message( + self, + chat_id: int, + html_text: str + ) -> None: """ - Sends message to Telegram user. + Sends HTML message to Telegram user. + """ + telegram.send_message( + chat_id=chat_id, + text=html_text, + parse_mode="HTML" + ) - - if message already was sent, then sent message - will be updated with new text. + def reply_to_message( + self, + incoming_message_id: int, + chat_id: int, + text: str + ) -> None: """ - incoming_message = g.telegram_message - chat = g.db_chat + Sends reply message to Telegram user. - if (self.sended_message is None): + - if message already was sent, then sent + message will be updated with new text. + """ + if self.sended_message is None: result = telegram.send_message( - reply_to_message_id=incoming_message.message_id, - chat_id=chat.telegram_id, + reply_to_message_id=incoming_message_id, + chat_id=chat_id, text=text ) - self.sended_message = telegram_interface.Message( + self.sended_message = TelegramMessage( result["content"] ) elif (text != self.sended_message.get_text()): telegram.edit_message_text( message_id=self.sended_message.message_id, - chat_id=chat.telegram_id, + chat_id=chat_id, text=text ) @@ -339,7 +427,7 @@ class PhotoHandler(AttachmentHandler): @staticmethod def handle(*args, **kwargs): handler = PhotoHandler() - handler.upload() + handler.upload(*args, **kwargs) @property def telegram_action(self): @@ -354,13 +442,24 @@ def raw_data_type(self): # dict, not list, because we will select biggest photo return dict - def get_attachment(self, message: telegram_interface.Message): + def create_help_message(self): + return ( + "Send a photos that you want to upload." + "\n\n" + "Note:" + "\n" + "- original name will be not saved" + "\n" + "- original quality and size will be decreased" + ) + + def get_attachment(self, message: TelegramMessage): photos = message.raw_data.get(self.raw_data_key, []) biggest_photo = None biggest_pixels_count = -1 for photo in photos: - if (self.is_too_big_file(photo)): + if self.is_too_big_file(photo): continue current_pixels_count = photo["width"] * photo["height"] @@ -379,7 +478,7 @@ class FileHandler(AttachmentHandler): @staticmethod def handle(*args, **kwargs): handler = FileHandler() - handler.upload() + handler.upload(*args, **kwargs) @property def telegram_action(self): @@ -393,6 +492,17 @@ def raw_data_key(self): def raw_data_type(self): return dict + def create_help_message(self): + return ( + "Send a files that you want to upload." + "\n\n" + "Note:" + "\n" + "- original name will be saved" + "\n" + "- original quality and size will be saved" + ) + def get_mime_type(self, attachment): # file name already contains type return "" @@ -405,7 +515,7 @@ class AudioHandler(AttachmentHandler): @staticmethod def handle(*args, **kwargs): handler = AudioHandler() - handler.upload() + handler.upload(*args, **kwargs) @property def telegram_action(self): @@ -419,18 +529,31 @@ def raw_data_key(self): def raw_data_type(self): return dict + def create_help_message(self): + return ( + "Send a music that you want to upload." + "\n\n" + "Note:" + "\n" + "- original name will be saved" + "\n" + "- original quality and size will be saved" + "\n" + "- original type may be changed" + ) + def create_file_name(self, attachment, file): name = file["file_unique_id"] - if ("title" in attachment): + if "title" in attachment: name = attachment["title"] - if ("performer" in attachment): + if "performer" in attachment: name = f"{attachment['performer']} - {name}" extension = self.get_mime_type(attachment) - if (extension): + if extension: name = f"{name}.{extension}" return name @@ -443,7 +566,7 @@ class VideoHandler(AttachmentHandler): @staticmethod def handle(*args, **kwargs): handler = VideoHandler() - handler.upload() + handler.upload(*args, **kwargs) @property def telegram_action(self): @@ -457,6 +580,19 @@ def raw_data_key(self): def raw_data_type(self): return dict + def create_help_message(self): + return ( + "Send a video that you want to upload." + "\n\n" + "Note:" + "\n" + "- original name will be saved" + "\n" + "- original quality and size will be saved" + "\n" + "- original type may be changed" + ) + class VoiceHandler(AttachmentHandler): """ @@ -465,7 +601,7 @@ class VoiceHandler(AttachmentHandler): @staticmethod def handle(*args, **kwargs): handler = VoiceHandler() - handler.upload() + handler.upload(*args, **kwargs) @property def telegram_action(self): @@ -479,6 +615,11 @@ def raw_data_key(self): def raw_data_type(self): return dict + def create_help_message(self): + return ( + "Send a voice message that you want to upload." + ) + class URLHandler(AttachmentHandler): """ @@ -487,7 +628,7 @@ class URLHandler(AttachmentHandler): @staticmethod def handle(*args, **kwargs): handler = URLHandler() - handler.upload() + handler.upload(*args, **kwargs) @property def telegram_action(self): @@ -501,7 +642,18 @@ def raw_data_key(self): def raw_data_type(self): return str - def get_attachment(self, message: telegram_interface.Message): + def create_help_message(self): + return ( + "Send a direct URL to file that you want to upload." + "\n\n" + "Note:" + "\n" + "- original name from URL will be saved" + "\n" + "- original quality and size will be saved" + ) + + def get_attachment(self, message: TelegramMessage): return message.get_entity_value(self.raw_data_key) def create_file_name(self, attachment, file): From 7a4523a44736d9748fb9aae937f7b66fcee7e17c Mon Sep 17 00:00:00 2001 From: Sergey Kuznetsov Date: Fri, 20 Nov 2020 16:58:14 +0300 Subject: [PATCH 057/103] Fix typo in help.py --- src/blueprints/telegram_bot/webhook/commands/help.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/blueprints/telegram_bot/webhook/commands/help.py b/src/blueprints/telegram_bot/webhook/commands/help.py index 8a52dba..644a706 100644 --- a/src/blueprints/telegram_bot/webhook/commands/help.py +++ b/src/blueprints/telegram_bot/webhook/commands/help.py @@ -63,7 +63,7 @@ def handle(*args, **kwargs): "Folder name should starts from root, " f'nested folders should be separated with "{to_code("/")}" character.' "\n" - f"{CommandName.SPACE.value} — get information about remaining space. " + f"{CommandName.SPACE.value} — get information about remaining space." "\n\n" "Yandex.Disk Access" "\n" From 8c8bd0ab2a33e1df480b62652f00827c0aa631d1 Mon Sep 17 00:00:00 2001 From: Sergey Kuznetsov Date: Fri, 20 Nov 2020 22:12:26 +0300 Subject: [PATCH 058/103] Add /element_info command --- CHANGELOG.md | 1 + README.md | 2 +- info/info.json | 4 + src/api/telegram/methods.py | 13 +- src/api/yandex/__init__.py | 6 +- src/api/yandex/methods.py | 14 ++ src/api/yandex/requests.py | 35 ++++ src/blueprints/_common/utils.py | 59 ++++++ .../telegram_bot/_common/command_names.py | 1 + .../telegram_bot/_common/yandex_disk.py | 38 ++++ .../telegram_bot/webhook/commands/__init__.py | 1 + .../webhook/commands/_common/utils.py | 175 ++++++++++++++++++ .../webhook/commands/element_info.py | 130 +++++++++++++ .../telegram_bot/webhook/commands/help.py | 3 + .../telegram_bot/webhook/dispatcher.py | 3 +- 15 files changed, 477 insertions(+), 8 deletions(-) create mode 100644 src/blueprints/telegram_bot/webhook/commands/element_info.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 4a13a0f..0007ad1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,6 +11,7 @@ - `/publish`: publishing of files or folders. - `/unpublish`: unpublishing of files or folders. - `/space`: getting of information about remaining Yandex.Disk space. +- `/element_info`: getting of information about file or folder. - Handling of files that exceed file size limit in 20 MB. At the moment the bot will asnwer with warning message, not trying to upload that file. [#3](https://github.com/Amaimersion/yandex-disk-telegram-bot/issues/3) - Help messages for each upload command will be sended when there are no suitable input data. diff --git a/README.md b/README.md index 296ce6b..3199ba0 100644 --- a/README.md +++ b/README.md @@ -36,7 +36,7 @@ - uploading of voice (limit is 20 MB). - publishing and unpublishing of files or folders. - creating of folders. -- getting of information about the disk. +- getting of information about file, folder or disk. ## Requirements diff --git a/info/info.json b/info/info.json index 04293d9..5e5df0b 100644 --- a/info/info.json +++ b/info/info.json @@ -61,6 +61,10 @@ "command": "space", "description": "Information about remaining space" }, + { + "command": "element_info", + "description": "Information about file or folder" + }, { "command": "grant_access", "description": "Grant me an access to your Yandex.Disk" diff --git a/src/api/telegram/methods.py b/src/api/telegram/methods.py index 0c41e97..f649e1c 100644 --- a/src/api/telegram/methods.py +++ b/src/api/telegram/methods.py @@ -42,12 +42,17 @@ def send_photo(**kwargs): https://core.telegram.org/bots/api#sendphoto - see `api/request.py` documentation for more. - - specify `photo` as tuple from `files` in + - if you want to send bytes, then specify `photo` as + tuple from `files` in https://requests.readthedocs.io/en/latest/api/#requests.request """ + files = None key = "photo" - files = { - key: kwargs.pop(key) - } + value = kwargs.get(key) + + if not isinstance(value, str): + files = { + key: kwargs.pop(key) + } return make_request("sendPhoto", data=kwargs, files=files) diff --git a/src/api/yandex/__init__.py b/src/api/yandex/__init__.py index 42ad11a..7c28271 100644 --- a/src/api/yandex/__init__.py +++ b/src/api/yandex/__init__.py @@ -4,9 +4,11 @@ create_folder, publish, unpublish, - get_disk_info + get_disk_info, + get_element_info ) from .requests import ( make_link_request, - create_user_oauth_url + create_user_oauth_url, + make_photo_preview_request ) diff --git a/src/api/yandex/methods.py b/src/api/yandex/methods.py index 32b9ac2..e91915d 100644 --- a/src/api/yandex/methods.py +++ b/src/api/yandex/methods.py @@ -81,3 +81,17 @@ def get_disk_info(user_token: str, **kwargs): data=kwargs, user_token=user_token ) + + +def get_element_info(user_token: str, **kwargs): + """ + https://yandex.ru/dev/disk/api/reference/meta.html/ + + - see `api/request.py` documentation for more. + """ + return make_disk_request( + http_method="GET", + api_method="resources", + data=kwargs, + user_token=user_token + ) diff --git a/src/api/yandex/requests.py b/src/api/yandex/requests.py index ac1de4b..cf93679 100644 --- a/src/api/yandex/requests.py +++ b/src/api/yandex/requests.py @@ -141,3 +141,38 @@ def make_link_request(data: dict, user_token: str): allow_redirects=False, verify=True ) + + +def make_photo_preview_request(photo_url: str, user_token: str): + """ + Makes request to URL in order to get bytes content of photo. + + Yandex requires user OAuth token in order to get + access to photo previews, so, it is why you should + use this method. + + - it will not raise in case of error HTTP code. + - see `api/request.py` documentation for more. + + :param photo_url: + URL of photo. + :param user_token: + User OAuth token to access this URL. + + :returns: + See `api/request.py`. + In case of `ok = True` under `content` will be bytes content + of requested photo. + """ + timeout = current_app.config["YANDEX_DISK_API_TIMEOUT"] + + return request( + raise_for_status=False, + content_type="bytes", + method="GET", + url=photo_url, + timeout=timeout, + auth=HTTPOAuthAuth(user_token), + allow_redirects=False, + verify=True + ) diff --git a/src/blueprints/_common/utils.py b/src/blueprints/_common/utils.py index d93e833..569a832 100644 --- a/src/blueprints/_common/utils.py +++ b/src/blueprints/_common/utils.py @@ -46,3 +46,62 @@ def get_current_iso_datetime(timespec="seconds") -> str: See https://docs.python.org/3.8/library/datetime.html#datetime.datetime.isoformat # noqa """ return datetime.now(timezone.utc).isoformat(timespec=timespec) + + +def convert_iso_datetime(date_string: str) -> dict: + """ + :returns: + Pretty-print information about ISO 8601 `date_string`. + """ + value = datetime.fromisoformat(date_string) + value_date = value.strftime("%d.%m.%Y") + value_time = value.strftime("%H:%M:%S") + value_timezone = value.strftime("%Z") + + return { + "date": value_date, + "time": value_time, + "timezone": value_timezone + } + + +def bytes_to_human_unit( + bytes_count: int, + factor: float, + suffix: str +) -> str: + """ + Converts bytes to human readable string. + + - function source: https://stackoverflow.com/a/1094933/8445442 + - https://en.wikipedia.org/wiki/Binary_prefix + - https://man7.org/linux/man-pages/man7/units.7.html + + :param bytes_count: + Count of bytes to convert. + :param factor: + Use `1024.0` for binary and `1000.0` for decimal. + :param suffix: + Use `iB` for binary and `B` for decimal. + """ + for unit in ["", "K", "M", "G", "T", "P", "E", "Z"]: + if abs(bytes_count) < factor: + return "%3.1f %s%s" % (bytes_count, unit, suffix) + + bytes_count /= factor + + return "%.1f %s%s" % (bytes_count, "Y", suffix) + + +def bytes_to_human_binary(bytes_count: int) -> str: + """ + Bytes -> binary representation. + """ + return bytes_to_human_unit(bytes_count, 1024.0, "iB") + + +def bytes_to_human_decimal(bytes_count: int) -> str: + """ + Bytes -> decimal representation. + """ + return bytes_to_human_unit(bytes_count, 1000.0, "B") diff --git a/src/blueprints/telegram_bot/_common/command_names.py b/src/blueprints/telegram_bot/_common/command_names.py index 6e08f53..6b86ecc 100644 --- a/src/blueprints/telegram_bot/_common/command_names.py +++ b/src/blueprints/telegram_bot/_common/command_names.py @@ -22,3 +22,4 @@ class CommandName(Enum): PUBLISH = "/publish" UNPUBLISH = "/unpublish" SPACE = "/space" + ELEMENT_INFO = "/element_info" diff --git a/src/blueprints/telegram_bot/_common/yandex_disk.py b/src/blueprints/telegram_bot/_common/yandex_disk.py index 3fd911a..f2c332d 100644 --- a/src/blueprints/telegram_bot/_common/yandex_disk.py +++ b/src/blueprints/telegram_bot/_common/yandex_disk.py @@ -59,6 +59,16 @@ class YandexAPIUnpublishItemError(Exception): pass +class YandexAPIGetElementInfoError(Exception): + """ + Unable to get information about Yandex.Disk + element (folder or file). + + - may contain human-readable error message. + """ + pass + + class YandexAPIExceededNumberOfStatusChecksError(Exception): """ There was too much attempts to check status @@ -308,6 +318,34 @@ def get_disk_info(user_access_token: str) -> dict: return response +def get_element_info( + user_access_token: str, + absolute_element_path: str +) -> dict: + try: + response = yandex.get_element_info( + user_access_token, + path=absolute_element_path, + preview_crop=False, + preview_size="L", + # at the moment we don't display any + # elements that embedded in directory + limit=0 + ) + except Exception as error: + raise YandexAPIRequestError(error) + + response = response["content"] + is_error = is_error_yandex_response(response) + + if is_error: + raise YandexAPIGetElementInfoError( + create_yandex_error_text(response) + ) + + return response + + def is_error_yandex_response(data: dict) -> bool: """ :returns: Yandex response contains error or not. diff --git a/src/blueprints/telegram_bot/webhook/commands/__init__.py b/src/blueprints/telegram_bot/webhook/commands/__init__.py index 429500c..d2b2c9a 100644 --- a/src/blueprints/telegram_bot/webhook/commands/__init__.py +++ b/src/blueprints/telegram_bot/webhook/commands/__init__.py @@ -38,3 +38,4 @@ from .publish import handle as publish_handler from .unpublish import handle as unpublish_handler from .space import handle as space_handler +from .element_info import handle as element_info_handler diff --git a/src/blueprints/telegram_bot/webhook/commands/_common/utils.py b/src/blueprints/telegram_bot/webhook/commands/_common/utils.py index 5c0f0a6..e35f284 100644 --- a/src/blueprints/telegram_bot/webhook/commands/_common/utils.py +++ b/src/blueprints/telegram_bot/webhook/commands/_common/utils.py @@ -1,5 +1,11 @@ from typing import Union +from collections import deque +import json +from src.blueprints._common.utils import ( + convert_iso_datetime, + bytes_to_human_decimal +) from src.blueprints.telegram_bot._common.telegram_interface import ( Message as TelegramMessage ) @@ -42,3 +48,172 @@ def extract_absolute_path( ).strip() return path + + +def create_element_info_html_text( + info: dict, + include_private_info: bool +) -> str: + """ + :param info: + - https://yandex.ru/dev/disk/api/reference/meta.html/ + - https://dev.yandex.net/disk-polygon/?lang=ru&tld=ru#!/v147disk47resources/GetResource # noqa + :param include_private_info: + Include private information in result. + "Private information" means information that + can compromise a user (for example, full path of element). + Use `True` when you want to display this text only to user, + use `False` when you can assume that user theoretically can + forward this text to some another user(s). + """ + text = deque() + + if "name" in info: + text.append( + f"Name: {info['name']}" + ) + + if "type" in info: + incoming_type = info["type"].lower() + value = "Unknown" + + if (incoming_type == "dir"): + value = "Folder" + elif (incoming_type == "file"): + if "media_type" in info: + value = info["media_type"] + else: + value = "File" + + if "mime_type" in info: + value = f"{value} ({info['mime_type']})" + + text.append( + f"Type: {value}" + ) + + if "size" in info: + bytes_count = info["size"] + value = f"{bytes_count:,} bytes" + + if (bytes_count >= 1000): + decimal = bytes_to_human_decimal(bytes_count) + value = f"{decimal} ({bytes_count:,} bytes)" + + text.append( + f"Size: {value}" + ) + + if ( + include_private_info and + ("created" in info) + ): + value = convert_iso_datetime(info["created"]) + text.append( + "Created: " + f"{value['date']} {value['time']} {value['timezone']}" + ) + + if ( + include_private_info and + ("modified" in info) + ): + value = convert_iso_datetime(info["modified"]) + text.append( + "Modified: " + f"{value['date']} {value['time']} {value['timezone']}" + ) + + if ( + include_private_info and + ("path" in info) + ): + text.append( + "Full path: " + f"{info['path']}" + ) + + if ( + include_private_info and + ("origin_path" in info) + ): + text.append( + "Origin path: " + f"{info['origin_path']}" + ) + + if ( + ("_embedded" in info) and + ("total" in info["_embedded"]) + ): + text.append( + f"Total elements: {info['_embedded']['total']}" + ) + + if "public_url" in info: + text.append( + f"Public URL: {info['public_url']}" + ) + + if "sha256" in info: + text.append( + f"SHA-256: {info['sha256']}" + ) + + if "md5" in info: + text.append( + f"MD5: {info['md5']}" + ) + + if ( + include_private_info and + ("share" in info) + ): + data = info["share"] + + if "is_owned" in data: + value = "No" + + if data["is_owned"]: + value = "Yes" + + text.append( + f"Shared access — Owner: {value}" + ) + + if "rights" in data: + value = data["rights"].lower() + + if (value == "rw"): + value = "Full access" + elif (value == "r"): + value = "Read" + elif (value == "w"): + value = "Write" + + text.append( + f"Shared access — Rights: {value}" + ) + + if "is_root" in data: + value = "No" + + if data["is_root"]: + value = "Yes" + + text.append( + f"Shared access — Root: {value}" + ) + + if ( + include_private_info and + ("exif" in info) and + info["exif"] + ): + exif = json.dumps(info["exif"], indent=4) + text.append( + "EXIF: " + f"{exif}" + ) + + return "\n".join(text) diff --git a/src/blueprints/telegram_bot/webhook/commands/element_info.py b/src/blueprints/telegram_bot/webhook/commands/element_info.py new file mode 100644 index 0000000..878232e --- /dev/null +++ b/src/blueprints/telegram_bot/webhook/commands/element_info.py @@ -0,0 +1,130 @@ +from flask import g, current_app + +from src.api import telegram +from src.api.yandex import make_photo_preview_request +from src.blueprints.telegram_bot._common.yandex_disk import ( + get_element_info, + YandexAPIGetElementInfoError, + YandexAPIRequestError +) +from src.blueprints.telegram_bot._common.stateful_chat import ( + set_disposable_handler +) +from src.blueprints.telegram_bot._common.command_names import ( + CommandName +) +from src.blueprints.telegram_bot.webhook.dispatcher_events import ( + DispatcherEvent +) +from ._common.responses import ( + cancel_command, + request_absolute_path, + send_yandex_disk_error +) +from ._common.decorators import ( + yd_access_token_required, + get_db_data +) +from ._common.utils import ( + extract_absolute_path, + create_element_info_html_text +) + + +@yd_access_token_required +@get_db_data +def handle(*args, **kwargs): + """ + Handles `/element_info` command. + """ + message = kwargs.get( + "message", + g.telegram_message + ) + user_id = kwargs.get( + "user_id", + g.telegram_user.id + ) + chat_id = kwargs.get( + "chat_id", + g.telegram_chat.id + ) + path = extract_absolute_path( + message, + CommandName.ELEMENT_INFO.value, + kwargs.get("route_source") + ) + + if not path: + set_disposable_handler( + user_id, + chat_id, + CommandName.ELEMENT_INFO.value, + [ + DispatcherEvent.PLAIN_TEXT.value, + DispatcherEvent.BOT_COMMAND.value, + DispatcherEvent.EMAIL.value, + DispatcherEvent.HASHTAG.value, + DispatcherEvent.URL.value + ], + current_app.config["RUNTIME_DISPOSABLE_HANDLER_EXPIRE"] + ) + + return request_absolute_path(chat_id) + + user = g.db_user + access_token = user.yandex_disk_token.get_access_token() + info = None + + try: + info = get_element_info(access_token, path) + except YandexAPIRequestError as error: + cancel_command(chat_id) + raise error + except YandexAPIGetElementInfoError as error: + send_yandex_disk_error(chat_id, str(error)) + + # it is expected error and should be + # logged only to user + return + + text = create_element_info_html_text(info, True) + params = { + "chat_id": chat_id, + "text": text, + "parse_mode": "HTML", + "disable_web_page_preview": True + } + download_url = info.get("file") + + if download_url: + params["reply_markup"] = { + "inline_keyboard": [[ + { + "text": "Download", + "url": download_url + } + ]] + } + + telegram.send_message(**params) + + preview = info.get("preview") + + if preview: + # Yandex requires user OAuth token to get preview + result = make_photo_preview_request(preview, access_token) + + if result["ok"]: + data = result["content"] + filename = info.get("name", "?") + + telegram.send_photo( + chat_id=chat_id, + photo=( + filename, + data, + "image/jpeg" + ), + disable_notification=True + ) diff --git a/src/blueprints/telegram_bot/webhook/commands/help.py b/src/blueprints/telegram_bot/webhook/commands/help.py index 644a706..c8ef69c 100644 --- a/src/blueprints/telegram_bot/webhook/commands/help.py +++ b/src/blueprints/telegram_bot/webhook/commands/help.py @@ -64,6 +64,9 @@ def handle(*args, **kwargs): f'nested folders should be separated with "{to_code("/")}" character.' "\n" f"{CommandName.SPACE.value} — get information about remaining space." + "\n" + f"{CommandName.ELEMENT_INFO.value} — get information about file or folder. " + "Send full path of element with this command." "\n\n" "Yandex.Disk Access" "\n" diff --git a/src/blueprints/telegram_bot/webhook/dispatcher.py b/src/blueprints/telegram_bot/webhook/dispatcher.py index 5ed1574..deefd35 100644 --- a/src/blueprints/telegram_bot/webhook/dispatcher.py +++ b/src/blueprints/telegram_bot/webhook/dispatcher.py @@ -177,7 +177,8 @@ def direct_dispatch( CommandName.CREATE_FOLDER.value: commands.create_folder_handler, CommandName.PUBLISH.value: commands.publish_handler, CommandName.UNPUBLISH.value: commands.unpublish_handler, - CommandName.SPACE.value: commands.space_handler + CommandName.SPACE.value: commands.space_handler, + CommandName.ELEMENT_INFO.value: commands.element_info_handler } handler = routes.get(command, fallback) From 7cde31fe636b181cccd6156edc5c59c0c14acbe5 Mon Sep 17 00:00:00 2001 From: Sergey Kuznetsov Date: Sat, 21 Nov 2020 18:37:39 +0300 Subject: [PATCH 059/103] Add returning of information in /publish --- .../telegram_bot/webhook/commands/publish.py | 32 +++++++++++++++---- 1 file changed, 26 insertions(+), 6 deletions(-) diff --git a/src/blueprints/telegram_bot/webhook/commands/publish.py b/src/blueprints/telegram_bot/webhook/commands/publish.py index d02d12d..9e891f1 100644 --- a/src/blueprints/telegram_bot/webhook/commands/publish.py +++ b/src/blueprints/telegram_bot/webhook/commands/publish.py @@ -3,6 +3,8 @@ from src.api import telegram from src.blueprints.telegram_bot._common.yandex_disk import ( publish_item, + get_element_info, + YandexAPIGetElementInfoError, YandexAPIPublishItemError, YandexAPIRequestError ) @@ -24,7 +26,10 @@ yd_access_token_required, get_db_data ) -from ._common.utils import extract_absolute_path +from ._common.utils import ( + extract_absolute_path, + create_element_info_html_text +) @yd_access_token_required @@ -72,10 +77,7 @@ def handle(*args, **kwargs): access_token = user.yandex_disk_token.get_access_token() try: - publish_item( - access_token, - path - ) + publish_item(access_token, path) except YandexAPIRequestError as error: cancel_command(chat_id) raise error @@ -86,7 +88,25 @@ def handle(*args, **kwargs): # logged only to user return + info = None + + try: + info = get_element_info(access_token, path) + except YandexAPIRequestError as error: + cancel_command(chat_id) + raise error + except YandexAPIGetElementInfoError as error: + send_yandex_disk_error(chat_id, str(error)) + + # it is expected error and should be + # logged only to user + return + + text = create_element_info_html_text(info, False) + telegram.send_message( chat_id=chat_id, - text="Published" + text=text, + parse_mode="HTML", + disable_web_page_preview=True ) From 02ddf837f3defbfb6fb93be3b3b5e762aad7447c Mon Sep 17 00:00:00 2001 From: Sergey Kuznetsov Date: Sat, 21 Nov 2020 20:18:41 +0300 Subject: [PATCH 060/103] Add public info (views, owner) for /element_info --- src/api/yandex/__init__.py | 3 +- src/api/yandex/methods.py | 14 +++ .../telegram_bot/_common/yandex_disk.py | 89 +++++++++++++++++-- .../webhook/commands/_common/utils.py | 41 ++++++++- .../webhook/commands/element_info.py | 6 +- 5 files changed, 141 insertions(+), 12 deletions(-) diff --git a/src/api/yandex/__init__.py b/src/api/yandex/__init__.py index 7c28271..c66d8bd 100644 --- a/src/api/yandex/__init__.py +++ b/src/api/yandex/__init__.py @@ -5,7 +5,8 @@ publish, unpublish, get_disk_info, - get_element_info + get_element_info, + get_element_public_info ) from .requests import ( make_link_request, diff --git a/src/api/yandex/methods.py b/src/api/yandex/methods.py index e91915d..13105df 100644 --- a/src/api/yandex/methods.py +++ b/src/api/yandex/methods.py @@ -95,3 +95,17 @@ def get_element_info(user_token: str, **kwargs): data=kwargs, user_token=user_token ) + + +def get_element_public_info(user_token: str, **kwargs): + """ + https://dev.yandex.net/disk-polygon/?lang=ru&tld=ru#!/v147disk47public47resources/GetPublicResource # noqa + + - see `api/request.py` documentation for more. + """ + return make_disk_request( + http_method="GET", + api_method="public/resources", + data=kwargs, + user_token=user_token + ) diff --git a/src/blueprints/telegram_bot/_common/yandex_disk.py b/src/blueprints/telegram_bot/_common/yandex_disk.py index f2c332d..5528ef7 100644 --- a/src/blueprints/telegram_bot/_common/yandex_disk.py +++ b/src/blueprints/telegram_bot/_common/yandex_disk.py @@ -320,17 +320,62 @@ def get_disk_info(user_access_token: str) -> dict: def get_element_info( user_access_token: str, - absolute_element_path: str + absolute_element_path: str, + get_public_info=False, + preview_size="L", + preview_crop=False, + embedded_elements_limit=0, + embedded_elements_offset=0, + embedded_elements_sort="name" ) -> dict: + """ + - https://yandex.ru/dev/disk/api/reference/meta.html + - https://dev.yandex.net/disk-polygon/?lang=ru&tld=ru#!/v147disk47public47resources/GetPublicResource # noqa + + :param user_access_token: + User access token. + :param absolute_element_path: + Absolute path of element. + :param get_public_info: + Make another HTTP request to get public info. + If `True`, then these fields will be added in + normal info: `views_count`, `owner`. + Set to `False` if you don't need this information - + this will improve speed. + :param preview_size: + Size of preview (if available). + :param preview_crop: + Allow cropping of preview. + :param embedded_elements_limit: + How many embedded elements (elements inside + folder) should be included in response. + Set to `0` if you don't need this information - + this will improve speed. + :param embedded_elements_offset: + Offset for embedded elements. + Note `sort` parameter. + :param embedded_elements_sort: + How to sort embedded elements. + Possible values: `name`, `path`, `created`, + `modified`, `size`. Append `-` for reverse + order (example: `-name`). + + :returns: + Information about object. + Check for keys before using them! + + :raises: + `YandexAPIRequestError`, `YandexAPIGetElementInfoError`. + """ try: response = yandex.get_element_info( user_access_token, path=absolute_element_path, - preview_crop=False, - preview_size="L", - # at the moment we don't display any - # elements that embedded in directory - limit=0 + preview_crop=preview_crop, + preview_size=preview_size, + limit=embedded_elements_limit, + offset=embedded_elements_offset, + sort=embedded_elements_sort ) except Exception as error: raise YandexAPIRequestError(error) @@ -343,6 +388,38 @@ def get_element_info( create_yandex_error_text(response) ) + if ( + get_public_info and + ("public_key" in response) + ): + public_info_response = None + + try: + public_info_response = yandex.get_element_public_info( + user_access_token, + public_key=response["public_key"], + # we need only these fields, because they + # are missing in normal info + fields="views_count,owner", + preview_crop=preview_crop, + preview_size=preview_size, + limit=embedded_elements_limit, + offset=embedded_elements_offset, + sort=embedded_elements_sort + ) + except Exception as error: + raise YandexAPIRequestError(error) + + public_info_response = public_info_response["content"] + is_error = is_error_yandex_response(public_info_response) + + if is_error: + raise YandexAPIGetElementInfoError( + create_yandex_error_text(response) + ) + + response = {**response, **public_info_response} + return response diff --git a/src/blueprints/telegram_bot/webhook/commands/_common/utils.py b/src/blueprints/telegram_bot/webhook/commands/_common/utils.py index e35f284..3d366a3 100644 --- a/src/blueprints/telegram_bot/webhook/commands/_common/utils.py +++ b/src/blueprints/telegram_bot/webhook/commands/_common/utils.py @@ -155,14 +155,37 @@ def create_element_info_html_text( f"Public URL: {info['public_url']}" ) - if "sha256" in info: + if ( + include_private_info and + ("views_count" in info) + ): text.append( - f"SHA-256: {info['sha256']}" + f"Views: {info['views_count']}" ) - if "md5" in info: + if ( + include_private_info and + ("owner" in info) + ): + data = info["owner"] + name = data.get("display_name") + login = data.get("login") + value = "?" + + if name: + value = name + + if ( + value and + login and + (value != login) + ): + value = f"{value} ({login})" + elif login: + value = login + text.append( - f"MD5: {info['md5']}" + f"Owner: {value}" ) if ( @@ -216,4 +239,14 @@ def create_element_info_html_text( f"{exif}" ) + if "sha256" in info: + text.append( + f"SHA-256: {info['sha256']}" + ) + + if "md5" in info: + text.append( + f"MD5: {info['md5']}" + ) + return "\n".join(text) diff --git a/src/blueprints/telegram_bot/webhook/commands/element_info.py b/src/blueprints/telegram_bot/webhook/commands/element_info.py index 878232e..5691413 100644 --- a/src/blueprints/telegram_bot/webhook/commands/element_info.py +++ b/src/blueprints/telegram_bot/webhook/commands/element_info.py @@ -77,7 +77,11 @@ def handle(*args, **kwargs): info = None try: - info = get_element_info(access_token, path) + info = get_element_info( + access_token, + path, + get_public_info=True + ) except YandexAPIRequestError as error: cancel_command(chat_id) raise error From 06c9ed33f289904d01e3d705b75adf66d3b10252 Mon Sep 17 00:00:00 2001 From: Sergey Kuznetsov Date: Sat, 21 Nov 2020 23:28:35 +0300 Subject: [PATCH 061/103] Refactoring of /element_info text --- .../webhook/commands/_common/utils.py | 34 +++++++++---------- 1 file changed, 17 insertions(+), 17 deletions(-) diff --git a/src/blueprints/telegram_bot/webhook/commands/_common/utils.py b/src/blueprints/telegram_bot/webhook/commands/_common/utils.py index 3d366a3..bec1d5b 100644 --- a/src/blueprints/telegram_bot/webhook/commands/_common/utils.py +++ b/src/blueprints/telegram_bot/webhook/commands/_common/utils.py @@ -70,7 +70,7 @@ def create_element_info_html_text( if "name" in info: text.append( - f"Name: {info['name']}" + f"Name: {info['name']}" ) if "type" in info: @@ -89,7 +89,7 @@ def create_element_info_html_text( value = f"{value} ({info['mime_type']})" text.append( - f"Type: {value}" + f"Type: {value}" ) if "size" in info: @@ -101,7 +101,7 @@ def create_element_info_html_text( value = f"{decimal} ({bytes_count:,} bytes)" text.append( - f"Size: {value}" + f"Size: {value}" ) if ( @@ -110,7 +110,7 @@ def create_element_info_html_text( ): value = convert_iso_datetime(info["created"]) text.append( - "Created: " + "Created: " f"{value['date']} {value['time']} {value['timezone']}" ) @@ -120,7 +120,7 @@ def create_element_info_html_text( ): value = convert_iso_datetime(info["modified"]) text.append( - "Modified: " + "Modified: " f"{value['date']} {value['time']} {value['timezone']}" ) @@ -129,7 +129,7 @@ def create_element_info_html_text( ("path" in info) ): text.append( - "Full path: " + "Full path: " f"{info['path']}" ) @@ -138,7 +138,7 @@ def create_element_info_html_text( ("origin_path" in info) ): text.append( - "Origin path: " + "Origin path: " f"{info['origin_path']}" ) @@ -147,12 +147,12 @@ def create_element_info_html_text( ("total" in info["_embedded"]) ): text.append( - f"Total elements: {info['_embedded']['total']}" + f"Total elements: {info['_embedded']['total']}" ) if "public_url" in info: text.append( - f"Public URL: {info['public_url']}" + f"Public URL: {info['public_url']}" ) if ( @@ -160,7 +160,7 @@ def create_element_info_html_text( ("views_count" in info) ): text.append( - f"Views: {info['views_count']}" + f"Views: {info['views_count']}" ) if ( @@ -185,7 +185,7 @@ def create_element_info_html_text( value = login text.append( - f"Owner: {value}" + f"Owner: {value}" ) if ( @@ -201,7 +201,7 @@ def create_element_info_html_text( value = "Yes" text.append( - f"Shared access — Owner: {value}" + f"Shared access — Owner: {value}" ) if "rights" in data: @@ -215,7 +215,7 @@ def create_element_info_html_text( value = "Write" text.append( - f"Shared access — Rights: {value}" + f"Shared access — Rights: {value}" ) if "is_root" in data: @@ -225,7 +225,7 @@ def create_element_info_html_text( value = "Yes" text.append( - f"Shared access — Root: {value}" + f"Shared access — Root: {value}" ) if ( @@ -235,18 +235,18 @@ def create_element_info_html_text( ): exif = json.dumps(info["exif"], indent=4) text.append( - "EXIF: " + "EXIF: " f"{exif}" ) if "sha256" in info: text.append( - f"SHA-256: {info['sha256']}" + f"SHA-256: {info['sha256']}" ) if "md5" in info: text.append( - f"MD5: {info['md5']}" + f"MD5: {info['md5']}" ) return "\n".join(text) From 1188db9b748c752a6df1521207da75d27522788c Mon Sep 17 00:00:00 2001 From: Sergey Kuznetsov Date: Sun, 22 Nov 2020 12:01:05 +0300 Subject: [PATCH 062/103] /upload: on success will return information abou uploaded file --- CHANGELOG.md | 1 + .../telegram_bot/_common/yandex_disk.py | 105 ++++++++++++------ .../telegram_bot/webhook/commands/upload.py | 74 +++++++++++- 3 files changed, 138 insertions(+), 42 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 0007ad1..98a62f3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -18,6 +18,7 @@ ### Changed - `/create_folder`: now it will wait for folder name if you send empty command, not deny operation. +- `/upload`: on success it will return information about uploaded file, not plain status. ### Fixed diff --git a/src/blueprints/telegram_bot/_common/yandex_disk.py b/src/blueprints/telegram_bot/_common/yandex_disk.py index 5528ef7..226d95b 100644 --- a/src/blueprints/telegram_bot/_common/yandex_disk.py +++ b/src/blueprints/telegram_bot/_common/yandex_disk.py @@ -196,7 +196,7 @@ def upload_file_with_url( folder_path: str, file_name: str, download_url: str -) -> Generator[str, None, None]: +) -> Generator[dict, None, None]: """ Uploads a file to Yandex.Disk using file download url. @@ -205,15 +205,24 @@ def upload_file_with_url( to app configuration. Because it is synchronous, it may take significant time to end this function! - :yields: status of operation in Yandex format (for example, - `"in progress"`). It will yields with some interval (according - to app configuration). Order is an order in which Yandex - sends the operation status. + :yields: + `dict` with `success`, `failed`, `completed`, `status`. + It will yields with some interval (according + to app configuration). Order is an order in which + Yandex sent an operation status, so, `status` can + be safely logged to user. + `status` - currenet string status of uploading + (for example, `in progress`). + `completed` - uploading is completed. + `success` - uploading is successfully ended. + `failed` - uploading is failed (unknown error, known + error will be throwed with `YandexAPIUploadFileError`). - :raises: YandexAPIRequestError - :raises: YandexAPICreateFolderError - :raises: YandexAPIUploadFileError - :raises: YandexAPIExceededNumberOfStatusChecksError + :raises: + `YandexAPIRequestError`, + `YandexAPICreateFolderError`, + `YandexAPIUploadFileError`, + `YandexAPIExceededNumberOfStatusChecksError` """ create_folder( user_access_token=user_access_token, @@ -236,14 +245,16 @@ def upload_file_with_url( operation_status_link = response["content"] is_error = is_error_yandex_response(operation_status_link) - if (is_error): + if is_error: raise YandexAPIUploadFileError( - create_yandex_error_text( - operation_status_link - ) + create_yandex_error_text(operation_status_link) ) - operation_status = {} + operation_status = None + is_error = False + is_success = False + is_failed = False + is_completed = False attempt = 0 max_attempts = current_app.config[ "YANDEX_DISK_API_CHECK_OPERATION_STATUS_MAX_ATTEMPTS" @@ -251,11 +262,12 @@ def upload_file_with_url( interval = current_app.config[ "YANDEX_DISK_API_CHECK_OPERATION_STATUS_INTERVAL" ] + too_many_attempts = (attempt >= max_attempts) while not ( - is_error_yandex_response(operation_status) or - yandex_operation_is_completed(operation_status) or - attempt >= max_attempts + is_error or + is_completed or + too_many_attempts ): sleep(interval) @@ -268,23 +280,30 @@ def upload_file_with_url( raise YandexAPIRequestError(error) operation_status = response["content"] - - if ("status" in operation_status): - yield operation_status["status"] - + is_error = is_error_yandex_response(operation_status) + is_success = yandex_operation_is_success(operation_status) + is_failed = yandex_operation_is_failed(operation_status) + is_completed = (is_success or is_failed) attempt += 1 + too_many_attempts = (attempt >= max_attempts) + + if not is_error: + yield { + "success": is_success, + "failed": is_failed, + "completed": (is_success or is_failed), + "status": operation_status.get( + "status", + "unknown" + ) + } - is_error = is_error_yandex_response(operation_status) - is_completed = yandex_operation_is_completed(operation_status) - - if (is_error): + if is_error: raise YandexAPIUploadFileError( - create_yandex_error_text( - operation_status - ) + create_yandex_error_text(operation_status) ) elif ( - (attempt >= max_attempts) and + too_many_attempts and not is_completed ): raise YandexAPIExceededNumberOfStatusChecksError() @@ -447,17 +466,29 @@ def create_yandex_error_text(data: dict) -> str: return (f"{error_name}: {error_description}") -def yandex_operation_is_completed(data: dict) -> bool: +def yandex_operation_is_success(data: dict) -> bool: + """ + :returns: + Yandex response contains status which + indicates that operation is successfully ended. + """ + return ( + ("status" in data) and + (data["status"] == "success") + ) + + +def yandex_operation_is_failed(data: dict) -> bool: """ - :returns: Yandex response contains completed - operation status or not. + :returns: + Yandex response contains status which + indicates that operation is failed. """ return ( ("status" in data) and - ( - (data["status"] == "success") or + (data["status"] in ( # Yandex documentation is different in some places - (data["status"] == "failure") or - (data["status"] == "failed") - ) + "failure", + "failed" + )) ) diff --git a/src/blueprints/telegram_bot/webhook/commands/upload.py b/src/blueprints/telegram_bot/webhook/commands/upload.py index fa6a015..e0c6b45 100644 --- a/src/blueprints/telegram_bot/webhook/commands/upload.py +++ b/src/blueprints/telegram_bot/webhook/commands/upload.py @@ -7,8 +7,12 @@ from src.blueprints.telegram_bot._common.telegram_interface import ( Message as TelegramMessage ) +from src.blueprints.telegram_bot._common.command_names import ( + CommandName +) from src.blueprints.telegram_bot._common.yandex_disk import ( upload_file_with_url, + get_element_info, YandexAPIRequestError, YandexAPICreateFolderError, YandexAPIUploadFileError, @@ -24,6 +28,9 @@ send_yandex_disk_error, AbortReason ) +from ._common.utils import ( + create_element_info_html_text +) class MessageHealth: @@ -323,10 +330,49 @@ def long_task(): file_name=file_name, download_url=download_url ): + success = status["success"] + text = "" + is_html_text = False + + if success: + info = None + full_path = f"{folder_path}/{file_name}" + + try: + info = get_element_info( + user_access_token, + full_path, + get_public_info=False + ) + except Exception: + pass + + if info: + text = create_element_info_html_text( + info, + include_private_info=True + ) + is_html_text = True + else: + text = ( + "Successfully completed, but failed " + "to get information about this file." + "\n" + "Type to see information:" + "\n" + f"{CommandName.ELEMENT_INFO.value} {full_path}" + ) + else: + # You shouldn't use HTML for this, + # because `upload_status` can be a same + upload_status = status["status"] + text = f"Status: {upload_status}" + self.reply_to_message( message.message_id, chat_id, - f"Status: {status}" + text, + is_html_text ) except YandexAPICreateFolderError as error: error_text = str(error) or ( @@ -388,26 +434,42 @@ def send_html_message( telegram.send_message( chat_id=chat_id, text=html_text, - parse_mode="HTML" + parse_mode="HTML", + disable_web_page_preview=True ) def reply_to_message( self, incoming_message_id: int, chat_id: int, - text: str + text: str, + html_text=False ) -> None: """ Sends reply message to Telegram user. - if message already was sent, then sent message will be updated with new text. + - NOTE: using HTML text may lead to error, + because text should be compared with already + sended text, but already sended text will not + contain HTML tags (even if they was before sending), + and `text` will, so, comparing already sended HTML + text and `text` always will results to `False`. """ + enabled_html = {} + + if html_text: + enabled_html["parse_mode"] = "HTML" + if self.sended_message is None: result = telegram.send_message( reply_to_message_id=incoming_message_id, chat_id=chat_id, - text=text + text=text, + allow_sending_without_reply=True, + disable_web_page_preview=True, + **enabled_html ) self.sended_message = TelegramMessage( result["content"] @@ -416,7 +478,9 @@ def reply_to_message( telegram.edit_message_text( message_id=self.sended_message.message_id, chat_id=chat_id, - text=text + text=text, + disable_web_page_preview=True, + **enabled_html ) From 2bea66d79bedd115611d7c3d16f669f86c91561e Mon Sep 17 00:00:00 2001 From: Sergey Kuznetsov Date: Mon, 23 Nov 2020 16:27:47 +0300 Subject: [PATCH 063/103] Add /public_upload commands --- CHANGELOG.md | 1 + README.md | 1 + info/info.json | 24 +++ .../telegram_bot/_common/command_names.py | 6 + .../telegram_bot/webhook/commands/__init__.py | 20 ++- .../telegram_bot/webhook/commands/help.py | 18 +- .../telegram_bot/webhook/commands/upload.py | 169 +++++++++++++++--- .../telegram_bot/webhook/dispatcher.py | 6 + 8 files changed, 210 insertions(+), 35 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 98a62f3..d35b6ed 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,7 @@ ### Added +- `/public_upload_photo`, `/public_upload_file`, `/public_upload_audio`, `/public_upload_video`, `/public_upload_voice`, `/public_upload_url`: uploading of files and then publishing them. - `/publish`: publishing of files or folders. - `/unpublish`: unpublishing of files or folders. - `/space`: getting of information about remaining Yandex.Disk space. diff --git a/README.md b/README.md index 3199ba0..47b5e6f 100644 --- a/README.md +++ b/README.md @@ -34,6 +34,7 @@ - uploading of audio (limit is 20 MB). - uploading of video (limit is 20 MB). - uploading of voice (limit is 20 MB). +- uploading for public access. - publishing and unpublishing of files or folders. - creating of folders. - getting of information about file, folder or disk. diff --git a/info/info.json b/info/info.json index 5e5df0b..b884812 100644 --- a/info/info.json +++ b/info/info.json @@ -25,26 +25,50 @@ "command": "upload_photo", "description": "Upload a photo with quality loss" }, + { + "command": "public_upload_photo", + "description": "Upload a photo and publish it" + }, { "command": "upload_file", "description": "Upload a file with original state" }, + { + "command": "public_upload_file", + "description": "Upload a file and publish it" + }, { "command": "upload_audio", "description": "Upload an audio with original quality" }, + { + "command": "public_upload_audio", + "description": "Upload an audio and publish it" + }, { "command": "upload_video", "description": "Upload a video with original quality" }, + { + "command": "public_upload_video", + "description": "Upload a video and publish it" + }, { "command": "upload_voice", "description": "Upload a voice message" }, + { + "command": "public_upload_voice", + "description": "Upload a voice and publish it" + }, { "command": "upload_url", "description": "Upload a file using direct URL" }, + { + "command": "public_upload_url", + "description": "Upload a file using direct URL and publish it" + }, { "command": "publish", "description": "Publish a file or folder using OS path" diff --git a/src/blueprints/telegram_bot/_common/command_names.py b/src/blueprints/telegram_bot/_common/command_names.py index 6b86ecc..f58997a 100644 --- a/src/blueprints/telegram_bot/_common/command_names.py +++ b/src/blueprints/telegram_bot/_common/command_names.py @@ -18,6 +18,12 @@ class CommandName(Enum): UPLOAD_VIDEO = "/upload_video" UPLOAD_VOICE = "/upload_voice" UPLOAD_URL = "/upload_url" + PUBLIC_UPLOAD_PHOTO = "/public_upload_photo" + PUBLIC_UPLOAD_FILE = "/public_upload_file" + PUBLIC_UPLOAD_AUDIO = "/public_upload_audio" + PUBLIC_UPLOAD_VIDEO = "/public_upload_video" + PUBLIC_UPLOAD_VOICE = "/public_upload_voice" + PUBLIC_UPLOAD_URL = "/public_upload_url" CREATE_FOLDER = "/create_folder" PUBLISH = "/publish" UNPUBLISH = "/unpublish" diff --git a/src/blueprints/telegram_bot/webhook/commands/__init__.py b/src/blueprints/telegram_bot/webhook/commands/__init__.py index d2b2c9a..5895a34 100644 --- a/src/blueprints/telegram_bot/webhook/commands/__init__.py +++ b/src/blueprints/telegram_bot/webhook/commands/__init__.py @@ -22,18 +22,26 @@ """ +from .upload import ( + handle_photo as upload_photo_handler, + handle_file as upload_file_handler, + handle_audio as upload_audio_handler, + handle_video as upload_video_handler, + handle_voice as upload_voice_handler, + handle_url as upload_url_handler, + handle_public_photo as public_upload_photo_handler, + handle_public_file as public_upload_file_handler, + handle_public_audio as public_upload_audio_handler, + handle_public_video as public_upload_video_handler, + handle_public_voice as public_upload_voice_handler, + handle_public_url as public_upload_url_handler, +) from .unknown import handle as unknown_handler from .help import handle as help_handler from .about import handle as about_handler from .settings import handle as settings_handler from .yd_auth import handle as yd_auth_handler from .yd_revoke import handle as yd_revoke_handler -from .upload import handle_photo as upload_photo_handler -from .upload import handle_file as upload_file_handler -from .upload import handle_audio as upload_audio_handler -from .upload import handle_video as upload_video_handler -from .upload import handle_voice as upload_voice_handler -from .upload import handle_url as upload_url_handler from .create_folder import handle as create_folder_handler from .publish import handle as publish_handler from .unpublish import handle as unpublish_handler diff --git a/src/blueprints/telegram_bot/webhook/commands/help.py b/src/blueprints/telegram_bot/webhook/commands/help.py index c8ef69c..9cb4d14 100644 --- a/src/blueprints/telegram_bot/webhook/commands/help.py +++ b/src/blueprints/telegram_bot/webhook/commands/help.py @@ -28,27 +28,33 @@ def handle(*args, **kwargs): "\n" f"{CommandName.UPLOAD_PHOTO.value} — upload a photo. " "Original name will be not saved, quality of photo will be decreased. " - "You can send photo without this command." + "You can send photo without this command. " + f"Use {CommandName.PUBLIC_UPLOAD_PHOTO.value} for public uploading." "\n" f"{CommandName.UPLOAD_FILE.value} — upload a file. " "Original name will be saved. " "For photos, original quality will be saved. " - "You can send file without this command." + "You can send file without this command. " + f"Use {CommandName.PUBLIC_UPLOAD_FILE.value} for public uploading." "\n" f"{CommandName.UPLOAD_AUDIO.value} — upload an audio. " "Original name will be saved, original type may be changed. " - "You can send audio file without this command." + "You can send audio file without this command. " + f"Use {CommandName.PUBLIC_UPLOAD_AUDIO.value} for public uploading." "\n" f"{CommandName.UPLOAD_VIDEO.value} — upload a video. " "Original name will be saved, original type may be changed. " - "You can send video file without this command." + "You can send video file without this command. " + f"Use {CommandName.PUBLIC_UPLOAD_VIDEO.value} for public uploading." "\n" f"{CommandName.UPLOAD_VOICE.value} — upload a voice. " - "You can send voice file without this command." + "You can send voice file without this command. " + f"Use {CommandName.PUBLIC_UPLOAD_VOICE.value} for public uploading." "\n" f"{CommandName.UPLOAD_URL.value} — upload a file using direct URL. " "Original name will be saved. " - "You can send direct URL to a file without this command." + "You can send direct URL to a file without this command. " + f"Use {CommandName.PUBLIC_UPLOAD_URL.value} for public uploading." "\n" f"{CommandName.PUBLISH.value} — publish a file or folder that already exists. " "Send full name of item to publish with this command. " diff --git a/src/blueprints/telegram_bot/webhook/commands/upload.py b/src/blueprints/telegram_bot/webhook/commands/upload.py index e0c6b45..b57aa7c 100644 --- a/src/blueprints/telegram_bot/webhook/commands/upload.py +++ b/src/blueprints/telegram_bot/webhook/commands/upload.py @@ -1,5 +1,6 @@ from abc import ABCMeta, abstractmethod from typing import Union +from collections import deque from flask import g, current_app @@ -13,6 +14,7 @@ from src.blueprints.telegram_bot._common.yandex_disk import ( upload_file_with_url, get_element_info, + publish_item, YandexAPIRequestError, YandexAPICreateFolderError, YandexAPIUploadFileError, @@ -125,6 +127,15 @@ def create_help_message(self) -> str: """ pass + @property + def public_upload(self) -> bool: + """ + :returns: + Upload file and then publish it. + Defaults to `False`. + """ + return False + def get_attachment( self, message: TelegramMessage @@ -331,12 +342,29 @@ def long_task(): download_url=download_url ): success = status["success"] - text = "" + text_content = deque() is_html_text = False if success: - info = None full_path = f"{folder_path}/{file_name}" + is_private_message = (not self.public_upload) + + if self.public_upload: + try: + publish_item( + user_access_token, + full_path + ) + except Exception as error: + print(error) + text_content.append( + "\n" + "Failed to publish. Type to do it:" + "\n" + f"{CommandName.PUBLISH.value} {full_path}" + ) + + info = None try: info = get_element_info( @@ -344,29 +372,43 @@ def long_task(): full_path, get_public_info=False ) - except Exception: - pass - - if info: - text = create_element_info_html_text( - info, - include_private_info=True - ) - is_html_text = True - else: - text = ( - "Successfully completed, but failed " - "to get information about this file." + except Exception as error: + print(error) + text_content.append( "\n" - "Type to see information:" + "Failed to get information. Type to do it:" "\n" f"{CommandName.ELEMENT_INFO.value} {full_path}" ) + + if text_content: + text_content.append( + "It is successfully uploaded, " + "but i failed to perform some actions. " + "You need to execute them manually." + ) + text_content.reverse() + + if info: + # extra line before info + if text_content: + text_content.append("") + + is_html_text = True + info_text = create_element_info_html_text( + info, + include_private_info=is_private_message + ) + text_content.append(info_text) else: # You shouldn't use HTML for this, # because `upload_status` can be a same upload_status = status["status"] - text = f"Status: {upload_status}" + text_content.append( + f"Status: {upload_status}" + ) + + text = "\n".join(text_content) self.reply_to_message( message.message_id, @@ -508,7 +550,8 @@ def raw_data_type(self): def create_help_message(self): return ( - "Send a photos that you want to upload." + "Send a photos that you want to upload" + f"{' and publish' if self.public_upload else ''}." "\n\n" "Note:" "\n" @@ -558,7 +601,8 @@ def raw_data_type(self): def create_help_message(self): return ( - "Send a files that you want to upload." + "Send a files that you want to upload" + f"{' and publish' if self.public_upload else ''}." "\n\n" "Note:" "\n" @@ -595,7 +639,8 @@ def raw_data_type(self): def create_help_message(self): return ( - "Send a music that you want to upload." + "Send a music that you want to upload" + f"{' and publish' if self.public_upload else ''}." "\n\n" "Note:" "\n" @@ -646,7 +691,8 @@ def raw_data_type(self): def create_help_message(self): return ( - "Send a video that you want to upload." + "Send a video that you want to upload" + f"{' and publish' if self.public_upload else ''}." "\n\n" "Note:" "\n" @@ -681,7 +727,8 @@ def raw_data_type(self): def create_help_message(self): return ( - "Send a voice message that you want to upload." + "Send a voice message that you want to upload" + f"{' and publish' if self.public_upload else ''}." ) @@ -708,7 +755,8 @@ def raw_data_type(self): def create_help_message(self): return ( - "Send a direct URL to file that you want to upload." + "Send a direct URL to file that you want to upload" + f"{' and publish' if self.public_upload else ''}." "\n\n" "Note:" "\n" @@ -724,9 +772,84 @@ def create_file_name(self, attachment, file): return attachment.split("/")[-1] +class PublicHandler: + """ + Handles public uploading. + """ + @property + def public_upload(self): + return True + + +class PublicPhotoHandler(PublicHandler, PhotoHandler): + """ + Handles public uploading of photo. + """ + @staticmethod + def handle(*args, **kwargs): + handler = PublicPhotoHandler() + handler.upload(*args, **kwargs) + + +class PublicFileHandler(PublicHandler, FileHandler): + """ + Handles public uploading of file. + """ + @staticmethod + def handle(*args, **kwargs): + handler = PublicFileHandler() + handler.upload(*args, **kwargs) + + +class PublicAudioHandler(PublicHandler, AudioHandler): + """ + Handles public uploading of audio. + """ + @staticmethod + def handle(*args, **kwargs): + handler = PublicAudioHandler() + handler.upload(*args, **kwargs) + + +class PublicVideoHandler(PublicHandler, VideoHandler): + """ + Handles public uploading of video. + """ + @staticmethod + def handle(*args, **kwargs): + handler = PublicVideoHandler() + handler.upload(*args, **kwargs) + + +class PublicVoiceHandler(PublicHandler, VoiceHandler): + """ + Handles public uploading of voice. + """ + @staticmethod + def handle(*args, **kwargs): + handler = PublicVoiceHandler() + handler.upload(*args, **kwargs) + + +class PublicURLHandler(PublicHandler, URLHandler): + """ + Handles public uploading of direct URL to file. + """ + @staticmethod + def handle(*args, **kwargs): + handler = PublicURLHandler() + handler.upload(*args, **kwargs) + + handle_photo = PhotoHandler.handle handle_file = FileHandler.handle handle_audio = AudioHandler.handle handle_video = VideoHandler.handle handle_voice = VoiceHandler.handle handle_url = URLHandler.handle +handle_public_photo = PublicPhotoHandler.handle +handle_public_file = PublicFileHandler.handle +handle_public_audio = PublicAudioHandler.handle +handle_public_video = PublicVideoHandler.handle +handle_public_voice = PublicVoiceHandler.handle +handle_public_url = PublicURLHandler.handle diff --git a/src/blueprints/telegram_bot/webhook/dispatcher.py b/src/blueprints/telegram_bot/webhook/dispatcher.py index deefd35..0fdf40d 100644 --- a/src/blueprints/telegram_bot/webhook/dispatcher.py +++ b/src/blueprints/telegram_bot/webhook/dispatcher.py @@ -174,6 +174,12 @@ def direct_dispatch( CommandName.UPLOAD_VIDEO.value: commands.upload_video_handler, CommandName.UPLOAD_VOICE.value: commands.upload_voice_handler, CommandName.UPLOAD_URL.value: commands.upload_url_handler, + CommandName.PUBLIC_UPLOAD_PHOTO.value: commands.public_upload_photo_handler, # noqa + CommandName.PUBLIC_UPLOAD_FILE.value: commands.public_upload_file_handler, # noqa + CommandName.PUBLIC_UPLOAD_AUDIO.value: commands.public_upload_audio_handler, # noqa + CommandName.PUBLIC_UPLOAD_VIDEO.value: commands.public_upload_video_handler, # noqa + CommandName.PUBLIC_UPLOAD_VOICE.value: commands.public_upload_voice_handler, # noqa + CommandName.PUBLIC_UPLOAD_URL.value: commands.public_upload_url_handler, # noqa CommandName.CREATE_FOLDER.value: commands.create_folder_handler, CommandName.PUBLISH.value: commands.publish_handler, CommandName.UNPUBLISH.value: commands.unpublish_handler, From 318eb5348201ea4a070fb5a45510e3542574a1b3 Mon Sep 17 00:00:00 2001 From: Sergey Kuznetsov Date: Mon, 23 Nov 2020 16:43:56 +0300 Subject: [PATCH 064/103] /upload_url: result name will contain only last path --- CHANGELOG.md | 1 + src/blueprints/telegram_bot/webhook/commands/upload.py | 3 ++- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index d35b6ed..628e2f2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -20,6 +20,7 @@ - `/create_folder`: now it will wait for folder name if you send empty command, not deny operation. - `/upload`: on success it will return information about uploaded file, not plain status. +- `/upload_url`: result name will not contain parameters, queries and fragments. ### Fixed diff --git a/src/blueprints/telegram_bot/webhook/commands/upload.py b/src/blueprints/telegram_bot/webhook/commands/upload.py index b57aa7c..8077b67 100644 --- a/src/blueprints/telegram_bot/webhook/commands/upload.py +++ b/src/blueprints/telegram_bot/webhook/commands/upload.py @@ -1,6 +1,7 @@ from abc import ABCMeta, abstractmethod from typing import Union from collections import deque +from urllib.parse import urlparse from flask import g, current_app @@ -769,7 +770,7 @@ def get_attachment(self, message: TelegramMessage): return message.get_entity_value(self.raw_data_key) def create_file_name(self, attachment, file): - return attachment.split("/")[-1] + return urlparse(attachment).path.split("/")[-1] class PublicHandler: From 6faff85bb95ba94d34c67224a4b827bbdd652f7c Mon Sep 17 00:00:00 2001 From: Sergey Kuznetsov Date: Mon, 23 Nov 2020 16:57:14 +0300 Subject: [PATCH 065/103] Refactoring of common/yandex_disk.py --- .../telegram_bot/_common/yandex_disk.py | 160 ++++++++++-------- 1 file changed, 88 insertions(+), 72 deletions(-) diff --git a/src/blueprints/telegram_bot/_common/yandex_disk.py b/src/blueprints/telegram_bot/_common/yandex_disk.py index 226d95b..2c9883d 100644 --- a/src/blueprints/telegram_bot/_common/yandex_disk.py +++ b/src/blueprints/telegram_bot/_common/yandex_disk.py @@ -6,6 +6,9 @@ from src.api import yandex +# region Exceptions + + class YandexAPIRequestError(Exception): """ Unknown error occurred during Yandex.Disk API request @@ -81,6 +84,68 @@ class YandexAPIExceededNumberOfStatusChecksError(Exception): pass +# endregion + + +# region Helpers + + +def create_yandex_error_text(data: dict) -> str: + """ + :returns: + Human error message from Yandex error response. + """ + error_name = data.get("error", "?") + error_description = ( + data.get("message") or + data.get("description") or + "?" + ) + + return (f"{error_name}: {error_description}") + + +def is_error_yandex_response(data: dict) -> bool: + """ + :returns: Yandex response contains error or not. + """ + return ("error" in data) + + +def yandex_operation_is_success(data: dict) -> bool: + """ + :returns: + Yandex response contains status which + indicates that operation is successfully ended. + """ + return ( + ("status" in data) and + (data["status"] == "success") + ) + + +def yandex_operation_is_failed(data: dict) -> bool: + """ + :returns: + Yandex response contains status which + indicates that operation is failed. + """ + return ( + ("status" in data) and + (data["status"] in ( + # Yandex documentation is different in some places + "failure", + "failed" + )) + ) + + +# endregion + + +# region API + + def create_folder( user_access_token: str, folder_name: str @@ -94,10 +159,12 @@ def create_folder( already exists, for example) from all folder names except last one. - :returns: Last (for last folder name) HTTP Status code. + :returns: + Last (for last folder name) HTTP Status code. - :raises: YandexAPIRequestError - :raises: YandexAPICreateFolderError + :raises: + `YandexAPIRequestError`, + `YandexAPICreateFolderError`. """ folders = [x for x in folder_name.split("/") if x] folder_path = "" @@ -122,7 +189,7 @@ def create_folder( if ( (last_status_code == 201) or (last_status_code in allowed_errors) or - (not is_error_yandex_response(response)) + not is_error_yandex_response(response) ): continue @@ -140,8 +207,9 @@ def publish_item( """ Publish an item that already exists on Yandex.Disk. - :raises: YandexAPIRequestError - :raises: YandexAPIPublishItemError + :raises: + `YandexAPIRequestError`, + `YandexAPIPublishItemError`. """ try: response = yandex.publish( @@ -154,11 +222,9 @@ def publish_item( response = response["content"] is_error = is_error_yandex_response(response) - if (is_error): + if is_error: raise YandexAPIPublishItemError( - create_yandex_error_text( - response - ) + create_yandex_error_text(response) ) @@ -169,8 +235,9 @@ def unpublish_item( """ Unpublish an item that already exists on Yandex.Disk. - :raises: YandexAPIRequestError - :raises: YandexAPIUnpublishItemError + :raises: + `YandexAPIRequestError`, + `YandexAPIUnpublishItemError`. """ try: response = yandex.unpublish( @@ -183,11 +250,9 @@ def unpublish_item( response = response["content"] is_error = is_error_yandex_response(response) - if (is_error): + if is_error: raise YandexAPIUnpublishItemError( - create_yandex_error_text( - response - ) + create_yandex_error_text(response) ) @@ -315,9 +380,11 @@ def get_disk_info(user_access_token: str) -> dict: - https://yandex.ru/dev/disk/api/reference/capacity-docpage/ - https://dev.yandex.net/disk-polygon/#!/v147disk - :returns: Information about user Yandex.Disk. + :returns: + Information about user Yandex.Disk. - :raises: YandexAPIRequestError + :raises: + `YandexAPIRequestError`. """ try: response = yandex.get_disk_info(user_access_token) @@ -327,11 +394,9 @@ def get_disk_info(user_access_token: str) -> dict: response = response["content"] is_error = is_error_yandex_response(response) - if (is_error): + if is_error: raise YandexAPIUploadFileError( - create_yandex_error_text( - response - ) + create_yandex_error_text(response) ) return response @@ -442,53 +507,4 @@ def get_element_info( return response -def is_error_yandex_response(data: dict) -> bool: - """ - :returns: Yandex response contains error or not. - """ - return ("error" in data) - - -def create_yandex_error_text(data: dict) -> str: - """ - :returns: Human error message from Yandex error response. - """ - error_name = data.get( - "error", - "?" - ) - error_description = ( - data.get("message") or - data.get("description") or - "?" - ) - - return (f"{error_name}: {error_description}") - - -def yandex_operation_is_success(data: dict) -> bool: - """ - :returns: - Yandex response contains status which - indicates that operation is successfully ended. - """ - return ( - ("status" in data) and - (data["status"] == "success") - ) - - -def yandex_operation_is_failed(data: dict) -> bool: - """ - :returns: - Yandex response contains status which - indicates that operation is failed. - """ - return ( - ("status" in data) and - (data["status"] in ( - # Yandex documentation is different in some places - "failure", - "failed" - )) - ) +# endregion From 928af1e7e10b338c70f97625aea7caec7772216f Mon Sep 17 00:00:00 2001 From: Sergey Kuznetsov Date: Tue, 24 Nov 2020 12:48:15 +0300 Subject: [PATCH 066/103] Fix a bug when : character in path led to error --- CHANGELOG.md | 1 + .../telegram_bot/_common/yandex_disk.py | 134 ++++++++++++++++-- 2 files changed, 122 insertions(+), 13 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 628e2f2..1ea0fa0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -27,6 +27,7 @@ - `/create_folder`: fixed a bug when bot could remove `/create_folder` occurrences from folder name. - `/create_folder`: fixed a bug when bot don't send any responses on invalid folder name. - Wrong information in help message for `/upload_video`. +- A bug when paths with `:` in name (for example, `Telegram Bot/folder:test`) led to `DiskPathFormatError` from Yandex. ## Project diff --git a/src/blueprints/telegram_bot/_common/yandex_disk.py b/src/blueprints/telegram_bot/_common/yandex_disk.py index 2c9883d..2454302 100644 --- a/src/blueprints/telegram_bot/_common/yandex_disk.py +++ b/src/blueprints/telegram_bot/_common/yandex_disk.py @@ -1,5 +1,6 @@ from time import sleep -from typing import Generator +from typing import Generator, Deque +from collections import deque from flask import current_app @@ -90,6 +91,105 @@ class YandexAPIExceededNumberOfStatusChecksError(Exception): # region Helpers +class YandexDiskPath: + """ + Yandex.Disk path to resource. + + - you should use this class, not raw strings from user! + """ + def __init__(self, *args): + """ + :param *args: + List of raw paths from user. + """ + self.separator = "/" + self.disk_namespace = "disk:" + self.trash_namespace = "trash:" + self.raw_paths = args + + def get_absolute_path(self) -> Deque[str]: + """ + :returns: + Iterable of resource names without separator. + Join result with separator will be a valid Yandex.Disk path. + + :examples: + 1) `self.raw_paths = ["Telegram Bot/test", "name.jpg"]` -> + `["disk:", "Telegram Bot", "test", "name.jpg"]`. + 2) `self.raw_paths = ["disk:/Telegram Bot//test", "/name.jpg/"]` -> + `["disk:", "Telegram Bot", "test", "name.jpg"]`. + """ + data = deque() + + for raw_path in self.raw_paths: + data.extend( + [x for x in raw_path.split(self.separator) if x] + ) + + if not data: + data.append(self.disk_namespace) + + namespace = data[0] + is_valid_namepsace = namespace in ( + self.disk_namespace, + self.trash_namespace + ) + + # Yandex.Disk path must starts with some namespace + # (Disk, Trash, etc.). Paths without valid namespace + # are invalid! However, they can work without namespace. + # But paths without namespace can lead to unexpected + # error at any time. So, you always should add namespace. + # For example, `Telegram Bot/12` will work, but + # `Telegram Bot/12:12` will lead to `DiskPathFormatError` + # from Yandex because Yandex assumes `12` as namespace. + # `disk:/Telegram Bot/12:12` will work fine. + if not is_valid_namepsace: + # Let's use Disk namespace by default + data.appendleft(self.disk_namespace) + + return data + + def create_absolute_path(self) -> str: + """ + :returns: + Valid absolute path. + """ + data = self.get_absolute_path() + + # if path is only namespace, then + # it should end with separator, + # otherwise there should be no + # separator at the end + if (len(data) == 1): + return f"{data.pop()}{self.separator}" + else: + return self.separator.join(data) + + def generate_absolute_path( + self, + include_namespace=True + ) -> Generator[str, None, None]: + """ + :yields: + Valid absolute path piece by piece. + + :examples: + 1) `create_absolute_path()` -> `disk:/Telegram Bot/folder/file.jpg` + `generate_absolute_path(True)` -> `[disk:/, disk:/Telegram Bot, + disk:/Telegram Bot/folder, disk:/Telegram Bot/folder/file.jpg]`. + """ + data = self.get_absolute_path() + absolute_path = data.popleft() + + if include_namespace: + yield f"{absolute_path}{self.separator}" + + for element in data: + absolute_path += f"{self.separator}{element}" + yield absolute_path + + def create_yandex_error_text(data: dict) -> str: """ :returns: @@ -166,19 +266,18 @@ def create_folder( `YandexAPIRequestError`, `YandexAPICreateFolderError`. """ - folders = [x for x in folder_name.split("/") if x] - folder_path = "" - last_status_code = 201 # root always created + path = YandexDiskPath(folder_name) + resources = path.generate_absolute_path(True) + last_status_code = 201 # namespace always created allowed_errors = [409] - for folder in folders: + for resource in resources: result = None - folder_path = f"{folder_path}/{folder}" try: result = yandex.create_folder( user_access_token, - path=folder_path + path=resource ) except Exception as error: raise YandexAPIRequestError(error) @@ -211,10 +310,13 @@ def publish_item( `YandexAPIRequestError`, `YandexAPIPublishItemError`. """ + path = YandexDiskPath(absolute_item_path) + absolute_path = path.create_absolute_path() + try: response = yandex.publish( user_access_token, - path=absolute_item_path + path=absolute_path ) except Exception as error: raise YandexAPIRequestError(error) @@ -239,10 +341,13 @@ def unpublish_item( `YandexAPIRequestError`, `YandexAPIUnpublishItemError`. """ + path = YandexDiskPath(absolute_item_path) + absolute_path = path.create_absolute_path() + try: response = yandex.unpublish( user_access_token, - path=absolute_item_path + path=absolute_path ) except Exception as error: raise YandexAPIRequestError(error) @@ -294,15 +399,15 @@ def upload_file_with_url( folder_name=folder_path ) - folders = [x for x in folder_path.split("/") if x] - full_path = "/".join(folders + [file_name]) + path = YandexDiskPath(folder_path, file_name) + absolute_path = path.create_absolute_path() response = None try: response = yandex.upload_file_with_url( user_access_token, url=download_url, - path=full_path + path=absolute_path ) except Exception as error: raise YandexAPIRequestError(error) @@ -451,10 +556,13 @@ def get_element_info( :raises: `YandexAPIRequestError`, `YandexAPIGetElementInfoError`. """ + path = YandexDiskPath(absolute_element_path) + absolute_path = path.create_absolute_path() + try: response = yandex.get_element_info( user_access_token, - path=absolute_element_path, + path=absolute_path, preview_crop=preview_crop, preview_size=preview_size, limit=embedded_elements_limit, From 99bb597ca93ca6cb6856335aa7b82f22d105c6f5 Mon Sep 17 00:00:00 2001 From: Sergey Kuznetsov Date: Tue, 24 Nov 2020 13:00:17 +0300 Subject: [PATCH 067/103] /upload_voice: result name will be ISO 8601 datetime --- CHANGELOG.md | 1 + src/blueprints/telegram_bot/webhook/commands/upload.py | 4 ++++ 2 files changed, 5 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 1ea0fa0..09784d2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -21,6 +21,7 @@ - `/create_folder`: now it will wait for folder name if you send empty command, not deny operation. - `/upload`: on success it will return information about uploaded file, not plain status. - `/upload_url`: result name will not contain parameters, queries and fragments. +- `/upload_voice`: result name will be ISO 8601 date (for example, `2020-11-24T09:57:46+00:00`), not ID from Telegram. ### Fixed diff --git a/src/blueprints/telegram_bot/webhook/commands/upload.py b/src/blueprints/telegram_bot/webhook/commands/upload.py index 8077b67..fc4eb02 100644 --- a/src/blueprints/telegram_bot/webhook/commands/upload.py +++ b/src/blueprints/telegram_bot/webhook/commands/upload.py @@ -6,6 +6,7 @@ from flask import g, current_app from src.api import telegram +from src.blueprints._common.utils import get_current_iso_datetime from src.blueprints.telegram_bot._common.telegram_interface import ( Message as TelegramMessage ) @@ -732,6 +733,9 @@ def create_help_message(self): f"{' and publish' if self.public_upload else ''}." ) + def create_file_name(self, attachment, file): + return get_current_iso_datetime() + class URLHandler(AttachmentHandler): """ From 7d790d427f90e09b316d9040f921043162cebc22 Mon Sep 17 00:00:00 2001 From: Sergey Kuznetsov Date: Tue, 24 Nov 2020 13:46:21 +0300 Subject: [PATCH 068/103] Add /disk_info command --- CHANGELOG.md | 1 + info/info.json | 4 + .../telegram_bot/_common/command_names.py | 1 + .../telegram_bot/webhook/commands/__init__.py | 1 + .../webhook/commands/disk_info.py | 139 ++++++++++++++++++ .../telegram_bot/webhook/commands/help.py | 2 + .../telegram_bot/webhook/dispatcher.py | 3 +- 7 files changed, 150 insertions(+), 1 deletion(-) create mode 100644 src/blueprints/telegram_bot/webhook/commands/disk_info.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 09784d2..4412ff0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -13,6 +13,7 @@ - `/unpublish`: unpublishing of files or folders. - `/space`: getting of information about remaining Yandex.Disk space. - `/element_info`: getting of information about file or folder. +- `/disk_info`: getting of information about Yandex.Disk. - Handling of files that exceed file size limit in 20 MB. At the moment the bot will asnwer with warning message, not trying to upload that file. [#3](https://github.com/Amaimersion/yandex-disk-telegram-bot/issues/3) - Help messages for each upload command will be sended when there are no suitable input data. diff --git a/info/info.json b/info/info.json index b884812..484cfa8 100644 --- a/info/info.json +++ b/info/info.json @@ -89,6 +89,10 @@ "command": "element_info", "description": "Information about file or folder" }, + { + "command": "disk_info", + "description": "Information about your Yandex.Disk" + }, { "command": "grant_access", "description": "Grant me an access to your Yandex.Disk" diff --git a/src/blueprints/telegram_bot/_common/command_names.py b/src/blueprints/telegram_bot/_common/command_names.py index f58997a..420578b 100644 --- a/src/blueprints/telegram_bot/_common/command_names.py +++ b/src/blueprints/telegram_bot/_common/command_names.py @@ -29,3 +29,4 @@ class CommandName(Enum): UNPUBLISH = "/unpublish" SPACE = "/space" ELEMENT_INFO = "/element_info" + DISK_INFO = "/disk_info" diff --git a/src/blueprints/telegram_bot/webhook/commands/__init__.py b/src/blueprints/telegram_bot/webhook/commands/__init__.py index 5895a34..386b302 100644 --- a/src/blueprints/telegram_bot/webhook/commands/__init__.py +++ b/src/blueprints/telegram_bot/webhook/commands/__init__.py @@ -47,3 +47,4 @@ from .unpublish import handle as unpublish_handler from .space import handle as space_handler from .element_info import handle as element_info_handler +from .disk_info import handle as disk_info_handler diff --git a/src/blueprints/telegram_bot/webhook/commands/disk_info.py b/src/blueprints/telegram_bot/webhook/commands/disk_info.py new file mode 100644 index 0000000..094d863 --- /dev/null +++ b/src/blueprints/telegram_bot/webhook/commands/disk_info.py @@ -0,0 +1,139 @@ +from collections import deque + +from flask import g, current_app + +from src.api import telegram +from src.blueprints._common.utils import bytes_to_human_binary +from src.blueprints.telegram_bot._common.yandex_disk import ( + get_disk_info, + YandexAPIRequestError +) +from ._common.responses import cancel_command +from ._common.decorators import ( + yd_access_token_required, + get_db_data +) + + +@yd_access_token_required +@get_db_data +def handle(*args, **kwargs): + """ + Handles `/disk_info` command. + """ + chat_id = kwargs.get( + "chat_id", + g.telegram_chat.id + ) + user = g.db_user + access_token = user.yandex_disk_token.get_access_token() + info = None + + try: + info = get_disk_info(access_token) + except YandexAPIRequestError as error: + cancel_command(chat_id) + raise error + + text = create_disk_info_html_text(info) + + telegram.send_message( + chat_id=chat_id, + text=text, + parse_mode="HTML", + disable_web_page_preview=True + ) + + +def create_disk_info_html_text(info: dict) -> str: + """ + :param info: + https://dev.yandex.net/disk-polygon/?lang=ru&tld=ru#!/v147disk/GetDisk + """ + text = deque() + + if "user" in info: + data = info["user"] + + if "display_name" in data: + text.append( + f"User — Name: {data['display_name']}" + ) + + if "login" in data: + text.append( + f"User — Login: {data['login']}" + ) + + if "country" in data: + text.append( + f"User — Country: {data['country']}" + ) + + if "is_paid" in info: + value = "?" + + if info["is_paid"]: + value = "Yes" + else: + value = "No" + + text.append( + f"Paid: {value}" + ) + + if "total_space" in info: + value = bytes_to_string(info["total_space"]) + + text.append( + f"Total space: {value}" + ) + + if "used_space" in info: + value = bytes_to_string(info["used_space"]) + + text.append( + f"Used space: {value}" + ) + + if "trash_size" in info: + value = bytes_to_string(info["trash_size"]) + + text.append( + f"Trash size: {value}" + ) + + if ( + ("total_space" in info) and + ("used_space" in info) and + ("trash_size" in info) + ): + bytes_count = ( + info["total_space"] - + info["used_space"] - + info["trash_size"] + ) + value = bytes_to_string(bytes_count) + + text.append( + f"Free space: {value}" + ) + + if "max_file_size" in info: + value = bytes_to_string(info["max_file_size"]) + + text.append( + f"Maximum file size: {value}" + ) + + return "\n".join(text) + + +def bytes_to_string(bytes_count: int) -> str: + value = f"{bytes_count:,} bytes" + + if (bytes_count >= 1000): + decimal = bytes_to_human_binary(bytes_count) + value = f"{decimal} ({bytes_count:,} bytes)" + + return value diff --git a/src/blueprints/telegram_bot/webhook/commands/help.py b/src/blueprints/telegram_bot/webhook/commands/help.py index 9cb4d14..6b365a0 100644 --- a/src/blueprints/telegram_bot/webhook/commands/help.py +++ b/src/blueprints/telegram_bot/webhook/commands/help.py @@ -73,6 +73,8 @@ def handle(*args, **kwargs): "\n" f"{CommandName.ELEMENT_INFO.value} — get information about file or folder. " "Send full path of element with this command." + "\n" + f"{CommandName.DISK_INFO.value} — get information about your Yandex.Disk. " "\n\n" "Yandex.Disk Access" "\n" diff --git a/src/blueprints/telegram_bot/webhook/dispatcher.py b/src/blueprints/telegram_bot/webhook/dispatcher.py index 0fdf40d..75a0693 100644 --- a/src/blueprints/telegram_bot/webhook/dispatcher.py +++ b/src/blueprints/telegram_bot/webhook/dispatcher.py @@ -184,7 +184,8 @@ def direct_dispatch( CommandName.PUBLISH.value: commands.publish_handler, CommandName.UNPUBLISH.value: commands.unpublish_handler, CommandName.SPACE.value: commands.space_handler, - CommandName.ELEMENT_INFO.value: commands.element_info_handler + CommandName.ELEMENT_INFO.value: commands.element_info_handler, + CommandName.DISK_INFO.value: commands.disk_info_handler } handler = routes.get(command, fallback) From 1e0df947698f9a34c46ed17b5559f871465e062c Mon Sep 17 00:00:00 2001 From: Sergey Kuznetsov Date: Tue, 24 Nov 2020 14:37:33 +0300 Subject: [PATCH 069/103] Rename /space to /space_info --- CHANGELOG.md | 2 +- info/info.json | 8 ++++---- src/blueprints/telegram_bot/_common/command_names.py | 2 +- src/blueprints/telegram_bot/webhook/commands/__init__.py | 2 +- src/blueprints/telegram_bot/webhook/commands/help.py | 4 ++-- .../webhook/commands/{space.py => space_info.py} | 0 src/blueprints/telegram_bot/webhook/dispatcher.py | 2 +- 7 files changed, 10 insertions(+), 10 deletions(-) rename src/blueprints/telegram_bot/webhook/commands/{space.py => space_info.py} (100%) diff --git a/CHANGELOG.md b/CHANGELOG.md index 4412ff0..11760e2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,7 +11,7 @@ - `/public_upload_photo`, `/public_upload_file`, `/public_upload_audio`, `/public_upload_video`, `/public_upload_voice`, `/public_upload_url`: uploading of files and then publishing them. - `/publish`: publishing of files or folders. - `/unpublish`: unpublishing of files or folders. -- `/space`: getting of information about remaining Yandex.Disk space. +- `/space_info`: getting of information about remaining Yandex.Disk space. - `/element_info`: getting of information about file or folder. - `/disk_info`: getting of information about Yandex.Disk. - Handling of files that exceed file size limit in 20 MB. At the moment the bot will asnwer with warning message, not trying to upload that file. [#3](https://github.com/Amaimersion/yandex-disk-telegram-bot/issues/3) diff --git a/info/info.json b/info/info.json index 484cfa8..3e5ef2d 100644 --- a/info/info.json +++ b/info/info.json @@ -81,14 +81,14 @@ "command": "create_folder", "description": "Create a folder using OS path" }, - { - "command": "space", - "description": "Information about remaining space" - }, { "command": "element_info", "description": "Information about file or folder" }, + { + "command": "space_info", + "description": "Information about remaining space" + }, { "command": "disk_info", "description": "Information about your Yandex.Disk" diff --git a/src/blueprints/telegram_bot/_common/command_names.py b/src/blueprints/telegram_bot/_common/command_names.py index 420578b..812b943 100644 --- a/src/blueprints/telegram_bot/_common/command_names.py +++ b/src/blueprints/telegram_bot/_common/command_names.py @@ -27,6 +27,6 @@ class CommandName(Enum): CREATE_FOLDER = "/create_folder" PUBLISH = "/publish" UNPUBLISH = "/unpublish" - SPACE = "/space" + SPACE_INFO = "/space_info" ELEMENT_INFO = "/element_info" DISK_INFO = "/disk_info" diff --git a/src/blueprints/telegram_bot/webhook/commands/__init__.py b/src/blueprints/telegram_bot/webhook/commands/__init__.py index 386b302..0e436c5 100644 --- a/src/blueprints/telegram_bot/webhook/commands/__init__.py +++ b/src/blueprints/telegram_bot/webhook/commands/__init__.py @@ -45,6 +45,6 @@ from .create_folder import handle as create_folder_handler from .publish import handle as publish_handler from .unpublish import handle as unpublish_handler -from .space import handle as space_handler +from .space_info import handle as space_info_handler from .element_info import handle as element_info_handler from .disk_info import handle as disk_info_handler diff --git a/src/blueprints/telegram_bot/webhook/commands/help.py b/src/blueprints/telegram_bot/webhook/commands/help.py index 6b365a0..7213e36 100644 --- a/src/blueprints/telegram_bot/webhook/commands/help.py +++ b/src/blueprints/telegram_bot/webhook/commands/help.py @@ -69,11 +69,11 @@ def handle(*args, **kwargs): "Folder name should starts from root, " f'nested folders should be separated with "{to_code("/")}" character.' "\n" - f"{CommandName.SPACE.value} — get information about remaining space." - "\n" f"{CommandName.ELEMENT_INFO.value} — get information about file or folder. " "Send full path of element with this command." "\n" + f"{CommandName.SPACE_INFO.value} — get information about remaining space." + "\n" f"{CommandName.DISK_INFO.value} — get information about your Yandex.Disk. " "\n\n" "Yandex.Disk Access" diff --git a/src/blueprints/telegram_bot/webhook/commands/space.py b/src/blueprints/telegram_bot/webhook/commands/space_info.py similarity index 100% rename from src/blueprints/telegram_bot/webhook/commands/space.py rename to src/blueprints/telegram_bot/webhook/commands/space_info.py diff --git a/src/blueprints/telegram_bot/webhook/dispatcher.py b/src/blueprints/telegram_bot/webhook/dispatcher.py index 75a0693..dbaf62b 100644 --- a/src/blueprints/telegram_bot/webhook/dispatcher.py +++ b/src/blueprints/telegram_bot/webhook/dispatcher.py @@ -183,7 +183,7 @@ def direct_dispatch( CommandName.CREATE_FOLDER.value: commands.create_folder_handler, CommandName.PUBLISH.value: commands.publish_handler, CommandName.UNPUBLISH.value: commands.unpublish_handler, - CommandName.SPACE.value: commands.space_handler, + CommandName.SPACE_INFO.value: commands.space_info_handler, CommandName.ELEMENT_INFO.value: commands.element_info_handler, CommandName.DISK_INFO.value: commands.disk_info_handler } From f55cad48ac327089f64cbd2b6d44cc91e220233a Mon Sep 17 00:00:00 2001 From: Sergey Kuznetsov Date: Tue, 24 Nov 2020 15:02:16 +0300 Subject: [PATCH 070/103] Refactoring of space_info.py --- .../webhook/commands/space_info.py | 23 +++++++++---------- 1 file changed, 11 insertions(+), 12 deletions(-) diff --git a/src/blueprints/telegram_bot/webhook/commands/space_info.py b/src/blueprints/telegram_bot/webhook/commands/space_info.py index 3fe874e..3258cd1 100644 --- a/src/blueprints/telegram_bot/webhook/commands/space_info.py +++ b/src/blueprints/telegram_bot/webhook/commands/space_info.py @@ -7,6 +7,7 @@ from plotly.io import to_image from src.api import telegram +from src.blueprints._common.utils import get_current_iso_datetime from src.blueprints.telegram_bot._common.yandex_disk import ( get_disk_info, YandexAPIRequestError @@ -14,10 +15,7 @@ from src.blueprints.telegram_bot._common.command_names import ( CommandName ) -from ._common.responses import ( - cancel_command, - AbortReason -) +from ._common.responses import cancel_command from ._common.decorators import ( yd_access_token_required, get_db_data @@ -28,7 +26,7 @@ @get_db_data def handle(*args, **kwargs): """ - Handles `/publish` command. + Handles `/space_info` command. """ user = g.db_user chat_id = kwargs.get( @@ -44,15 +42,16 @@ def handle(*args, **kwargs): cancel_command(chat_id) raise error - current_date = get_current_date() + current_utc_date = get_current_utc_datetime() + current_iso_date = get_current_iso_datetime() jpeg_image = create_space_chart( total_space=disk_info["total_space"], used_space=disk_info["used_space"], trash_size=disk_info["trash_size"], - caption=current_date + caption=current_utc_date ) - filename = f"{to_filename(current_date)}.jpg" - file_caption = f"Yandex.Disk space at {current_date}" + filename = f"{to_filename(current_iso_date)}.jpg" + file_caption = f"Yandex.Disk space at {current_utc_date}" telegram.send_photo( chat_id=chat_id, @@ -75,8 +74,8 @@ def create_space_chart( Creates Yandex.Disk space chart. - all sizes (total, used, trash) should be - specified in binary bytes (B). They will be - converted to binary gigabytes (GB). + specified in binary bytes. They will be + converted to binary gigabytes. :returns: JPEG image as bytes. """ @@ -165,7 +164,7 @@ def b_to_gb(value: int) -> int: return (value / 1024 / 1024 / 1024) -def get_current_date() -> str: +def get_current_utc_datetime() -> str: """ :returns: Current date as string representation. """ From d16ca946040b752909a663aec11e425e14d4a363 Mon Sep 17 00:00:00 2001 From: Sergey Kuznetsov Date: Tue, 24 Nov 2020 15:32:53 +0300 Subject: [PATCH 071/103] Add /commands command --- CHANGELOG.md | 1 + info/info.json | 4 + .../telegram_bot/_common/command_names.py | 1 + .../telegram_bot/webhook/commands/__init__.py | 1 + .../webhook/commands/commands_list.py | 130 ++++++++++++++++++ .../telegram_bot/webhook/commands/help.py | 3 + .../telegram_bot/webhook/dispatcher.py | 3 +- 7 files changed, 142 insertions(+), 1 deletion(-) create mode 100644 src/blueprints/telegram_bot/webhook/commands/commands_list.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 11760e2..674a432 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -14,6 +14,7 @@ - `/space_info`: getting of information about remaining Yandex.Disk space. - `/element_info`: getting of information about file or folder. - `/disk_info`: getting of information about Yandex.Disk. +- `/commands`: full list of available commands without help message. - Handling of files that exceed file size limit in 20 MB. At the moment the bot will asnwer with warning message, not trying to upload that file. [#3](https://github.com/Amaimersion/yandex-disk-telegram-bot/issues/3) - Help messages for each upload command will be sended when there are no suitable input data. diff --git a/info/info.json b/info/info.json index 3e5ef2d..fb481a6 100644 --- a/info/info.json +++ b/info/info.json @@ -21,6 +21,10 @@ "command": "about", "description": "Read about me" }, + { + "command": "commands", + "description": "See full list of commands" + }, { "command": "upload_photo", "description": "Upload a photo with quality loss" diff --git a/src/blueprints/telegram_bot/_common/command_names.py b/src/blueprints/telegram_bot/_common/command_names.py index 812b943..5c91be8 100644 --- a/src/blueprints/telegram_bot/_common/command_names.py +++ b/src/blueprints/telegram_bot/_common/command_names.py @@ -30,3 +30,4 @@ class CommandName(Enum): SPACE_INFO = "/space_info" ELEMENT_INFO = "/element_info" DISK_INFO = "/disk_info" + COMMANDS_LIST = "/commands" diff --git a/src/blueprints/telegram_bot/webhook/commands/__init__.py b/src/blueprints/telegram_bot/webhook/commands/__init__.py index 0e436c5..dfb8bba 100644 --- a/src/blueprints/telegram_bot/webhook/commands/__init__.py +++ b/src/blueprints/telegram_bot/webhook/commands/__init__.py @@ -48,3 +48,4 @@ from .space_info import handle as space_info_handler from .element_info import handle as element_info_handler from .disk_info import handle as disk_info_handler +from .commands_list import handle as commands_list_handler diff --git a/src/blueprints/telegram_bot/webhook/commands/commands_list.py b/src/blueprints/telegram_bot/webhook/commands/commands_list.py new file mode 100644 index 0000000..95b8b83 --- /dev/null +++ b/src/blueprints/telegram_bot/webhook/commands/commands_list.py @@ -0,0 +1,130 @@ +from collections import deque + +from flask import g + +from src.api import telegram +from src.blueprints.telegram_bot._common.command_names import CommandName + + +def handle(*args, **kwargs): + """ + Handles `/commands` command. + """ + chat_id = kwargs.get( + "chat_id", + g.telegram_chat.id + ) + text = create_commands_list_html_text() + + telegram.send_message( + chat_id=chat_id, + text=text, + parse_mode="HTML" + ) + + +def create_commands_list_html_text() -> str: + content = ( + { + "name": "Yandex.Disk", + "commands": ( + { + "name": CommandName.UPLOAD_PHOTO.value + }, + { + "name": CommandName.PUBLIC_UPLOAD_PHOTO.value + }, + { + "name": CommandName.UPLOAD_FILE.value + }, + { + "name": CommandName.PUBLIC_UPLOAD_FILE.value + }, + { + "name": CommandName.UPLOAD_AUDIO.value + }, + { + "name": CommandName.PUBLIC_UPLOAD_AUDIO.value + }, + { + "name": CommandName.UPLOAD_VIDEO.value + }, + { + "name": CommandName.PUBLIC_UPLOAD_VIDEO.value + }, + { + "name": CommandName.UPLOAD_VOICE.value + }, + { + "name": CommandName.PUBLIC_UPLOAD_VOICE.value + }, + { + "name": CommandName.UPLOAD_URL.value + }, + { + "name": CommandName.PUBLIC_UPLOAD_URL.value + }, + { + "name": CommandName.PUBLISH.value + }, + { + "name": CommandName.UNPUBLISH.value + }, + { + "name": CommandName.CREATE_FOLDER.value + }, + { + "name": CommandName.ELEMENT_INFO.value + }, + { + "name": CommandName.SPACE_INFO.value + }, + { + "name": CommandName.DISK_INFO.value + } + ) + }, + { + "name": "Yandex.Disk Access", + "commands": ( + { + "name": CommandName.YD_AUTH.value + }, + { + "name": CommandName.YD_REVOKE.value + } + ) + }, + { + "name": "Settings", + "commands": ( + { + "name": CommandName.SETTINGS.value + }, + ) + }, + { + "name": "Information", + "commands": ( + { + "name": CommandName.ABOUT.value + }, + { + "name": CommandName.COMMANDS_LIST.value + } + ) + } + ) + text = deque() + + for group in content: + text.append( + f"{group['name']}" + ) + + for command in group["commands"]: + text.append(command["name"]) + + text.append("") + + return "\n".join(text) diff --git a/src/blueprints/telegram_bot/webhook/commands/help.py b/src/blueprints/telegram_bot/webhook/commands/help.py index 7213e36..bd9e02d 100644 --- a/src/blueprints/telegram_bot/webhook/commands/help.py +++ b/src/blueprints/telegram_bot/webhook/commands/help.py @@ -89,6 +89,9 @@ def handle(*args, **kwargs): "Information" "\n" f"{CommandName.ABOUT.value} — read about me" + "\n" + f"{CommandName.COMMANDS_LIST.value} — see full list of " + "available commands without help message" ) chat_id = kwargs.get( diff --git a/src/blueprints/telegram_bot/webhook/dispatcher.py b/src/blueprints/telegram_bot/webhook/dispatcher.py index dbaf62b..ca5d688 100644 --- a/src/blueprints/telegram_bot/webhook/dispatcher.py +++ b/src/blueprints/telegram_bot/webhook/dispatcher.py @@ -185,7 +185,8 @@ def direct_dispatch( CommandName.UNPUBLISH.value: commands.unpublish_handler, CommandName.SPACE_INFO.value: commands.space_info_handler, CommandName.ELEMENT_INFO.value: commands.element_info_handler, - CommandName.DISK_INFO.value: commands.disk_info_handler + CommandName.DISK_INFO.value: commands.disk_info_handler, + CommandName.COMMANDS_LIST.value: commands.commands_list_handler } handler = routes.get(command, fallback) From a51c2c04cd9ef50998013235db614036b4641054 Mon Sep 17 00:00:00 2001 From: Sergey Kuznetsov Date: Tue, 24 Nov 2020 17:26:51 +0300 Subject: [PATCH 072/103] Refactoring of commands content generation --- CHANGELOG.md | 1 + .../commands/_common/commands_content.py | 178 ++++++++++++++++++ .../webhook/commands/commands_list.py | 95 +--------- .../telegram_bot/webhook/commands/help.py | 140 ++++++-------- 4 files changed, 237 insertions(+), 177 deletions(-) create mode 100644 src/blueprints/telegram_bot/webhook/commands/_common/commands_content.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 674a432..23d32bc 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,7 @@ ### Improved - Text of some bot responses. +- Formatting of help message. ### Added diff --git a/src/blueprints/telegram_bot/webhook/commands/_common/commands_content.py b/src/blueprints/telegram_bot/webhook/commands/_common/commands_content.py new file mode 100644 index 0000000..47fdb5b --- /dev/null +++ b/src/blueprints/telegram_bot/webhook/commands/_common/commands_content.py @@ -0,0 +1,178 @@ +# flake8: noqa +# Long lines are allowed here, but try to avoid them + + +from flask import current_app + +from src.blueprints.telegram_bot._common.command_names import CommandName + + +def to_code(text: str) -> str: + return f"{text}" + + +commands_html_content = ( + { + "name": "Yandex.Disk", + "commands": ( + { + "name": CommandName.UPLOAD_PHOTO.value, + "help": ( + "upload a photo. Original name will be " + "not saved, quality of photo will be decreased. " + "You can send photo without this command. " + f"Use {CommandName.PUBLIC_UPLOAD_PHOTO.value} " + "for public uploading" + ) + }, + { + "name": CommandName.PUBLIC_UPLOAD_PHOTO.value + }, + { + "name": CommandName.UPLOAD_FILE.value, + "help": ( + "upload a file. Original name will be saved. " + "For photos, original quality will be saved. " + "You can send file without this command. " + f"Use {CommandName.PUBLIC_UPLOAD_FILE.value} " + "for public uploading" + ) + }, + { + "name": CommandName.PUBLIC_UPLOAD_FILE.value + }, + { + "name": CommandName.UPLOAD_AUDIO.value, + "help": ( + "upload an audio. Original name will be saved, " + "original type may be changed. " + "You can send audio file without this command. " + f"Use {CommandName.PUBLIC_UPLOAD_AUDIO.value} " + "for public uploading" + ) + }, + { + "name": CommandName.PUBLIC_UPLOAD_AUDIO.value + }, + { + "name": CommandName.UPLOAD_VIDEO.value, + "help": ( + "upload a video. Original name will be saved, " + "original type may be changed. " + "You can send video file without this command. " + f"Use {CommandName.PUBLIC_UPLOAD_VIDEO.value} " + "for public uploading" + ) + }, + { + "name": CommandName.PUBLIC_UPLOAD_VIDEO.value + }, + { + "name": CommandName.UPLOAD_VOICE.value, + "help": ( + "upload a voice. " + "You can send voice file without this command. " + f"Use {CommandName.PUBLIC_UPLOAD_VOICE.value} " + "for public uploading" + ) + }, + { + "name": CommandName.PUBLIC_UPLOAD_VOICE.value + }, + { + "name": CommandName.UPLOAD_URL.value, + "help": ( + "upload a file using direct URL. " + "Original name will be saved. " + "You can send direct URL to a " + "file without this command. " + f"Use {CommandName.PUBLIC_UPLOAD_URL.value} " + "for public uploading" + ) + }, + { + "name": CommandName.PUBLIC_UPLOAD_URL.value + }, + { + "name": CommandName.PUBLISH.value, + "help": ( + "publish a file or folder that " + "already exists. Send full name of " + "element to publish with this command. " + f'Example: {to_code(f"Telegram Bot/files/photo.jpeg")}' + ) + }, + { + "name": CommandName.UNPUBLISH.value, + "help": ( + "unpublish a file or folder that " + "already exists. Send full name of " + "element to unpublish with this command. " + f'Example: {to_code(f"Telegram Bot/files/photo.jpeg")}' + ) + }, + { + "name": CommandName.CREATE_FOLDER.value, + "help": ( + "create a folder. Send folder name to " + "create with this command. Folder name " + "should starts from root, nested folders should be " + f'separated with "{to_code("/")}" character' + ) + }, + { + "name": CommandName.ELEMENT_INFO.value, + "help": ( + "get information about file or folder. " + "Send full path of element with this command" + ) + }, + { + "name": CommandName.SPACE_INFO.value, + "help": "get information about remaining space" + }, + { + "name": CommandName.DISK_INFO.value, + "help": "get information about your Yandex.Disk" + } + ) + }, + { + "name": "Yandex.Disk Access", + "commands": ( + { + "name": CommandName.YD_AUTH.value, + "help": "grant me access to your Yandex.Disk" + }, + { + "name": CommandName.YD_REVOKE.value, + "help": "revoke my access to your Yandex.Disk" + } + ) + }, + { + "name": "Settings", + "commands": ( + { + "name": CommandName.SETTINGS.value, + "help": "edit your settings" + }, + ) + }, + { + "name": "Information", + "commands": ( + { + "name": CommandName.ABOUT.value, + "help": "read about me" + }, + { + "name": CommandName.COMMANDS_LIST.value, + "help": ( + "see full list of available " + "commands without help message" + ) + } + ) + } +) diff --git a/src/blueprints/telegram_bot/webhook/commands/commands_list.py b/src/blueprints/telegram_bot/webhook/commands/commands_list.py index 95b8b83..473ed9b 100644 --- a/src/blueprints/telegram_bot/webhook/commands/commands_list.py +++ b/src/blueprints/telegram_bot/webhook/commands/commands_list.py @@ -3,7 +3,7 @@ from flask import g from src.api import telegram -from src.blueprints.telegram_bot._common.command_names import CommandName +from ._common.commands_content import commands_html_content def handle(*args, **kwargs): @@ -24,100 +24,9 @@ def handle(*args, **kwargs): def create_commands_list_html_text() -> str: - content = ( - { - "name": "Yandex.Disk", - "commands": ( - { - "name": CommandName.UPLOAD_PHOTO.value - }, - { - "name": CommandName.PUBLIC_UPLOAD_PHOTO.value - }, - { - "name": CommandName.UPLOAD_FILE.value - }, - { - "name": CommandName.PUBLIC_UPLOAD_FILE.value - }, - { - "name": CommandName.UPLOAD_AUDIO.value - }, - { - "name": CommandName.PUBLIC_UPLOAD_AUDIO.value - }, - { - "name": CommandName.UPLOAD_VIDEO.value - }, - { - "name": CommandName.PUBLIC_UPLOAD_VIDEO.value - }, - { - "name": CommandName.UPLOAD_VOICE.value - }, - { - "name": CommandName.PUBLIC_UPLOAD_VOICE.value - }, - { - "name": CommandName.UPLOAD_URL.value - }, - { - "name": CommandName.PUBLIC_UPLOAD_URL.value - }, - { - "name": CommandName.PUBLISH.value - }, - { - "name": CommandName.UNPUBLISH.value - }, - { - "name": CommandName.CREATE_FOLDER.value - }, - { - "name": CommandName.ELEMENT_INFO.value - }, - { - "name": CommandName.SPACE_INFO.value - }, - { - "name": CommandName.DISK_INFO.value - } - ) - }, - { - "name": "Yandex.Disk Access", - "commands": ( - { - "name": CommandName.YD_AUTH.value - }, - { - "name": CommandName.YD_REVOKE.value - } - ) - }, - { - "name": "Settings", - "commands": ( - { - "name": CommandName.SETTINGS.value - }, - ) - }, - { - "name": "Information", - "commands": ( - { - "name": CommandName.ABOUT.value - }, - { - "name": CommandName.COMMANDS_LIST.value - } - ) - } - ) text = deque() - for group in content: + for group in commands_html_content: text.append( f"{group['name']}" ) diff --git a/src/blueprints/telegram_bot/webhook/commands/help.py b/src/blueprints/telegram_bot/webhook/commands/help.py index bd9e02d..0f34435 100644 --- a/src/blueprints/telegram_bot/webhook/commands/help.py +++ b/src/blueprints/telegram_bot/webhook/commands/help.py @@ -1,109 +1,81 @@ -# flake8: noqa +from collections import deque from flask import g, current_app from src.api import telegram -from src.blueprints.telegram_bot._common.command_names import CommandName +from ._common.commands_content import ( + to_code, + commands_html_content +) def handle(*args, **kwargs): """ Handles `/help` command. """ + chat_id = kwargs.get( + "chat_id", + g.telegram_chat.id + ) + text = create_help_html_text() + telegram.send_message( + chat_id=chat_id, + parse_mode="HTML", + text=text, + disable_web_page_preview=True + ) + + +def create_help_html_text() -> str: yd_upload_default_folder = current_app.config[ "YANDEX_DISK_API_DEFAULT_UPLOAD_FOLDER" ] file_size_limit_in_mb = int(current_app.config[ "TELEGRAM_API_MAX_FILE_SIZE" ] / 1024 / 1024) + bullet_char = "•" + text = deque() - text = ( - "You can control me by sending these commands:" - "\n\n" - "Yandex.Disk" - "\n" - f'For uploading "{to_code(yd_upload_default_folder)}" folder is used by default.' - "\n" - f'Maximum size of every upload (except URL) is {file_size_limit_in_mb} MB.' - "\n" - f"{CommandName.UPLOAD_PHOTO.value} — upload a photo. " - "Original name will be not saved, quality of photo will be decreased. " - "You can send photo without this command. " - f"Use {CommandName.PUBLIC_UPLOAD_PHOTO.value} for public uploading." - "\n" - f"{CommandName.UPLOAD_FILE.value} — upload a file. " - "Original name will be saved. " - "For photos, original quality will be saved. " - "You can send file without this command. " - f"Use {CommandName.PUBLIC_UPLOAD_FILE.value} for public uploading." - "\n" - f"{CommandName.UPLOAD_AUDIO.value} — upload an audio. " - "Original name will be saved, original type may be changed. " - "You can send audio file without this command. " - f"Use {CommandName.PUBLIC_UPLOAD_AUDIO.value} for public uploading." - "\n" - f"{CommandName.UPLOAD_VIDEO.value} — upload a video. " - "Original name will be saved, original type may be changed. " - "You can send video file without this command. " - f"Use {CommandName.PUBLIC_UPLOAD_VIDEO.value} for public uploading." - "\n" - f"{CommandName.UPLOAD_VOICE.value} — upload a voice. " - "You can send voice file without this command. " - f"Use {CommandName.PUBLIC_UPLOAD_VOICE.value} for public uploading." - "\n" - f"{CommandName.UPLOAD_URL.value} — upload a file using direct URL. " - "Original name will be saved. " - "You can send direct URL to a file without this command. " - f"Use {CommandName.PUBLIC_UPLOAD_URL.value} for public uploading." - "\n" - f"{CommandName.PUBLISH.value} — publish a file or folder that already exists. " - "Send full name of item to publish with this command. " - f'Example: {to_code(f"/{yd_upload_default_folder}/files/photo.jpeg")}' - "\n" - f"{CommandName.UNPUBLISH.value} — unpublish a file or folder that already exists. " - "Send full name of item to unpublish with this command. " - f'Example: {to_code(f"/{yd_upload_default_folder}/files/photo.jpeg")}' - "\n" - f"{CommandName.CREATE_FOLDER.value} — create a folder. " - "Send folder name to create with this command. " - "Folder name should starts from root, " - f'nested folders should be separated with "{to_code("/")}" character.' - "\n" - f"{CommandName.ELEMENT_INFO.value} — get information about file or folder. " - "Send full path of element with this command." - "\n" - f"{CommandName.SPACE_INFO.value} — get information about remaining space." - "\n" - f"{CommandName.DISK_INFO.value} — get information about your Yandex.Disk. " + text.append( + "You can interact with " + 'Yandex.Disk ' + "by using me. To control me send following commands." "\n\n" - "Yandex.Disk Access" + "Note:" "\n" - f"{CommandName.YD_AUTH.value} — grant me access to your Yandex.Disk" + f"{bullet_char} for uploading " + f'"{to_code(yd_upload_default_folder)}" ' + "folder is used by default," "\n" - f"{CommandName.YD_REVOKE.value} — revoke my access to your Yandex.Disk" - "\n\n" - "Settings" - "\n" - f"{CommandName.SETTINGS.value} — edit your settings" - "\n\n" - "Information" + f"{bullet_char} maximum size of every upload " + f"(except URL) is {file_size_limit_in_mb} MB." "\n" - f"{CommandName.ABOUT.value} — read about me" - "\n" - f"{CommandName.COMMANDS_LIST.value} — see full list of " - "available commands without help message" ) - chat_id = kwargs.get( - "chat_id", - g.telegram_chat.id - ) - telegram.send_message( - chat_id=chat_id, - parse_mode="HTML", - text=text - ) + for group in commands_html_content: + group_name = group["name"] + commands = group["commands"] + group_added = False + + text.append( + f"{group_name}" + ) + + for command in commands: + command_name = command["name"] + help_message = command.get("help") + + if help_message: + text.append( + f"{bullet_char} {command_name} — {help_message}." + ) + group_added = True + if group_added: + # extra line + text.append("") + else: + # we don't want empty group name + text.pop() -def to_code(text: str) -> str: - return f"{text}" + return "\n".join(text) From 0c3103a62fc42501fa862383a5441c262ce109d6 Mon Sep 17 00:00:00 2001 From: Sergey Kuznetsov Date: Tue, 24 Nov 2020 22:40:16 +0300 Subject: [PATCH 073/103] Add db indexes --- CHANGELOG.md | 1 + ...add_index_property_for_frequent_columns.py | 44 +++++++++++++++++++ src/database/models/chat.py | 1 + src/database/models/user.py | 1 + src/database/models/yandex_disk_token.py | 1 + 5 files changed, 48 insertions(+) create mode 100644 migrations/versions/c8db92e01cf4_add_index_property_for_frequent_columns.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 23d32bc..af4c36c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -48,6 +48,7 @@ - Support for different env-type files (based on current environment). Initially it was only for production. - Web Site: 302 (redirect to Telegram) will be returned instead of 404 (not found page), but only in production mode. - Debug configuration for VSCode. +- DB: add indexes for frequent using columns. ### Changed diff --git a/migrations/versions/c8db92e01cf4_add_index_property_for_frequent_columns.py b/migrations/versions/c8db92e01cf4_add_index_property_for_frequent_columns.py new file mode 100644 index 0000000..5c6735a --- /dev/null +++ b/migrations/versions/c8db92e01cf4_add_index_property_for_frequent_columns.py @@ -0,0 +1,44 @@ +"""Add index property for frequent columns + +Revision ID: c8db92e01cf4 +Revises: 67ffbddd6efe +Create Date: 2020-11-24 22:27:44.498208 + +""" +from alembic import op +import sqlalchemy as sa + + +# revision identifiers, used by Alembic. +revision = 'c8db92e01cf4' +down_revision = '67ffbddd6efe' +branch_labels = None +depends_on = None + + +def upgrade(): + # ### commands auto generated by Alembic - please adjust! ### + with op.batch_alter_table('chats', schema=None) as batch_op: + batch_op.create_index(batch_op.f('ix_chats_telegram_id'), ['telegram_id'], unique=True) + + with op.batch_alter_table('users', schema=None) as batch_op: + batch_op.create_index(batch_op.f('ix_users_telegram_id'), ['telegram_id'], unique=True) + + with op.batch_alter_table('yandex_disk_tokens', schema=None) as batch_op: + batch_op.create_index(batch_op.f('ix_yandex_disk_tokens_user_id'), ['user_id'], unique=True) + + # ### end Alembic commands ### + + +def downgrade(): + # ### commands auto generated by Alembic - please adjust! ### + with op.batch_alter_table('yandex_disk_tokens', schema=None) as batch_op: + batch_op.drop_index(batch_op.f('ix_yandex_disk_tokens_user_id')) + + with op.batch_alter_table('users', schema=None) as batch_op: + batch_op.drop_index(batch_op.f('ix_users_telegram_id')) + + with op.batch_alter_table('chats', schema=None) as batch_op: + batch_op.drop_index(batch_op.f('ix_chats_telegram_id')) + + # ### end Alembic commands ### diff --git a/src/database/models/chat.py b/src/database/models/chat.py index 0f3f72d..8891cef 100644 --- a/src/database/models/chat.py +++ b/src/database/models/chat.py @@ -45,6 +45,7 @@ class Chat(db.Model): telegram_id = db.Column( db.BigInteger, unique=True, + index=True, nullable=False, comment="Unique ID to identificate chat in Telegram" ) diff --git a/src/database/models/user.py b/src/database/models/user.py index 44c46c8..9aa8028 100644 --- a/src/database/models/user.py +++ b/src/database/models/user.py @@ -46,6 +46,7 @@ class User(db.Model): telegram_id = db.Column( db.Integer, unique=True, + index=True, nullable=False, comment="Unique ID to identificate user in Telegram" ) diff --git a/src/database/models/yandex_disk_token.py b/src/database/models/yandex_disk_token.py index 2f2652e..03245b1 100644 --- a/src/database/models/yandex_disk_token.py +++ b/src/database/models/yandex_disk_token.py @@ -69,6 +69,7 @@ class YandexDiskToken(db.Model): db.Integer, db.ForeignKey("users.id"), unique=True, + index=True, nullable=False, comment="Tokens belongs to this user" ) From 2ddfcdbc2b52031e4018f22feb752c59083a3eeb Mon Sep 17 00:00:00 2001 From: Sergey Kuznetsov Date: Wed, 25 Nov 2020 16:03:17 +0300 Subject: [PATCH 074/103] /upload: rename URLHandler class --- .../telegram_bot/webhook/commands/upload.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/src/blueprints/telegram_bot/webhook/commands/upload.py b/src/blueprints/telegram_bot/webhook/commands/upload.py index fc4eb02..fe4c09a 100644 --- a/src/blueprints/telegram_bot/webhook/commands/upload.py +++ b/src/blueprints/telegram_bot/webhook/commands/upload.py @@ -737,13 +737,13 @@ def create_file_name(self, attachment, file): return get_current_iso_datetime() -class URLHandler(AttachmentHandler): +class DirectURLHandler(AttachmentHandler): """ Handles uploading of direct URL to file. """ @staticmethod def handle(*args, **kwargs): - handler = URLHandler() + handler = DirectURLHandler() handler.upload(*args, **kwargs) @property @@ -836,13 +836,13 @@ def handle(*args, **kwargs): handler.upload(*args, **kwargs) -class PublicURLHandler(PublicHandler, URLHandler): +class PublicDirectURLHandler(PublicHandler, DirectURLHandler): """ Handles public uploading of direct URL to file. """ @staticmethod def handle(*args, **kwargs): - handler = PublicURLHandler() + handler = PublicDirectURLHandler() handler.upload(*args, **kwargs) @@ -851,10 +851,10 @@ def handle(*args, **kwargs): handle_audio = AudioHandler.handle handle_video = VideoHandler.handle handle_voice = VoiceHandler.handle -handle_url = URLHandler.handle +handle_url = DirectURLHandler.handle handle_public_photo = PublicPhotoHandler.handle handle_public_file = PublicFileHandler.handle handle_public_audio = PublicAudioHandler.handle handle_public_video = PublicVideoHandler.handle handle_public_voice = PublicVoiceHandler.handle -handle_public_url = PublicURLHandler.handle +handle_public_url = PublicDirectURLHandler.handle From 3e3c9791559ca01beabcc81c01c120b46ad876d7 Mon Sep 17 00:00:00 2001 From: Sergey Kuznetsov Date: Wed, 25 Nov 2020 18:18:16 +0300 Subject: [PATCH 075/103] /upload handler: remove unnecessary getting of message attachment --- .../telegram_bot/webhook/commands/upload.py | 33 ++++++++----------- 1 file changed, 14 insertions(+), 19 deletions(-) diff --git a/src/blueprints/telegram_bot/webhook/commands/upload.py b/src/blueprints/telegram_bot/webhook/commands/upload.py index fe4c09a..63ff247 100644 --- a/src/blueprints/telegram_bot/webhook/commands/upload.py +++ b/src/blueprints/telegram_bot/webhook/commands/upload.py @@ -158,11 +158,12 @@ def get_attachment( def check_message_health( self, - message: TelegramMessage + attachment: Union[dict, str, None] ) -> MessageHealth: """ - :param message: - Incoming Telegram message. + :param attachment: + Attachment of incoming Telegram message. + `None` means there is no attachment. :returns: See `MessageHealth` documentation. @@ -170,20 +171,22 @@ def check_message_health( in case of `ok = true`. """ health = MessageHealth(True) - value = self.get_attachment(message) - if not isinstance(value, self.raw_data_type): + if attachment is None: + health.ok = False + health.abort_reason = AbortReason.NO_SUITABLE_DATA + elif not isinstance(attachment, self.raw_data_type): health.ok = False health.abort_reason = AbortReason.NO_SUITABLE_DATA elif ( - (type(value) in [str]) and - (len(value) == 0) + (type(attachment) in [str]) and + (len(attachment) == 0) ): health.ok = False health.abort_reason = AbortReason.NO_SUITABLE_DATA elif ( - isinstance(value, dict) and - self.is_too_big_file(value) + isinstance(attachment, dict) and + self.is_too_big_file(attachment) ): health.ok = False health.abort_reason = AbortReason.EXCEED_FILE_SIZE_LIMIT @@ -273,7 +276,8 @@ def upload(self, *args, **kwargs) -> None: "message", g.telegram_message ) - message_health = self.check_message_health(message) + attachment = self.get_attachment(message) + message_health = self.check_message_health(attachment) if not message_health.ok: reason = ( @@ -289,15 +293,6 @@ def upload(self, *args, **kwargs) -> None: else: return abort_command(chat_id, reason) - attachment = self.get_attachment(message) - data_is_empty = (attachment is None) - - if data_is_empty: - return self.send_html_message( - chat_id, - self.create_help_message() - ) - try: telegram.send_chat_action( chat_id=chat_id, From ed5f40f9a0d1ec7277159edc291066ee19bc534e Mon Sep 17 00:00:00 2001 From: Sergey Kuznetsov Date: Sat, 28 Nov 2020 00:57:40 +0300 Subject: [PATCH 076/103] /upload_url: add youtube-dl support --- CHANGELOG.md | 1 + README.md | 3 +- info/info.json | 4 +- requirements.txt | 1 + .../telegram_bot/_common/youtube_dl.py | 149 ++++++++++++++++++ .../commands/_common/commands_content.py | 15 +- .../telegram_bot/webhook/commands/upload.py | 127 ++++++++++++++- 7 files changed, 291 insertions(+), 9 deletions(-) create mode 100644 src/blueprints/telegram_bot/_common/youtube_dl.py diff --git a/CHANGELOG.md b/CHANGELOG.md index af4c36c..d8dd260 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -16,6 +16,7 @@ - `/element_info`: getting of information about file or folder. - `/disk_info`: getting of information about Yandex.Disk. - `/commands`: full list of available commands without help message. +- `/upload_url`: added `youtube-dl` support. - Handling of files that exceed file size limit in 20 MB. At the moment the bot will asnwer with warning message, not trying to upload that file. [#3](https://github.com/Amaimersion/yandex-disk-telegram-bot/issues/3) - Help messages for each upload command will be sended when there are no suitable input data. diff --git a/README.md b/README.md index 47b5e6f..66888cf 100644 --- a/README.md +++ b/README.md @@ -30,10 +30,11 @@ - uploading of photos (limit is 20 MB). - uploading of files (limit is 20 MB). -- uploading of files using direct URL. - uploading of audio (limit is 20 MB). - uploading of video (limit is 20 MB). - uploading of voice (limit is 20 MB). +- uploading of files using direct URL. +- uploading of various resources (YouTube, for example) with help of `youtube-dl`. - uploading for public access. - publishing and unpublishing of files or folders. - creating of folders. diff --git a/info/info.json b/info/info.json index fb481a6..d6dae65 100644 --- a/info/info.json +++ b/info/info.json @@ -67,11 +67,11 @@ }, { "command": "upload_url", - "description": "Upload a file using direct URL" + "description": "Upload a resource using URL" }, { "command": "public_upload_url", - "description": "Upload a file using direct URL and publish it" + "description": "Upload a resource using URL and publish it" }, { "command": "publish", diff --git a/requirements.txt b/requirements.txt index 09cc1e8..8903ad9 100644 --- a/requirements.txt +++ b/requirements.txt @@ -43,4 +43,5 @@ text-unidecode==1.3 toml==0.10.2 urllib3==1.25.11 Werkzeug==1.0.1 +youtube-dl==2020.11.24 -e . diff --git a/src/blueprints/telegram_bot/_common/youtube_dl.py b/src/blueprints/telegram_bot/_common/youtube_dl.py new file mode 100644 index 0000000..7ecd6eb --- /dev/null +++ b/src/blueprints/telegram_bot/_common/youtube_dl.py @@ -0,0 +1,149 @@ +import youtube_dl + + +# region Exceptions + + +class CustomYoutubeDLError(Exception): + """ + Base exception for all custom `youtube_dl` errors. + """ + pass + + +class UnsupportedURLError(CustomYoutubeDLError): + """ + Provided URL is not supported by `youtube_dl` or + not allowed by custom rules to be handled. + """ + pass + + +class UnexpectedError(CustomYoutubeDLError): + """ + Some unexpected error occured. + """ + pass + + +# endregion + + +# region Utils + + +class Logger: + """ + Custom logger for `youtube_dl`. + """ + def debug(self, message: str) -> None: + pass + + def warning(self, message: str) -> None: + pass + + def error(self, message: str) -> None: + pass + + +# endregion + + +# region youtube_dl + + +def extract_info(url: str) -> dict: + """ + Extracts info from URL. + This info can be used to download this resource. + + - see this for supported sites - + https://github.com/ytdl-org/youtube-dl/blob/master/docs/supportedsites.md + + :param url: + URL to resource. + You can safely pass any URL + (YouTube video, direct URL to JPG, plain page, etc.) + + :returns: + `direct_url` - can be used to download resource, + `filename` - recommended filename. + If provided URL not supported, then error will be + raised. So, if result is successfully returned, + then it is 100% valid result which can be used to + download resource. + + :raises: + `UnsupportedURLError`, + `UnexpectedError`. + """ + result = { + "direct_url": None, + "filename": None + } + info = None + + # TODO: implement execution timeout, + # because long requests blocks server requests + try: + info = ydl.extract_info(url) + except youtube_dl.DownloadError as error: + raise UnsupportedURLError(str(error)) + except Exception as error: + raise UnexpectedError(str(error)) + + direct_url = info.get("url") + + if not direct_url: + raise UnsupportedURLError( + "youtube_dl didn't return direct URL" + ) + + result["direct_url"] = direct_url + + if info.get("direct"): + result["filename"] = info.get("webpage_url_basename") + else: + result["filename"] = ydl.prepare_filename(info) + + return result + + +# endregion + + +options = { + # We using `youtube_dl` to get information + # for download and pass further to another service + # (Yandex.Disk, for example). + # We don't downloading anything. + # Almost all videos with > Full HD + # don't provide direct MP4, they are + # using MPEG-DASH streaming. + # It is means there is two streams: + # one for video and one for audio. + # These two streams should be converted + # into one to get video with audio. + # It is really expensive for public server. + # Also, videos with high resolution have + # large size, which affects at download time + # by another service (Yandex.Disk, for example). + # Probably it can even break the download. + # So, passing Full HD (maxium) videos is a golden mean. + # `best` is fallback if there is no + # needed things (they almost always presented), + # and in that case final result can be a + # not that user expected. + "format": "[height <= 1080]/best", + "youtube_include_dash_manifest": False, + "logger": Logger(), + # disable downloading at all + "simulate": True, + "skip_download": True, + # Ignore YouTube playlists, because large + # playlists can take long time to parse + # (affects at server response time) + "extract_flat": "in_playlist", + "noplaylist": True +} +ydl = youtube_dl.YoutubeDL(options) diff --git a/src/blueprints/telegram_bot/webhook/commands/_common/commands_content.py b/src/blueprints/telegram_bot/webhook/commands/_common/commands_content.py index 47fdb5b..4d12a5f 100644 --- a/src/blueprints/telegram_bot/webhook/commands/_common/commands_content.py +++ b/src/blueprints/telegram_bot/webhook/commands/_common/commands_content.py @@ -82,10 +82,17 @@ def to_code(text: str) -> str: { "name": CommandName.UPLOAD_URL.value, "help": ( - "upload a file using direct URL. " - "Original name will be saved. " - "You can send direct URL to a " - "file without this command. " + "upload a some resource using URL. " + "For direct URL's to file original name, " + "quality and size will be saved. " + "For URL's to some resource best name and " + "best possible quality will be selected. " + '"Resource" means anything: YouTube video, ' + "Twitch clip, music track, etc. " + "Not everything will work as you expect, " + "but some URL's will. " + 'See this ' + "for all supported sites. " f"Use {CommandName.PUBLIC_UPLOAD_URL.value} " "for public uploading" ) diff --git a/src/blueprints/telegram_bot/webhook/commands/upload.py b/src/blueprints/telegram_bot/webhook/commands/upload.py index 63ff247..cd78588 100644 --- a/src/blueprints/telegram_bot/webhook/commands/upload.py +++ b/src/blueprints/telegram_bot/webhook/commands/upload.py @@ -7,6 +7,7 @@ from src.api import telegram from src.blueprints._common.utils import get_current_iso_datetime +from src.blueprints.telegram_bot._common import youtube_dl from src.blueprints.telegram_bot._common.telegram_interface import ( Message as TelegramMessage ) @@ -772,6 +773,116 @@ def create_file_name(self, attachment, file): return urlparse(attachment).path.split("/")[-1] +class IntellectualURLHandler(DirectURLHandler): + """ + Handles uploading of direct URL to file. + + But unlike `DirectURLHandler`, this handler will + try to guess which content user actually assumed. + For example, if user passed URL to YouTube video, + `DirectURLHandler` will download HTML page, but + `IntellectualURLHandler` will download a video. + + In short, `DirectURLHandler` makes minimum changes + to input content; `IntellectualURLHandler` makes + maximum changes, but these changes focused for + "that user actually wants" and "it should be right". + + What this handler does: + - using `youtube_dl` gets direct URL to + input video/music URL, and gets right filename + - in case if nothing works, then fallback to `DirectURLHandler` + """ + def __init__(self): + super().__init__() + + self.input_url = None + self.youtube_dl_info = None + + @staticmethod + def handle(*args, **kwargs): + handler = IntellectualURLHandler() + handler.upload(*args, **kwargs) + + def create_help_message(self): + return ( + "Send an URL to resource that you want to upload" + f"{' and publish' if self.public_upload else ''}." + "\n\n" + "Note:" + "\n" + "- for direct URL's to file original name, " + "quality and size will be saved" + "\n" + "- for URL's to some resource best name amd " + "best possible quality will be selected" + "\n" + "- i will try to guess what resource you actually assumed. " + "For example, you can send URL to YouTube video or " + "Twitch clip, and video from that URL will be uploaded" + "\n" + "- you can send URL to any resource: video, audio, image, " + "text, page, etc. Not everything will work as you expect, " + "but some URL's will" + "\n" + "- i'm using youtube-dl, if that means anything to you (:" + ) + + def get_attachment(self, message: TelegramMessage): + self.input_url = super().get_attachment(message) + + if not self.input_url: + return None + + best_url = self.input_url + + try: + self.youtube_dl_info = youtube_dl.extract_info( + self.input_url + ) + except youtube_dl.UnsupportedURLError: + # Unsupported URL's is expected here, + # let's treat them as direct URL's to files + pass + except youtube_dl.UnexpectedError as error: + # TODO: + # Something goes wrong in `youtube_dl`. + # It is better to log this error to user, + # because there can be restrictions or limits, + # but there also can be some internal info + # which shouldn't be printed to user. + # At the moment there is no best way for UX, so, + # let's just print this information in logs. + print( + "Unexpected youtube_dl error:", + error + ) + + if self.youtube_dl_info: + best_url = self.youtube_dl_info["direct_url"] + + return best_url + + def create_file_name(self, attachment, file): + input_filename = super().create_file_name( + self.input_url, + file + ) + youtube_dl_filename = None + + if self.youtube_dl_info: + youtube_dl_filename = self.youtube_dl_info.get( + "filename" + ) + + best_filename = ( + youtube_dl_filename or + input_filename + ) + + return best_filename + + class PublicHandler: """ Handles public uploading. @@ -841,15 +952,27 @@ def handle(*args, **kwargs): handler.upload(*args, **kwargs) +class PublicIntellectualURLHandler(PublicHandler, IntellectualURLHandler): + """ + Handles public uploading of direct URL to file. + + - see `IntellectualURLHandler` documentation. + """ + @staticmethod + def handle(*args, **kwargs): + handler = PublicIntellectualURLHandler() + handler.upload(*args, **kwargs) + + handle_photo = PhotoHandler.handle handle_file = FileHandler.handle handle_audio = AudioHandler.handle handle_video = VideoHandler.handle handle_voice = VoiceHandler.handle -handle_url = DirectURLHandler.handle +handle_url = IntellectualURLHandler.handle handle_public_photo = PublicPhotoHandler.handle handle_public_file = PublicFileHandler.handle handle_public_audio = PublicAudioHandler.handle handle_public_video = PublicVideoHandler.handle handle_public_voice = PublicVoiceHandler.handle -handle_public_url = PublicDirectURLHandler.handle +handle_public_url = PublicIntellectualURLHandler.handle From 663a2b7e5e196b1aa896cc4faa9e4173ab0c81de Mon Sep 17 00:00:00 2001 From: Sergey Kuznetsov Date: Sat, 28 Nov 2020 14:06:03 +0300 Subject: [PATCH 077/103] /upload_url: fix a wrong name for url's with empty path --- .../telegram_bot/webhook/commands/upload.py | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/src/blueprints/telegram_bot/webhook/commands/upload.py b/src/blueprints/telegram_bot/webhook/commands/upload.py index cd78588..afa5c56 100644 --- a/src/blueprints/telegram_bot/webhook/commands/upload.py +++ b/src/blueprints/telegram_bot/webhook/commands/upload.py @@ -770,7 +770,16 @@ def get_attachment(self, message: TelegramMessage): return message.get_entity_value(self.raw_data_key) def create_file_name(self, attachment, file): - return urlparse(attachment).path.split("/")[-1] + parse_result = urlparse(attachment) + filename = parse_result.path.split("/")[-1] + + # for example, `https://ya.ru` leads to + # empty path, so, `filename` will be empty + # in that case. Then let it be `ya.ru` + if not filename: + filename = parse_result.netloc + + return filename class IntellectualURLHandler(DirectURLHandler): From 21aecdecd503d0ac441c6ef7f1980ff52370d9aa Mon Sep 17 00:00:00 2001 From: Sergey Kuznetsov Date: Sat, 28 Nov 2020 14:33:56 +0300 Subject: [PATCH 078/103] Refactoring of message when exceed number of status checks --- src/blueprints/telegram_bot/webhook/commands/upload.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/src/blueprints/telegram_bot/webhook/commands/upload.py b/src/blueprints/telegram_bot/webhook/commands/upload.py index afa5c56..9b2bb82 100644 --- a/src/blueprints/telegram_bot/webhook/commands/upload.py +++ b/src/blueprints/telegram_bot/webhook/commands/upload.py @@ -332,6 +332,8 @@ def upload(self, *args, **kwargs) -> None: ] def long_task(): + full_path = f"{folder_path}/{file_name}" + try: for status in upload_file_with_url( user_access_token=user_access_token, @@ -344,7 +346,6 @@ def long_task(): is_html_text = False if success: - full_path = f"{folder_path}/{file_name}" is_private_message = (not self.public_upload) if self.public_upload: @@ -439,7 +440,10 @@ def long_task(): except YandexAPIExceededNumberOfStatusChecksError: error_text = ( "I can't track operation status of " - "this anymore. Perform manual checking." + "this anymore. It can be uploaded " + "after a while. Type to check:" + "\n" + f"{CommandName.ELEMENT_INFO.value} {full_path}" ) return self.reply_to_message( From 547b1d8a6374232368932c18cfeb0697ae697a81 Mon Sep 17 00:00:00 2001 From: Sergey Kuznetsov Date: Thu, 3 Dec 2020 19:51:33 +0300 Subject: [PATCH 079/103] Add RQ and enable it for /upload --- CHANGELOG.md | 2 + requirements.txt | 1 + src/app.py | 7 +- .../telegram_bot/webhook/commands/upload.py | 409 ++++++++++++------ src/configs/flask.py | 27 ++ src/extensions.py | 49 ++- worker.py | 37 ++ 7 files changed, 406 insertions(+), 126 deletions(-) create mode 100644 worker.py diff --git a/CHANGELOG.md b/CHANGELOG.md index d8dd260..81b1e01 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,7 @@ - Text of some bot responses. - Formatting of help message. +- `/upload`: multiple items will be handled at a same time, not one by one. ### Added @@ -46,6 +47,7 @@ - Stateful chat support. Now bot can store custom user data (in different namespaces: user, chat, user in chat); determine Telegram message types; register single use handler (call once for message) with optional timeout for types of message; subscribe handlers with optional timeout for types of messages. - [Console Client](https://yandex.ru/dev/oauth/doc/dg/reference/console-client.html) Yandex.OAuth method. By default it is disabled, and default one is [Auto Code Client](https://yandex.ru/dev/oauth/doc/dg/reference/auto-code-client.html/). +- RQ (job queue). It requires Redis to be enabled, and as Redis it is also optional. However, it is highly recommended to use it. - Support for different env-type files (based on current environment). Initially it was only for production. - Web Site: 302 (redirect to Telegram) will be returned instead of 404 (not found page), but only in production mode. - Debug configuration for VSCode. diff --git a/requirements.txt b/requirements.txt index 8903ad9..2996156 100644 --- a/requirements.txt +++ b/requirements.txt @@ -37,6 +37,7 @@ pytz==2020.1 redis==3.5.3 requests==2.24.0 retrying==1.3.3 +rq==1.7.0 six==1.15.0 SQLAlchemy==1.3.20 text-unidecode==1.3 diff --git a/src/app.py b/src/app.py index beaf55c..05b7ff6 100644 --- a/src/app.py +++ b/src/app.py @@ -15,7 +15,8 @@ from .extensions import ( db, migrate, - redis_client + redis_client, + task_queue ) from .blueprints import ( telegram_bot_blueprint, @@ -68,6 +69,10 @@ def configure_extensions(app: Flask) -> None: # Redis redis_client.init_app(app) + # RQ + if redis_client.is_enabled: + task_queue.init_app(app, redis_client.connection) + def configure_blueprints(app: Flask) -> None: """ diff --git a/src/blueprints/telegram_bot/webhook/commands/upload.py b/src/blueprints/telegram_bot/webhook/commands/upload.py index 9b2bb82..713f56a 100644 --- a/src/blueprints/telegram_bot/webhook/commands/upload.py +++ b/src/blueprints/telegram_bot/webhook/commands/upload.py @@ -6,6 +6,7 @@ from flask import g, current_app from src.api import telegram +from src.extensions import task_queue from src.blueprints._common.utils import get_current_iso_datetime from src.blueprints.telegram_bot._common import youtube_dl from src.blueprints.telegram_bot._common.telegram_interface import ( @@ -96,6 +97,17 @@ def telegram_action(self) -> str: """ pass + @property + @abstractmethod + def telegram_command(self) -> str: + """ + :returns: + With what Telegram command this handler + is associated. It is exact command name + (`/upload_photo`, for example). + """ + pass + @property @abstractmethod def raw_data_key(self) -> str: @@ -263,11 +275,22 @@ def is_too_big_file(self, file: dict) -> bool: @yd_access_token_required @get_db_data - def upload(self, *args, **kwargs) -> None: + def init_upload(self, *args, **kwargs) -> None: """ - Uploads an attachment. + Initializes uploading process of message attachment. + Attachment will be prepared for uploading, and if + everything is ok, then uploading will be automatically + started, otherwise error will be logged back to user. + + - it is expected entry point for dispatcher. + - `*args`, `**kwargs` - arguments from dispatcher. - `*args`, `**kwargs` - arguments from dispatcher. + NOTE: + Depending on app configuration uploading can start + in same or separate process. If it is same process, + then this function will take a long time to complete, + if it is separate process, then this function will + be completed fast. """ chat_id = kwargs.get( "chat_id", @@ -324,148 +347,234 @@ def upload(self, *args, **kwargs) -> None: file["file_path"] ) + message_id = message.message_id user = g.db_user file_name = self.create_file_name(attachment, file) user_access_token = user.yandex_disk_token.get_access_token() folder_path = current_app.config[ "YANDEX_DISK_API_DEFAULT_UPLOAD_FOLDER" ] + arguments = ( + folder_path, + file_name, + download_url, + user_access_token, + chat_id, + message_id + ) - def long_task(): - full_path = f"{folder_path}/{file_name}" + # Everything is fine by this moment. + # Because task workers can be busy, + # it can take a while to start uploading. + # Let's indicate to user that uploading + # process is started and user shouldn't + # send any data again + self.reply_to_message( + message_id, + chat_id, + "Status: pending", + False + ) - try: - for status in upload_file_with_url( - user_access_token=user_access_token, - folder_path=folder_path, - file_name=file_name, - download_url=download_url - ): - success = status["success"] - text_content = deque() - is_html_text = False - - if success: - is_private_message = (not self.public_upload) - - if self.public_upload: - try: - publish_item( - user_access_token, - full_path - ) - except Exception as error: - print(error) - text_content.append( - "\n" - "Failed to publish. Type to do it:" - "\n" - f"{CommandName.PUBLISH.value} {full_path}" - ) - - info = None + if task_queue.is_enabled: + job_timeout = current_app.config[ + "YANDEX_DISK_WORKER_UPLOAD_JOB_TIMEOUT" + ] + ttl = current_app.config[ + "YANDEX_DISK_WORKER_UPLOAD_TTL" + ] + result_ttl = current_app.config[ + "YANDEX_DISK_WORKER_UPLOAD_RESULT_TTL" + ] + failure_ttl = current_app.config[ + "YANDEX_DISK_WORKER_UPLOAD_FAILURE_TTL" + ] + + task_queue.enqueue( + self.start_upload, + args=arguments, + description=self.telegram_command, + job_timeout=job_timeout, + ttl=ttl, + result_ttl=result_ttl, + failure_ttl=failure_ttl + ) + else: + # NOTE: current thread will + # be blocked for a long time + self.start_upload(*arguments) + + def start_upload( + self, + folder_path: str, + file_name: str, + download_url: str, + user_access_token: str, + chat_id: int, + message_id: int + ) -> None: + """ + Starts uploading of provided URL. + + It will send provided URL to Yandex.Disk API, + after that operation monitoring will be started. + See app configuration for monitoring config. + + NOTE: + This function requires long time to complete. + And because it is sync function, it will block + your thread. + + :param folder_path: + Yandex.Disk path where to put file. + :param file_name: + Name (with extension) of result file. + :param download_url: + Direct URL to file. Yandex.Disk will download it. + :param user_access_token: + Access token of user to access Yandex.Disk API. + :param chat_id: + ID of incoming Telegram chat. + :param message_id: + ID of incoming Telegram message. + This message will be reused to edit this message + with new status instead of sending it every time. + :raises: + Raises error if occurs. + """ + full_path = f"{folder_path}/{file_name}" + + try: + for status in upload_file_with_url( + user_access_token=user_access_token, + folder_path=folder_path, + file_name=file_name, + download_url=download_url + ): + success = status["success"] + text_content = deque() + is_html_text = False + + if success: + is_private_message = (not self.public_upload) + + if self.public_upload: try: - info = get_element_info( + publish_item( user_access_token, - full_path, - get_public_info=False + full_path ) except Exception as error: print(error) text_content.append( "\n" - "Failed to get information. Type to do it:" + "Failed to publish. Type to do it:" "\n" - f"{CommandName.ELEMENT_INFO.value} {full_path}" + f"{CommandName.PUBLISH.value} {full_path}" ) - if text_content: - text_content.append( - "It is successfully uploaded, " - "but i failed to perform some actions. " - "You need to execute them manually." - ) - text_content.reverse() + info = None - if info: - # extra line before info - if text_content: - text_content.append("") + try: + info = get_element_info( + user_access_token, + full_path, + get_public_info=False + ) + except Exception as error: + print(error) + text_content.append( + "\n" + "Failed to get information. Type to do it:" + "\n" + f"{CommandName.ELEMENT_INFO.value} {full_path}" + ) - is_html_text = True - info_text = create_element_info_html_text( - info, - include_private_info=is_private_message - ) - text_content.append(info_text) - else: - # You shouldn't use HTML for this, - # because `upload_status` can be a same - upload_status = status["status"] + if text_content: text_content.append( - f"Status: {upload_status}" + "It is successfully uploaded, " + "but i failed to perform some actions. " + "You need to execute them manually." ) + text_content.reverse() - text = "\n".join(text_content) + if info: + # extra line before info + if text_content: + text_content.append("") - self.reply_to_message( - message.message_id, - chat_id, - text, - is_html_text + is_html_text = True + info_text = create_element_info_html_text( + info, + include_private_info=is_private_message + ) + text_content.append(info_text) + else: + # You shouldn't use HTML for this, + # because `upload_status` can be a same + upload_status = status["status"] + text_content.append( + f"Status: {upload_status}" ) - except YandexAPICreateFolderError as error: - error_text = str(error) or ( - "I can't create default upload folder " - "due to an unknown Yandex.Disk error." - ) - return send_yandex_disk_error( + text = "\n".join(text_content) + + self.reply_to_message( + message_id, chat_id, - error_text, - message.message_id - ) - except YandexAPIUploadFileError as error: - error_text = str(error) or ( - "I can't upload this due " - "to an unknown Yandex.Disk error." + text, + is_html_text ) + except YandexAPICreateFolderError as error: + error_text = str(error) or ( + "I can't create default upload folder " + "due to an unknown Yandex.Disk error." + ) - return send_yandex_disk_error( + return send_yandex_disk_error( + chat_id, + error_text, + message_id + ) + except YandexAPIUploadFileError as error: + error_text = str(error) or ( + "I can't upload this due " + "to an unknown Yandex.Disk error." + ) + + return send_yandex_disk_error( + chat_id, + error_text, + message_id + ) + except YandexAPIExceededNumberOfStatusChecksError: + error_text = ( + "I can't track operation status of " + "this anymore. It can be uploaded " + "after a while. Type to check:" + "\n" + f"{CommandName.ELEMENT_INFO.value} {full_path}" + ) + + return self.reply_to_message( + message_id, + chat_id, + error_text + ) + except Exception as error: + if self.sended_message is None: + cancel_command( chat_id, - error_text, - message.message_id - ) - except YandexAPIExceededNumberOfStatusChecksError: - error_text = ( - "I can't track operation status of " - "this anymore. It can be uploaded " - "after a while. Type to check:" - "\n" - f"{CommandName.ELEMENT_INFO.value} {full_path}" + reply_to_message=message_id ) - - return self.reply_to_message( - message.message_id, + else: + cancel_command( chat_id, - error_text + edit_message=self.sended_message.message_id ) - except Exception as error: - if self.sended_message is None: - cancel_command( - chat_id, - reply_to_message=message.message_id - ) - else: - cancel_command( - chat_id, - edit_message=self.sended_message.message_id - ) - - raise error - long_task() + raise error def send_html_message( self, @@ -535,12 +644,16 @@ class PhotoHandler(AttachmentHandler): @staticmethod def handle(*args, **kwargs): handler = PhotoHandler() - handler.upload(*args, **kwargs) + handler.init_upload(*args, **kwargs) @property def telegram_action(self): return "upload_photo" + @property + def telegram_command(self): + return CommandName.UPLOAD_PHOTO.value + @property def raw_data_key(self): return "photo" @@ -587,12 +700,16 @@ class FileHandler(AttachmentHandler): @staticmethod def handle(*args, **kwargs): handler = FileHandler() - handler.upload(*args, **kwargs) + handler.init_upload(*args, **kwargs) @property def telegram_action(self): return "upload_document" + @property + def telegram_command(self): + return CommandName.UPLOAD_FILE.value + @property def raw_data_key(self): return "document" @@ -625,12 +742,16 @@ class AudioHandler(AttachmentHandler): @staticmethod def handle(*args, **kwargs): handler = AudioHandler() - handler.upload(*args, **kwargs) + handler.init_upload(*args, **kwargs) @property def telegram_action(self): return "upload_audio" + @property + def telegram_command(self): + return CommandName.UPLOAD_AUDIO.value + @property def raw_data_key(self): return "audio" @@ -677,12 +798,16 @@ class VideoHandler(AttachmentHandler): @staticmethod def handle(*args, **kwargs): handler = VideoHandler() - handler.upload(*args, **kwargs) + handler.init_upload(*args, **kwargs) @property def telegram_action(self): return "upload_video" + @property + def telegram_command(self): + return CommandName.UPLOAD_VIDEO.value + @property def raw_data_key(self): return "video" @@ -713,12 +838,16 @@ class VoiceHandler(AttachmentHandler): @staticmethod def handle(*args, **kwargs): handler = VoiceHandler() - handler.upload(*args, **kwargs) + handler.init_upload(*args, **kwargs) @property def telegram_action(self): return "upload_audio" + @property + def telegram_command(self): + return CommandName.UPLOAD_VOICE.value + @property def raw_data_key(self): return "voice" @@ -744,12 +873,16 @@ class DirectURLHandler(AttachmentHandler): @staticmethod def handle(*args, **kwargs): handler = DirectURLHandler() - handler.upload(*args, **kwargs) + handler.init_upload(*args, **kwargs) @property def telegram_action(self): return "upload_document" + @property + def telegram_command(self): + return CommandName.UPLOAD_URL.value + @property def raw_data_key(self): return "url" @@ -815,7 +948,7 @@ def __init__(self): @staticmethod def handle(*args, **kwargs): handler = IntellectualURLHandler() - handler.upload(*args, **kwargs) + handler.init_upload(*args, **kwargs) def create_help_message(self): return ( @@ -912,7 +1045,11 @@ class PublicPhotoHandler(PublicHandler, PhotoHandler): @staticmethod def handle(*args, **kwargs): handler = PublicPhotoHandler() - handler.upload(*args, **kwargs) + handler.init_upload(*args, **kwargs) + + @property + def telegram_command(self): + return CommandName.PUBLIC_UPLOAD_PHOTO.value class PublicFileHandler(PublicHandler, FileHandler): @@ -922,7 +1059,11 @@ class PublicFileHandler(PublicHandler, FileHandler): @staticmethod def handle(*args, **kwargs): handler = PublicFileHandler() - handler.upload(*args, **kwargs) + handler.init_upload(*args, **kwargs) + + @property + def telegram_command(self): + return CommandName.PUBLIC_UPLOAD_FILE.value class PublicAudioHandler(PublicHandler, AudioHandler): @@ -932,7 +1073,11 @@ class PublicAudioHandler(PublicHandler, AudioHandler): @staticmethod def handle(*args, **kwargs): handler = PublicAudioHandler() - handler.upload(*args, **kwargs) + handler.init_upload(*args, **kwargs) + + @property + def telegram_command(self): + return CommandName.PUBLIC_UPLOAD_AUDIO.value class PublicVideoHandler(PublicHandler, VideoHandler): @@ -942,7 +1087,11 @@ class PublicVideoHandler(PublicHandler, VideoHandler): @staticmethod def handle(*args, **kwargs): handler = PublicVideoHandler() - handler.upload(*args, **kwargs) + handler.init_upload(*args, **kwargs) + + @property + def telegram_command(self): + return CommandName.PUBLIC_UPLOAD_VIDEO.value class PublicVoiceHandler(PublicHandler, VoiceHandler): @@ -952,7 +1101,11 @@ class PublicVoiceHandler(PublicHandler, VoiceHandler): @staticmethod def handle(*args, **kwargs): handler = PublicVoiceHandler() - handler.upload(*args, **kwargs) + handler.init_upload(*args, **kwargs) + + @property + def telegram_command(self): + return CommandName.PUBLIC_UPLOAD_VOICE.value class PublicDirectURLHandler(PublicHandler, DirectURLHandler): @@ -962,7 +1115,11 @@ class PublicDirectURLHandler(PublicHandler, DirectURLHandler): @staticmethod def handle(*args, **kwargs): handler = PublicDirectURLHandler() - handler.upload(*args, **kwargs) + handler.init_upload(*args, **kwargs) + + @property + def telegram_command(self): + return CommandName.PUBLIC_UPLOAD_URL.value class PublicIntellectualURLHandler(PublicHandler, IntellectualURLHandler): @@ -974,7 +1131,11 @@ class PublicIntellectualURLHandler(PublicHandler, IntellectualURLHandler): @staticmethod def handle(*args, **kwargs): handler = PublicIntellectualURLHandler() - handler.upload(*args, **kwargs) + handler.init_upload(*args, **kwargs) + + @property + def telegram_command(self): + return CommandName.PUBLIC_UPLOAD_URL.value handle_photo = PhotoHandler.handle diff --git a/src/configs/flask.py b/src/configs/flask.py index 7cd34e8..2621243 100644 --- a/src/configs/flask.py +++ b/src/configs/flask.py @@ -82,6 +82,9 @@ class Config: # this value at all. # Set to 0 to disable expiration RUNTIME_DISPOSABLE_HANDLER_EXPIRE = 60 * 10 + # RQ (background tasks queue) is enabled. + # Also depends on `REDIS_URL` + RUNTIME_RQ_ENABLED = True # Flask DEBUG = False @@ -143,6 +146,30 @@ class Config: # in this folder files will be uploaded by default # if user not specified custom folder. YANDEX_DISK_API_DEFAULT_UPLOAD_FOLDER = "Telegram Bot" + # Maximum runtime of uploading process before it’s interrupted. + # In seconds. This value shouldn't be less than + # `YANDEX_DISK_API_CHECK_OPERATION_STATUS_MAX_ATTEMPTS` * + # `YANDEX_DISK_API_CHECK_OPERATION_STATUS_INTERVAL`. + # This timeout should be used only for force quit if + # uploading function start behave incorrectly. + # Use `MAX_ATTEMPTS` and `INTERVAL` for expected quit. + # Applied only if task queue (RQ, for example) is enabled + YANDEX_DISK_WORKER_UPLOAD_JOB_TIMEOUT = 30 + # Maximum queued time of upload function before it's discarded. + # "Queued" means function awaits execution. + # In seconds. `None` for infinite awaiting. + # Applied only if task queue (RQ, for example) is enabled + YANDEX_DISK_WORKER_UPLOAD_TTL = None + # How long successful result of uploading is kept. + # In seconds. + # Applied only if task queue (RQ, for example) is enabled + YANDEX_DISK_WORKER_UPLOAD_RESULT_TTL = 0 + # How long failed result of uploading is kept. + # "Failed result" means function raises an error, + # not any logical error returns from function. + # In seconds. + # Applied only if task queue (RQ, for example) is enabled + YANDEX_DISK_WORKER_UPLOAD_FAILURE_TTL = 0 # Google Analytics GOOGLE_ANALYTICS_UA = os.getenv("GOOGLE_ANALYTICS_UA") diff --git a/src/extensions.py b/src/extensions.py index 49a4ed3..5eb5cb0 100644 --- a/src/extensions.py +++ b/src/extensions.py @@ -16,6 +16,7 @@ from flask_migrate import Migrate from sqlalchemy.pool import NullPool import redis +from rq import Queue as RQ # Database @@ -56,9 +57,13 @@ def __setitem__(self, name, value): def __delitem__(self, name): del self._redis_client[name] + @property + def connection(self) -> redis.Redis: + return self._redis_client + @property def is_enabled(self) -> bool: - return (self._redis_client is not None) + return (self.connection is not None) def init_app(self, app: Flask, **kwargs) -> None: self._redis_client = None @@ -75,3 +80,45 @@ def init_app(self, app: Flask, **kwargs) -> None: redis_client: Union[redis.Redis, FlaskRedis] = FlaskRedis() + + +# Redis Queue + +class RedisQueue: + def __init__(self): + self._queue = None + + def __getattr__(self, name): + return getattr(self._queue, name) + + def __getitem__(self, name): + return self._queue[name] + + def __setitem__(self, name, value): + self._queue[name] = value + + def __delitem__(self, name): + del self._queue[name] + + @property + def is_enabled(self) -> bool: + return (self._queue is not None) + + def init_app( + self, + app: Flask, + redis_connection: redis.Redis, + **kwargs + ) -> None: + enabled = app.config.get("RUNTIME_RQ_ENABLED") + + if not enabled: + return + + self._queue = RQ( + connection=redis_connection, + name="default" + ) + + +task_queue: Union[RQ, RedisQueue] = RedisQueue() diff --git a/worker.py b/worker.py new file mode 100644 index 0000000..c5df9d5 --- /dev/null +++ b/worker.py @@ -0,0 +1,37 @@ +""" +Runs RQ worker. +""" + +import redis +from rq import Worker, Queue, Connection + +from src.app import create_app + + +def main(): + app = create_app() + redis_url = app.config.get("REDIS_URL") + + if not redis_url: + raise Exception("Redis URL is not specified") + + connection = redis.from_url(redis_url) + listen = ['default'] + + with Connection(connection): + # we should bind Flask app context + # to worker context in order worker + # have access to valid `current_app`. + # It will be not actual app that is + # currently being running, but app + # with same ENV configuration + with app.app_context(): + worker = Worker( + map(Queue, listen) + ) + + worker.work() + + +if __name__ == "__main__": + main() From 6ff325cd0349ab1646611d5fe89a702f37f4e7a0 Mon Sep 17 00:00:00 2001 From: Sergey Kuznetsov Date: Thu, 3 Dec 2020 19:52:04 +0300 Subject: [PATCH 080/103] Improve documentation of Flask configuration --- src/configs/flask.py | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/src/configs/flask.py b/src/configs/flask.py index 2621243..67d316f 100644 --- a/src/configs/flask.py +++ b/src/configs/flask.py @@ -52,13 +52,18 @@ class YandexOAuthAPIMethod(Enum): class Config: """ Notes: - - don't remove any key from configuration, because code logic - may depend on this. Instead set disable value (if code logic + - don't remove any keys from configuration, because code logic + can depend on this. Instead set disable value (if code logic supports it); or set empty value and edit code logic to handle such values. - - keep in mind that Heroku have 30 seconds request timeout. - So, if your configuration value can exceed 30 seconds, then - request will be terminated by Heroku. + - keep in mind that Telegram, Heroku, etc. have request timeout. + It is about 30 seconds, but actual value can be different. + If you don't end current request in a long time, then it will + be force closed. Telegram will send new request in that case. + Try to always use background task queue, not block current thread. + If you have no opportunity to use background task queue, then + change current configuration in order request with blocked thread + cannot take long time to complete. """ # Project # name of app that will be used in HTML and so on From eca67450ff5b7d7375caa67fe55c466225470a09 Mon Sep 17 00:00:00 2001 From: Sergey Kuznetsov Date: Thu, 3 Dec 2020 19:59:47 +0300 Subject: [PATCH 081/103] /upload: increase maxium time of checking of operation status --- CHANGELOG.md | 1 + src/configs/flask.py | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 81b1e01..d4df432 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -25,6 +25,7 @@ - `/create_folder`: now it will wait for folder name if you send empty command, not deny operation. - `/upload`: on success it will return information about uploaded file, not plain status. +- `/upload`: increase maxium time of checking of operation status from 10 seconds to 16. - `/upload_url`: result name will not contain parameters, queries and fragments. - `/upload_voice`: result name will be ISO 8601 date (for example, `2020-11-24T09:57:46+00:00`), not ID from Telegram. diff --git a/src/configs/flask.py b/src/configs/flask.py index 67d316f..6c69ab2 100644 --- a/src/configs/flask.py +++ b/src/configs/flask.py @@ -142,7 +142,7 @@ class Config: # maximum number of checks of operation status # (for example, if file is downloaded by Yandex.Disk). # It is blocks request until check ending! - YANDEX_DISK_API_CHECK_OPERATION_STATUS_MAX_ATTEMPTS = 5 + YANDEX_DISK_API_CHECK_OPERATION_STATUS_MAX_ATTEMPTS = 8 # interval in seconds between checks of operation status. # It is blocks request until check ending! # For example, if max. attempts is 5 and interval is 2, From c77a50507a6f6bfc315dcf0c85ec60aa905ff3b4 Mon Sep 17 00:00:00 2001 From: Sergey Kuznetsov Date: Thu, 3 Dec 2020 20:17:10 +0300 Subject: [PATCH 082/103] /upload: fix a bug when edited message dont gets saved --- .../telegram_bot/webhook/commands/upload.py | 17 +++++++++++++---- 1 file changed, 13 insertions(+), 4 deletions(-) diff --git a/src/blueprints/telegram_bot/webhook/commands/upload.py b/src/blueprints/telegram_bot/webhook/commands/upload.py index 713f56a..1cc1235 100644 --- a/src/blueprints/telegram_bot/webhook/commands/upload.py +++ b/src/blueprints/telegram_bot/webhook/commands/upload.py @@ -615,6 +615,8 @@ def reply_to_message( if html_text: enabled_html["parse_mode"] = "HTML" + result = None + if self.sended_message is None: result = telegram.send_message( reply_to_message_id=incoming_message_id, @@ -624,11 +626,8 @@ def reply_to_message( disable_web_page_preview=True, **enabled_html ) - self.sended_message = TelegramMessage( - result["content"] - ) elif (text != self.sended_message.get_text()): - telegram.edit_message_text( + result = telegram.edit_message_text( message_id=self.sended_message.message_id, chat_id=chat_id, text=text, @@ -636,6 +635,16 @@ def reply_to_message( **enabled_html ) + new_message_sended = ( + (result is not None) and + result["ok"] + ) + + if new_message_sended: + self.sended_message = TelegramMessage( + result["content"] + ) + class PhotoHandler(AttachmentHandler): """ From 600f30999938f6e4ed6fcec02dee8f7a64814f27 Mon Sep 17 00:00:00 2001 From: Sergey Kuznetsov Date: Thu, 3 Dec 2020 20:42:45 +0300 Subject: [PATCH 083/103] /element_info: preview will be sended from worker --- .../webhook/commands/element_info.py | 75 +++++++++++++++---- src/configs/flask.py | 3 + 2 files changed, 63 insertions(+), 15 deletions(-) diff --git a/src/blueprints/telegram_bot/webhook/commands/element_info.py b/src/blueprints/telegram_bot/webhook/commands/element_info.py index 5691413..511148b 100644 --- a/src/blueprints/telegram_bot/webhook/commands/element_info.py +++ b/src/blueprints/telegram_bot/webhook/commands/element_info.py @@ -1,5 +1,6 @@ from flask import g, current_app +from src.extensions import task_queue from src.api import telegram from src.api.yandex import make_photo_preview_request from src.blueprints.telegram_bot._common.yandex_disk import ( @@ -111,24 +112,68 @@ def handle(*args, **kwargs): ]] } + # We will send message without preview, + # because it can take a while to download + # preview file and send it. We will + # send it later if it is available. telegram.send_message(**params) - preview = info.get("preview") + preview_url = info.get("preview") - if preview: - # Yandex requires user OAuth token to get preview - result = make_photo_preview_request(preview, access_token) + if preview_url: + filename = info.get("name", "preview.jpg") + arguments = ( + preview_url, + filename, + access_token, + chat_id + ) - if result["ok"]: - data = result["content"] - filename = info.get("name", "?") + if task_queue.is_enabled: + job_timeout = current_app.config[ + "YANDEX_DISK_WORKER_ELEMENT_INFO_TIMEOUT" + ] - telegram.send_photo( - chat_id=chat_id, - photo=( - filename, - data, - "image/jpeg" - ), - disable_notification=True + task_queue.enqueue( + send_preview, + args=arguments, + description=CommandName.ELEMENT_INFO.value, + job_timeout=job_timeout, + result_ttl=0, + failure_ttl=0 ) + else: + # NOTE: current thread will + # be blocked for a while + send_preview(*arguments) + + +def send_preview( + preview_url, + filename, + user_access_token, + chat_id +): + """ + Downloads preview from Yandex.Disk and sends it to user. + + - requires user Yandex.Disk access token to + download preview file. + """ + result = make_photo_preview_request( + preview_url, + user_access_token + ) + + if result["ok"]: + data = result["content"] + + telegram.send_photo( + chat_id=chat_id, + photo=( + filename, + data, + "image/jpeg" + ), + disable_notification=True + ) diff --git a/src/configs/flask.py b/src/configs/flask.py index 6c69ab2..41fc56c 100644 --- a/src/configs/flask.py +++ b/src/configs/flask.py @@ -175,6 +175,9 @@ class Config: # In seconds. # Applied only if task queue (RQ, for example) is enabled YANDEX_DISK_WORKER_UPLOAD_FAILURE_TTL = 0 + # See `YANDEX_DISK_WORKER_UPLOAD_JOB_TIMEOUT` + # documentation. This value is for `/element_info` worker. + YANDEX_DISK_WORKER_ELEMENT_INFO_TIMEOUT = 5 # Google Analytics GOOGLE_ANALYTICS_UA = os.getenv("GOOGLE_ANALYTICS_UA") From 077fb683e83e5ba5f31e3ea19bb6ae80462f0d62 Mon Sep 17 00:00:00 2001 From: Sergey Kuznetsov Date: Thu, 3 Dec 2020 20:53:46 +0300 Subject: [PATCH 084/103] /space_info: photo will be sended from worker --- .../webhook/commands/space_info.py | 53 +++++++++++++++---- src/configs/flask.py | 3 ++ 2 files changed, 46 insertions(+), 10 deletions(-) diff --git a/src/blueprints/telegram_bot/webhook/commands/space_info.py b/src/blueprints/telegram_bot/webhook/commands/space_info.py index 3258cd1..af1ff56 100644 --- a/src/blueprints/telegram_bot/webhook/commands/space_info.py +++ b/src/blueprints/telegram_bot/webhook/commands/space_info.py @@ -1,11 +1,12 @@ from string import ascii_letters, digits from datetime import datetime, timezone -from flask import g +from flask import g, current_app from plotly.graph_objects import Pie, Figure from plotly.express import colors from plotly.io import to_image +from src.extensions import task_queue from src.api import telegram from src.blueprints._common.utils import get_current_iso_datetime from src.blueprints.telegram_bot._common.yandex_disk import ( @@ -52,17 +53,29 @@ def handle(*args, **kwargs): ) filename = f"{to_filename(current_iso_date)}.jpg" file_caption = f"Yandex.Disk space at {current_utc_date}" - - telegram.send_photo( - chat_id=chat_id, - photo=( - filename, - jpeg_image, - "image/jpeg" - ), - caption=file_caption + arguments = ( + jpeg_image, + filename, + file_caption, + chat_id ) + if task_queue.is_enabled: + job_timeout = current_app.config[ + "YANDEX_DISK_WORKER_SPACE_INFO_TIMEOUT" + ] + + task_queue.enqueue( + send_photo, + args=arguments, + description=CommandName.SPACE_INFO.value, + job_timeout=job_timeout, + result_ttl=0, + failure_ttl=0 + ) + else: + send_photo(*arguments) + def create_space_chart( total_space: int, @@ -187,3 +200,23 @@ def to_filename(value: str) -> str: filename = "".join(x for x in filename if x in valid_chars) return filename + + +def send_photo( + content: bytes, + filename: str, + file_caption: str, + chat_id: str +): + """ + Sends photo to user. + """ + telegram.send_photo( + chat_id=chat_id, + photo=( + filename, + content, + "image/jpeg" + ), + caption=file_caption + ) diff --git a/src/configs/flask.py b/src/configs/flask.py index 41fc56c..57ef89f 100644 --- a/src/configs/flask.py +++ b/src/configs/flask.py @@ -178,6 +178,9 @@ class Config: # See `YANDEX_DISK_WORKER_UPLOAD_JOB_TIMEOUT` # documentation. This value is for `/element_info` worker. YANDEX_DISK_WORKER_ELEMENT_INFO_TIMEOUT = 5 + # See `YANDEX_DISK_WORKER_UPLOAD_JOB_TIMEOUT` + # documentation. This value is for `/space_info` worker. + YANDEX_DISK_WORKER_SPACE_INFO_TIMEOUT = 5 # Google Analytics GOOGLE_ANALYTICS_UA = os.getenv("GOOGLE_ANALYTICS_UA") From d765c7b7fc9a95ac354f7aab336f6e4ada77c1ba Mon Sep 17 00:00:00 2001 From: Sergey Kuznetsov Date: Thu, 3 Dec 2020 20:54:39 +0300 Subject: [PATCH 085/103] Add missing types in element_info.py --- .../telegram_bot/webhook/commands/element_info.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/blueprints/telegram_bot/webhook/commands/element_info.py b/src/blueprints/telegram_bot/webhook/commands/element_info.py index 511148b..6101899 100644 --- a/src/blueprints/telegram_bot/webhook/commands/element_info.py +++ b/src/blueprints/telegram_bot/webhook/commands/element_info.py @@ -149,10 +149,10 @@ def handle(*args, **kwargs): def send_preview( - preview_url, - filename, - user_access_token, - chat_id + preview_url: str, + filename: str, + user_access_token: str, + chat_id: int ): """ Downloads preview from Yandex.Disk and sends it to user. From beda9e0e392f1d0bbdae01807cc64821ef244998 Mon Sep 17 00:00:00 2001 From: Sergey Kuznetsov Date: Thu, 3 Dec 2020 21:04:51 +0300 Subject: [PATCH 086/103] Refactoring of flask config --- .../webhook/commands/element_info.py | 2 +- .../webhook/commands/space_info.py | 2 +- .../telegram_bot/webhook/commands/upload.py | 8 +-- src/configs/flask.py | 61 ++++++++++--------- 4 files changed, 37 insertions(+), 36 deletions(-) diff --git a/src/blueprints/telegram_bot/webhook/commands/element_info.py b/src/blueprints/telegram_bot/webhook/commands/element_info.py index 6101899..987a3af 100644 --- a/src/blueprints/telegram_bot/webhook/commands/element_info.py +++ b/src/blueprints/telegram_bot/webhook/commands/element_info.py @@ -131,7 +131,7 @@ def handle(*args, **kwargs): if task_queue.is_enabled: job_timeout = current_app.config[ - "YANDEX_DISK_WORKER_ELEMENT_INFO_TIMEOUT" + "RUNTIME_ELEMENT_INFO_WORKER_TIMEOUT" ] task_queue.enqueue( diff --git a/src/blueprints/telegram_bot/webhook/commands/space_info.py b/src/blueprints/telegram_bot/webhook/commands/space_info.py index af1ff56..db4f9c6 100644 --- a/src/blueprints/telegram_bot/webhook/commands/space_info.py +++ b/src/blueprints/telegram_bot/webhook/commands/space_info.py @@ -62,7 +62,7 @@ def handle(*args, **kwargs): if task_queue.is_enabled: job_timeout = current_app.config[ - "YANDEX_DISK_WORKER_SPACE_INFO_TIMEOUT" + "RUNTIME_SPACE_INFO_WORKER_TIMEOUT" ] task_queue.enqueue( diff --git a/src/blueprints/telegram_bot/webhook/commands/upload.py b/src/blueprints/telegram_bot/webhook/commands/upload.py index 1cc1235..0111e04 100644 --- a/src/blueprints/telegram_bot/webhook/commands/upload.py +++ b/src/blueprints/telegram_bot/webhook/commands/upload.py @@ -378,16 +378,16 @@ def init_upload(self, *args, **kwargs) -> None: if task_queue.is_enabled: job_timeout = current_app.config[ - "YANDEX_DISK_WORKER_UPLOAD_JOB_TIMEOUT" + "RUNTIME_UPLOAD_WORKER_JOB_TIMEOUT" ] ttl = current_app.config[ - "YANDEX_DISK_WORKER_UPLOAD_TTL" + "RUNTIME_UPLOAD_WORKER_UPLOAD_TTL" ] result_ttl = current_app.config[ - "YANDEX_DISK_WORKER_UPLOAD_RESULT_TTL" + "RUNTIME_UPLOAD_WORKER_RESULT_TTL" ] failure_ttl = current_app.config[ - "YANDEX_DISK_WORKER_UPLOAD_FAILURE_TTL" + "RUNTIME_UPLOAD_WORKER_FAILURE_TTL" ] task_queue.enqueue( diff --git a/src/configs/flask.py b/src/configs/flask.py index 57ef89f..e203794 100644 --- a/src/configs/flask.py +++ b/src/configs/flask.py @@ -90,6 +90,37 @@ class Config: # RQ (background tasks queue) is enabled. # Also depends on `REDIS_URL` RUNTIME_RQ_ENABLED = True + # Maximum runtime of uploading process in `/upload` + # before it’s interrupted. In seconds. + # This value shouldn't be less than + # `YANDEX_DISK_API_CHECK_OPERATION_STATUS_MAX_ATTEMPTS` * + # `YANDEX_DISK_API_CHECK_OPERATION_STATUS_INTERVAL`. + # This timeout should be used only for force quit if + # uploading function start behave incorrectly. + # Use `MAX_ATTEMPTS` and `INTERVAL` for expected quit. + # Applied only if task queue (RQ, for example) is enabled + RUNTIME_UPLOAD_WORKER_JOB_TIMEOUT = 30 + # Maximum queued time of upload function before it's discarded. + # "Queued" means function awaits execution. + # In seconds. `None` for infinite awaiting. + # Applied only if task queue (RQ, for example) is enabled + RUNTIME_UPLOAD_WORKER_UPLOAD_TTL = None + # How long successful result of uploading is kept. + # In seconds. + # Applied only if task queue (RQ, for example) is enabled + RUNTIME_UPLOAD_WORKER_RESULT_TTL = 0 + # How long failed result of uploading is kept. + # "Failed result" means function raises an error, + # not any logical error returns from function. + # In seconds. + # Applied only if task queue (RQ, for example) is enabled + RUNTIME_UPLOAD_WORKER_FAILURE_TTL = 0 + # See `RUNTIME_UPLOAD_WORKER_JOB_TIMEOUT` documentation. + # This value is for `/element_info` worker. + RUNTIME_ELEMENT_INFO_WORKER_TIMEOUT = 5 + # See `RUNTIME_UPLOAD_WORKER_JOB_TIMEOUT` documentation. + # This value is for `/space_info` worker. + RUNTIME_SPACE_INFO_WORKER_TIMEOUT = 5 # Flask DEBUG = False @@ -151,36 +182,6 @@ class Config: # in this folder files will be uploaded by default # if user not specified custom folder. YANDEX_DISK_API_DEFAULT_UPLOAD_FOLDER = "Telegram Bot" - # Maximum runtime of uploading process before it’s interrupted. - # In seconds. This value shouldn't be less than - # `YANDEX_DISK_API_CHECK_OPERATION_STATUS_MAX_ATTEMPTS` * - # `YANDEX_DISK_API_CHECK_OPERATION_STATUS_INTERVAL`. - # This timeout should be used only for force quit if - # uploading function start behave incorrectly. - # Use `MAX_ATTEMPTS` and `INTERVAL` for expected quit. - # Applied only if task queue (RQ, for example) is enabled - YANDEX_DISK_WORKER_UPLOAD_JOB_TIMEOUT = 30 - # Maximum queued time of upload function before it's discarded. - # "Queued" means function awaits execution. - # In seconds. `None` for infinite awaiting. - # Applied only if task queue (RQ, for example) is enabled - YANDEX_DISK_WORKER_UPLOAD_TTL = None - # How long successful result of uploading is kept. - # In seconds. - # Applied only if task queue (RQ, for example) is enabled - YANDEX_DISK_WORKER_UPLOAD_RESULT_TTL = 0 - # How long failed result of uploading is kept. - # "Failed result" means function raises an error, - # not any logical error returns from function. - # In seconds. - # Applied only if task queue (RQ, for example) is enabled - YANDEX_DISK_WORKER_UPLOAD_FAILURE_TTL = 0 - # See `YANDEX_DISK_WORKER_UPLOAD_JOB_TIMEOUT` - # documentation. This value is for `/element_info` worker. - YANDEX_DISK_WORKER_ELEMENT_INFO_TIMEOUT = 5 - # See `YANDEX_DISK_WORKER_UPLOAD_JOB_TIMEOUT` - # documentation. This value is for `/space_info` worker. - YANDEX_DISK_WORKER_SPACE_INFO_TIMEOUT = 5 # Google Analytics GOOGLE_ANALYTICS_UA = os.getenv("GOOGLE_ANALYTICS_UA") From 6cd832050818e8344d2e53d574b4066426f34725 Mon Sep 17 00:00:00 2001 From: Sergey Kuznetsov Date: Fri, 4 Dec 2020 19:55:04 +0300 Subject: [PATCH 087/103] Add worker start in Heroku config --- Procfile | 1 + 1 file changed, 1 insertion(+) diff --git a/Procfile b/Procfile index bbc0b8c..bd96018 100644 --- a/Procfile +++ b/Procfile @@ -1,2 +1,3 @@ release: . ./scripts/env/production.sh; flask db upgrade web: bin/start-nginx bash ./scripts/wsgi/production.sh gunicorn +worker: python worker.py From 36ad940aaca54e2e2e0bc7dcc87b9a50a25142ab Mon Sep 17 00:00:00 2001 From: Sergey Kuznetsov Date: Fri, 4 Dec 2020 20:18:29 +0300 Subject: [PATCH 088/103] Fix Heroku worker command (missing ENV) --- Procfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Procfile b/Procfile index bd96018..bcc0089 100644 --- a/Procfile +++ b/Procfile @@ -1,3 +1,3 @@ release: . ./scripts/env/production.sh; flask db upgrade web: bin/start-nginx bash ./scripts/wsgi/production.sh gunicorn -worker: python worker.py +worker: . ./scripts/env/production.sh; python worker.py From ba4fd636d2c1676fe9462146ef9b9b84053d3ee9 Mon Sep 17 00:00:00 2001 From: Sergey Kuznetsov Date: Fri, 4 Dec 2020 20:49:31 +0300 Subject: [PATCH 089/103] Change YD status check attempts to 5 --- src/configs/flask.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/configs/flask.py b/src/configs/flask.py index e203794..283072d 100644 --- a/src/configs/flask.py +++ b/src/configs/flask.py @@ -173,7 +173,7 @@ class Config: # maximum number of checks of operation status # (for example, if file is downloaded by Yandex.Disk). # It is blocks request until check ending! - YANDEX_DISK_API_CHECK_OPERATION_STATUS_MAX_ATTEMPTS = 8 + YANDEX_DISK_API_CHECK_OPERATION_STATUS_MAX_ATTEMPTS = 5 # interval in seconds between checks of operation status. # It is blocks request until check ending! # For example, if max. attempts is 5 and interval is 2, From 182170a566148d7ae1d807920f379a94dde7ab15 Mon Sep 17 00:00:00 2001 From: Sergey Kuznetsov Date: Sat, 5 Dec 2020 23:06:24 +0300 Subject: [PATCH 090/103] Add ngrok scripts --- scripts/ngrok/run.sh | 3 +++ scripts/ngrok/set_webhook.sh | 8 ++++++++ 2 files changed, 11 insertions(+) create mode 100644 scripts/ngrok/run.sh create mode 100644 scripts/ngrok/set_webhook.sh diff --git a/scripts/ngrok/run.sh b/scripts/ngrok/run.sh new file mode 100644 index 0000000..205d26b --- /dev/null +++ b/scripts/ngrok/run.sh @@ -0,0 +1,3 @@ +#!/bin/bash + +ngrok http 8000 -bind-tls=true diff --git a/scripts/ngrok/set_webhook.sh b/scripts/ngrok/set_webhook.sh new file mode 100644 index 0000000..25cc7e7 --- /dev/null +++ b/scripts/ngrok/set_webhook.sh @@ -0,0 +1,8 @@ +#!/bin/bash + +ngrok_url=`curl http://localhost:4040/api/tunnels/command_line --silent --show-error | jq '.public_url' --raw-output` +webhook_url="$ngrok_url/telegram_bot/webhook" +bot_token=$1 +max_connections=30 + +source ./scripts/telegram/set_webhook.sh $bot_token $webhook_url $max_connections From 66d92851d04e9fb5faf83343b3d70b0dd613565d Mon Sep 17 00:00:00 2001 From: Sergey Kuznetsov Date: Sat, 5 Dec 2020 23:20:56 +0300 Subject: [PATCH 091/103] worker.py: change single quotes to double --- worker.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/worker.py b/worker.py index c5df9d5..bd4642f 100644 --- a/worker.py +++ b/worker.py @@ -16,7 +16,7 @@ def main(): raise Exception("Redis URL is not specified") connection = redis.from_url(redis_url) - listen = ['default'] + listen = ["default"] with Connection(connection): # we should bind Flask app context From b169b616cca0a48691abaeed62185f71f583e841 Mon Sep 17 00:00:00 2001 From: Sergey Kuznetsov Date: Fri, 11 Dec 2020 14:20:04 +0300 Subject: [PATCH 092/103] Add support for forwarding and media messages --- CHANGELOG.md | 2 + .../_common/telegram_interface.py | 7 ++ .../telegram_bot/webhook/commands/upload.py | 107 ++++++++++++++++- .../telegram_bot/webhook/dispatcher.py | 113 +++++++++++++++++- .../telegram_bot/webhook/dispatcher_events.py | 1 + src/configs/flask.py | 8 ++ 6 files changed, 230 insertions(+), 8 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index d4df432..5ecae4c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -19,6 +19,8 @@ - `/commands`: full list of available commands without help message. - `/upload_url`: added `youtube-dl` support. - Handling of files that exceed file size limit in 20 MB. At the moment the bot will asnwer with warning message, not trying to upload that file. [#3](https://github.com/Amaimersion/yandex-disk-telegram-bot/issues/3) +- Now you can forward many attachments and add single command. This command will be applied to all forwarders attachments. +- Now you can send many attachments as a one group and add single command. This command will be applied to all attachments of that group. - Help messages for each upload command will be sended when there are no suitable input data. ### Changed diff --git a/src/blueprints/telegram_bot/_common/telegram_interface.py b/src/blueprints/telegram_bot/_common/telegram_interface.py index 4fce4a1..d36091b 100644 --- a/src/blueprints/telegram_bot/_common/telegram_interface.py +++ b/src/blueprints/telegram_bot/_common/telegram_interface.py @@ -131,6 +131,13 @@ def get_text(self) -> str: "" ) + def get_date(self) -> int: + """ + :returns: + "Date the message was sent in Unix time" + """ + return self.raw_data["date"] + def get_text_without_entities( self, without: List[str] diff --git a/src/blueprints/telegram_bot/webhook/commands/upload.py b/src/blueprints/telegram_bot/webhook/commands/upload.py index 0111e04..bdae3f8 100644 --- a/src/blueprints/telegram_bot/webhook/commands/upload.py +++ b/src/blueprints/telegram_bot/webhook/commands/upload.py @@ -1,12 +1,12 @@ from abc import ABCMeta, abstractmethod -from typing import Union +from typing import Union, Set from collections import deque from urllib.parse import urlparse from flask import g, current_app from src.api import telegram -from src.extensions import task_queue +from src.extensions import task_queue, redis_client from src.blueprints._common.utils import get_current_iso_datetime from src.blueprints.telegram_bot._common import youtube_dl from src.blueprints.telegram_bot._common.telegram_interface import ( @@ -24,6 +24,12 @@ YandexAPIUploadFileError, YandexAPIExceededNumberOfStatusChecksError ) +from src.blueprints.telegram_bot._common.stateful_chat import ( + set_disposable_handler +) +from src.blueprints.telegram_bot.webhook.dispatcher_events import ( + DispatcherEvent +) from ._common.decorators import ( yd_access_token_required, get_db_data @@ -101,10 +107,12 @@ def telegram_action(self) -> str: @abstractmethod def telegram_command(self) -> str: """ + - use `CommandName` enum. + :returns: With what Telegram command this handler - is associated. It is exact command name - (`/upload_photo`, for example). + is associated. It is exact command name, + not enum value. """ pass @@ -130,6 +138,20 @@ def raw_data_type(self) -> type: """ pass + @property + @abstractmethod + def dispatcher_events(self) -> Set[str]: + """ + - use `DispatcherEvent` enum. + + :returns: + With what dispatcher events this handler + is associated. These events will be used + to set disposable handler. It is exact event + names, not enum values. + """ + pass + @abstractmethod def create_help_message(self) -> str: """ @@ -273,6 +295,41 @@ def is_too_big_file(self, file: dict) -> bool: return (size > limit) + def set_disposable_handler( + self, + user_id: int, + chat_id: int + ) -> None: + """ + Sets disposable handler. + + It is means that next message with matched + `self.dispatcher_events` will be forwarded to + `self.telegram_command`. + + - will be used when user didn't sent any + suitable data for handling. + + :param user_id: + Telegram ID of current user. + :param chat_id: + Telegram ID of current chat. + """ + if not redis_client.is_enabled: + return + + expire = current_app.config[ + "RUNTIME_DISPOSABLE_HANDLER_EXPIRE" + ] + + set_disposable_handler( + user_id, + chat_id, + self.telegram_command, + self.dispatcher_events, + expire + ) + @yd_access_token_required @get_db_data def init_upload(self, *args, **kwargs) -> None: @@ -292,6 +349,10 @@ def init_upload(self, *args, **kwargs) -> None: if it is separate process, then this function will be completed fast. """ + user_id = kwargs.get( + "chat_id", + g.db_user.telegram_id + ) chat_id = kwargs.get( "chat_id", g.db_chat.telegram_id @@ -310,6 +371,8 @@ def init_upload(self, *args, **kwargs) -> None: ) if (reason == AbortReason.NO_SUITABLE_DATA): + self.set_disposable_handler(user_id, chat_id) + return self.send_html_message( chat_id, self.create_help_message() @@ -672,6 +735,12 @@ def raw_data_type(self): # dict, not list, because we will select biggest photo return dict + @property + def dispatcher_events(self): + return [ + DispatcherEvent.PHOTO.value + ] + def create_help_message(self): return ( "Send a photos that you want to upload" @@ -727,6 +796,12 @@ def raw_data_key(self): def raw_data_type(self): return dict + @property + def dispatcher_events(self): + return [ + DispatcherEvent.FILE.value + ] + def create_help_message(self): return ( "Send a files that you want to upload" @@ -769,6 +844,12 @@ def raw_data_key(self): def raw_data_type(self): return dict + @property + def dispatcher_events(self): + return [ + DispatcherEvent.AUDIO.value + ] + def create_help_message(self): return ( "Send a music that you want to upload" @@ -825,6 +906,12 @@ def raw_data_key(self): def raw_data_type(self): return dict + @property + def dispatcher_events(self): + return [ + DispatcherEvent.VIDEO.value + ] + def create_help_message(self): return ( "Send a video that you want to upload" @@ -865,6 +952,12 @@ def raw_data_key(self): def raw_data_type(self): return dict + @property + def dispatcher_events(self): + return [ + DispatcherEvent.VOICE.value + ] + def create_help_message(self): return ( "Send a voice message that you want to upload" @@ -900,6 +993,12 @@ def raw_data_key(self): def raw_data_type(self): return str + @property + def dispatcher_events(self): + return [ + DispatcherEvent.URL.value + ] + def create_help_message(self): return ( "Send a direct URL to file that you want to upload" diff --git a/src/blueprints/telegram_bot/webhook/dispatcher.py b/src/blueprints/telegram_bot/webhook/dispatcher.py index ca5d688..b284c0f 100644 --- a/src/blueprints/telegram_bot/webhook/dispatcher.py +++ b/src/blueprints/telegram_bot/webhook/dispatcher.py @@ -6,11 +6,14 @@ from collections import deque import traceback +from flask import current_app from src.extensions import redis_client from src.blueprints.telegram_bot._common.stateful_chat import ( get_disposable_handler, delete_disposable_handler, - get_subscribed_handlers + get_subscribed_handlers, + set_user_chat_data, + get_user_chat_data ) from src.blueprints.telegram_bot._common.telegram_interface import ( Message as TelegramMessage @@ -32,13 +35,20 @@ def intellectual_dispatch( Priority of handlers: 1) if disposable handler exists and message events matches, - then only that handler will be called and after removed + then only that handler will be called and after removed. + That disposable handler will be associated with message date. 2) if subscribed handlers exists, then only ones with events matched to message events will be called. If nothing is matched, then forwarding to № 3 3) attempt to get first `bot_command` entity from message. - If nothing found, then forwarding to № 4 - 4) guessing of command that user assumed based on + It will be treated as direct command. That direct command will be + associated with message date. If nothing found, then forwarding to № 4 + 4) attempt to get command based on message date. + For example, when `/upload_photo` was used for message that was + sent on `1607677734` date, and after that separate message was + sent on same `1607677734` date, then it is the case. + If nothing found, then forwarding to № 5 + 5) guessing of command that user assumed based on content of message Events matching: @@ -62,6 +72,7 @@ def intellectual_dispatch( """ user_id = message.get_user().id chat_id = message.get_chat().id + message_date = message.get_date() stateful_chat_is_enabled = redis_client.is_enabled disposable_handler = None subscribed_handlers = None @@ -111,6 +122,52 @@ def intellectual_dispatch( route_source = RouteSource.DIRECT_COMMAND handler_names.append(command) + should_bind_command_to_date = ( + stateful_chat_is_enabled and + ( + route_source in + ( + RouteSource.DISPOSABLE_HANDLER, + RouteSource.DIRECT_COMMAND + ) + ) + ) + should_get_command_by_date = ( + stateful_chat_is_enabled and + (not handler_names) + ) + + if should_bind_command_to_date: + # we expect only one active command + command = handler_names[0] + + # we need to handle cases when user forwards + # many separate messages (one with direct command and + # others without any command but with some attachments). + # These messages will be sended by Telegram one by one + # (it is means we got separate direct command and + # separate attachments without that any commands). + # We also using `RouteSource.DISPOSABLE_HANDLER` + # because user can start command without any attachments, + # but forward multiple attachments at once or send + # media group (media group messages have same date). + bind_command_to_date( + user_id, + chat_id, + message_date, + command + ) + elif should_get_command_by_date: + command = get_command_by_date( + user_id, + chat_id, + message_date + ) + + if command: + route_source = RouteSource.SAME_DATE_COMMAND + handler_names.append(command) + if not handler_names: route_source = RouteSource.GUESSED_COMMAND handler_names.append(guess_bot_command(message)) @@ -299,3 +356,51 @@ def match_events( `True` - match found, `False` otherwise. """ return any(x in b for x in a) + + +def bind_command_to_date( + user_id: int, + chat_id: int, + date: int, + command: str +) -> None: + """ + Binds command to date. + + You can use it to detect right command for messages + with same date but without specific command. + + - stateful chat should be enabled. + """ + key = f"dispatcher:date:{date}:command" + expire = current_app.config[ + "RUNTIME_SAME_DATE_COMMAND_EXPIRE" + ] + + set_user_chat_data( + user_id, + chat_id, + key, + command, + expire + ) + + +def get_command_by_date( + user_id: int, + chat_id: int, + date: int +) -> Union[str, None]: + """ + - stateful chat should be enabled. + + :returns: + Value that was set using `bind_command_to_date()`. + """ + key = f"dispatcher:date:{date}:command" + + return get_user_chat_data( + user_id, + chat_id, + key + ) diff --git a/src/blueprints/telegram_bot/webhook/dispatcher_events.py b/src/blueprints/telegram_bot/webhook/dispatcher_events.py index 421fe0a..08c656e 100644 --- a/src/blueprints/telegram_bot/webhook/dispatcher_events.py +++ b/src/blueprints/telegram_bot/webhook/dispatcher_events.py @@ -63,5 +63,6 @@ class RouteSource(Enum): """ DISPOSABLE_HANDLER = auto() SUBSCRIBED_HANDLER = auto() + SAME_DATE_COMMAND = auto() DIRECT_COMMAND = auto() GUESSED_COMMAND = auto() diff --git a/src/configs/flask.py b/src/configs/flask.py index 283072d..acf8ca9 100644 --- a/src/configs/flask.py +++ b/src/configs/flask.py @@ -87,6 +87,14 @@ class Config: # this value at all. # Set to 0 to disable expiration RUNTIME_DISPOSABLE_HANDLER_EXPIRE = 60 * 10 + # Dispatcher will bind command to message date. + # How long this data should be stored. In seconds. + # We don't need to memorize it for a long, because + # bot expects messages with exact same date to be sent + # in a short period of time (for example, "Forward" sents + # messages one by one as fast as server process them; + # or user uploads all files as media group within 1 minute). + RUNTIME_SAME_DATE_COMMAND_EXPIRE = 60 * 2 # RQ (background tasks queue) is enabled. # Also depends on `REDIS_URL` RUNTIME_RQ_ENABLED = True From 953355ee6e057ae0159e6820a0717f69b33f2f90 Mon Sep 17 00:00:00 2001 From: Sergey Kuznetsov Date: Fri, 11 Dec 2020 18:33:19 +0300 Subject: [PATCH 093/103] /upload_voice: remove T separator from file name --- CHANGELOG.md | 2 +- src/blueprints/_common/utils.py | 4 ++-- src/blueprints/telegram_bot/webhook/commands/upload.py | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 5ecae4c..33cc18d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -29,7 +29,7 @@ - `/upload`: on success it will return information about uploaded file, not plain status. - `/upload`: increase maxium time of checking of operation status from 10 seconds to 16. - `/upload_url`: result name will not contain parameters, queries and fragments. -- `/upload_voice`: result name will be ISO 8601 date (for example, `2020-11-24T09:57:46+00:00`), not ID from Telegram. +- `/upload_voice`: result name will be ISO 8601 date, but without `T` separator (for example, `2020-11-24 09:57:46+00:00`), not ID from Telegram. ### Fixed diff --git a/src/blueprints/_common/utils.py b/src/blueprints/_common/utils.py index 569a832..c1a832d 100644 --- a/src/blueprints/_common/utils.py +++ b/src/blueprints/_common/utils.py @@ -41,11 +41,11 @@ def get_current_datetime() -> dict: } -def get_current_iso_datetime(timespec="seconds") -> str: +def get_current_iso_datetime(sep="T", timespec="seconds") -> str: """ See https://docs.python.org/3.8/library/datetime.html#datetime.datetime.isoformat # noqa """ - return datetime.now(timezone.utc).isoformat(timespec=timespec) + return datetime.now(timezone.utc).isoformat(sep, timespec) def convert_iso_datetime(date_string: str) -> dict: diff --git a/src/blueprints/telegram_bot/webhook/commands/upload.py b/src/blueprints/telegram_bot/webhook/commands/upload.py index bdae3f8..327b529 100644 --- a/src/blueprints/telegram_bot/webhook/commands/upload.py +++ b/src/blueprints/telegram_bot/webhook/commands/upload.py @@ -965,7 +965,7 @@ def create_help_message(self): ) def create_file_name(self, attachment, file): - return get_current_iso_datetime() + return get_current_iso_datetime(sep=" ") class DirectURLHandler(AttachmentHandler): From 0e1b27c8d1a712efbb00fd26826e66bea5dffdab Mon Sep 17 00:00:00 2001 From: Sergey Kuznetsov Date: Fri, 11 Dec 2020 18:46:17 +0300 Subject: [PATCH 094/103] Refactoring of Flask config --- src/configs/flask.py | 74 ++++++++++++++++++++++++++++++++++---------- 1 file changed, 58 insertions(+), 16 deletions(-) diff --git a/src/configs/flask.py b/src/configs/flask.py index acf8ca9..4f59003 100644 --- a/src/configs/flask.py +++ b/src/configs/flask.py @@ -17,10 +17,10 @@ def load_env(): } file_name = file_names.get(config_name) - if (file_name is None): + if file_name is None: raise Exception( - "Unable to map configuration name " - "and .env.* files" + "Unable to map configuration name and .env.* files. " + "Did you forget to set env variables?" ) load_dotenv(file_name) @@ -31,7 +31,7 @@ def load_env(): class YandexOAuthAPIMethod(Enum): """ - Which method to use for OAuth. + Which method to use for Yandex OAuth API. """ # When user give access, he will be redirected # to the app site, and app will extract code @@ -52,8 +52,8 @@ class YandexOAuthAPIMethod(Enum): class Config: """ Notes: - - don't remove any keys from configuration, because code logic - can depend on this. Instead set disable value (if code logic + - don't remove any keys from this configuration, because code + logic can depend on this. Instead, set "off" value (if code logic supports it); or set empty value and edit code logic to handle such values. - keep in mind that Telegram, Heroku, etc. have request timeout. @@ -63,9 +63,10 @@ class Config: Try to always use background task queue, not block current thread. If you have no opportunity to use background task queue, then change current configuration in order request with blocked thread - cannot take long time to complete. + will be not able to take long time to complete. """ - # Project + # region Project + # name of app that will be used in HTML and so on PROJECT_APP_NAME = "Yandex.Disk Telegram Bot" PROJECT_AUTHOR = "Sergey Kuznetsov" @@ -75,7 +76,10 @@ class Config: PROJECT_URL_FOR_QUESTION = "https://github.com/Amaimersion/yandex-disk-telegram-bot/issues/new?template=question.md" # noqa PROJECT_URL_FOR_BOT = "https://t.me/Ya_Disk_Bot" - # Runtime (interaction of bot with user, behavior of bot) + # endregion + + # region Runtime (interaction of bot with user, behavior of bot, etc.) + # Default value (in seconds) when setted but unused # disposable handler should expire and be removed. # Example: user send `/create_folder` but didn't send @@ -87,6 +91,7 @@ class Config: # this value at all. # Set to 0 to disable expiration RUNTIME_DISPOSABLE_HANDLER_EXPIRE = 60 * 10 + # Dispatcher will bind command to message date. # How long this data should be stored. In seconds. # We don't need to memorize it for a long, because @@ -95,9 +100,11 @@ class Config: # messages one by one as fast as server process them; # or user uploads all files as media group within 1 minute). RUNTIME_SAME_DATE_COMMAND_EXPIRE = 60 * 2 + # RQ (background tasks queue) is enabled. # Also depends on `REDIS_URL` RUNTIME_RQ_ENABLED = True + # Maximum runtime of uploading process in `/upload` # before it’s interrupted. In seconds. # This value shouldn't be less than @@ -108,47 +115,65 @@ class Config: # Use `MAX_ATTEMPTS` and `INTERVAL` for expected quit. # Applied only if task queue (RQ, for example) is enabled RUNTIME_UPLOAD_WORKER_JOB_TIMEOUT = 30 + # Maximum queued time of upload function before it's discarded. # "Queued" means function awaits execution. # In seconds. `None` for infinite awaiting. # Applied only if task queue (RQ, for example) is enabled RUNTIME_UPLOAD_WORKER_UPLOAD_TTL = None + # How long successful result of uploading is kept. # In seconds. # Applied only if task queue (RQ, for example) is enabled RUNTIME_UPLOAD_WORKER_RESULT_TTL = 0 + # How long failed result of uploading is kept. # "Failed result" means function raises an error, # not any logical error returns from function. # In seconds. # Applied only if task queue (RQ, for example) is enabled RUNTIME_UPLOAD_WORKER_FAILURE_TTL = 0 + # See `RUNTIME_UPLOAD_WORKER_JOB_TIMEOUT` documentation. # This value is for `/element_info` worker. RUNTIME_ELEMENT_INFO_WORKER_TIMEOUT = 5 + # See `RUNTIME_UPLOAD_WORKER_JOB_TIMEOUT` documentation. # This value is for `/space_info` worker. RUNTIME_SPACE_INFO_WORKER_TIMEOUT = 5 - # Flask + # endregion + + # region Flask + DEBUG = False TESTING = False SECRET_KEY = os.getenv("FLASK_SECRET_KEY") - # Flask SQLAlchemy + # endregion + + # region Flask SQLAlchemy + SQLALCHEMY_DATABASE_URI = os.getenv( "DATABASE_URL", "sqlite:///temp.sqlite" ) SQLALCHEMY_TRACK_MODIFICATIONS = False - # Redis + # endregion + + # region Redis + REDIS_URL = os.getenv("REDIS_URL") - # Telegram API + # endregion + + # region Telegram API + # stop waiting for a Telegram response # after a given number of seconds TELEGRAM_API_TIMEOUT = 5 + # maximum file size in bytes that bot # can handle by itself. # It is Telegram limit, not bot. @@ -159,41 +184,58 @@ class Config: # to create exactly 20M file TELEGRAM_API_MAX_FILE_SIZE = 20 * 1024 * 1024 - # Yandex OAuth API + # endregion + + # region Yandex OAuth API + # stop waiting for a Yandex response # after a given number of seconds YANDEX_OAUTH_API_TIMEOUT = 15 + # see `YandexOAuthAPIMethod` for more YANDEX_OAUTH_API_METHOD = YandexOAuthAPIMethod.AUTO_CODE_CLIENT + # `insert_token` (controls `INSERT` operation) # will contain N random bytes. Each byte will be # converted to two hex digits YANDEX_OAUTH_API_INSERT_TOKEN_BYTES = 8 + # lifetime of `insert_token` in seconds starting # from date of issue. It is better to find # best combination between `bytes` and `lifetime` YANDEX_OAUTH_API_INSERT_TOKEN_LIFETIME = 60 * 10 - # Yandex.Disk API + # endregion + + # region Yandex.Disk API + # stop waiting for a Yandex response # after a given number of seconds YANDEX_DISK_API_TIMEOUT = 5 + # maximum number of checks of operation status # (for example, if file is downloaded by Yandex.Disk). # It is blocks request until check ending! YANDEX_DISK_API_CHECK_OPERATION_STATUS_MAX_ATTEMPTS = 5 + # interval in seconds between checks of operation status. # It is blocks request until check ending! # For example, if max. attempts is 5 and interval is 2, # then request will be blocked maximum for (5 * 2) seconds. YANDEX_DISK_API_CHECK_OPERATION_STATUS_INTERVAL = 2 + # in this folder files will be uploaded by default # if user not specified custom folder. YANDEX_DISK_API_DEFAULT_UPLOAD_FOLDER = "Telegram Bot" - # Google Analytics + # endregion + + # region Google Analytics + GOOGLE_ANALYTICS_UA = os.getenv("GOOGLE_ANALYTICS_UA") + # endregion + class ProductionConfig(Config): DEBUG = False From d47258cab95ca730346f149e9460bd374ecb4b09 Mon Sep 17 00:00:00 2001 From: Sergey Kuznetsov Date: Fri, 11 Dec 2020 19:07:37 +0300 Subject: [PATCH 095/103] Refactoring of how to check if stateful chat is enabled --- src/blueprints/telegram_bot/_common/stateful_chat.py | 7 ++++++- src/blueprints/telegram_bot/webhook/commands/upload.py | 5 +++-- .../telegram_bot/webhook/commands/yd_auth.py | 6 +++--- src/blueprints/telegram_bot/webhook/dispatcher.py | 10 +++++----- 4 files changed, 17 insertions(+), 11 deletions(-) diff --git a/src/blueprints/telegram_bot/_common/stateful_chat.py b/src/blueprints/telegram_bot/_common/stateful_chat.py index d6351ad..6395493 100644 --- a/src/blueprints/telegram_bot/_common/stateful_chat.py +++ b/src/blueprints/telegram_bot/_common/stateful_chat.py @@ -6,7 +6,8 @@ this module manages the state and dispatcher manages the behavior, dispatcher should be implemented independently. -- requires Redis to be enabled +- requires Redis to be enabled. Use `stateful_chat_is_enabled()` +to check if stateful chat is enabled and can be used. - you shouldn't import things that starts with `_`, because they are intended for internal usage only @@ -96,6 +97,10 @@ def _create_key(*args) -> str: return _SEPARATOR.join(map(str, args)) +def stateful_chat_is_enabled() -> bool: + return redis_client.is_enabled + + # endregion diff --git a/src/blueprints/telegram_bot/webhook/commands/upload.py b/src/blueprints/telegram_bot/webhook/commands/upload.py index 327b529..9858491 100644 --- a/src/blueprints/telegram_bot/webhook/commands/upload.py +++ b/src/blueprints/telegram_bot/webhook/commands/upload.py @@ -6,7 +6,7 @@ from flask import g, current_app from src.api import telegram -from src.extensions import task_queue, redis_client +from src.extensions import task_queue from src.blueprints._common.utils import get_current_iso_datetime from src.blueprints.telegram_bot._common import youtube_dl from src.blueprints.telegram_bot._common.telegram_interface import ( @@ -25,6 +25,7 @@ YandexAPIExceededNumberOfStatusChecksError ) from src.blueprints.telegram_bot._common.stateful_chat import ( + stateful_chat_is_enabled, set_disposable_handler ) from src.blueprints.telegram_bot.webhook.dispatcher_events import ( @@ -315,7 +316,7 @@ def set_disposable_handler( :param chat_id: Telegram ID of current chat. """ - if not redis_client.is_enabled: + if not stateful_chat_is_enabled(): return expire = current_app.config[ diff --git a/src/blueprints/telegram_bot/webhook/commands/yd_auth.py b/src/blueprints/telegram_bot/webhook/commands/yd_auth.py index 067d9b9..1aa59b2 100644 --- a/src/blueprints/telegram_bot/webhook/commands/yd_auth.py +++ b/src/blueprints/telegram_bot/webhook/commands/yd_auth.py @@ -1,7 +1,6 @@ from flask import g, current_app from src.api import telegram -from src.extensions import redis_client from src.configs.flask import YandexOAuthAPIMethod from src.blueprints._common.utils import ( absolute_url_for, @@ -9,6 +8,7 @@ ) from src.blueprints.telegram_bot._common import yandex_oauth from src.blueprints.telegram_bot._common.stateful_chat import ( + stateful_chat_is_enabled, set_disposable_handler, set_user_chat_data, get_user_chat_data, @@ -101,9 +101,9 @@ def run_auto_code_client(db_user, chat_id: int) -> None: def start_console_client(db_user, chat_id: int) -> None: - if not redis_client.is_enabled: + if not stateful_chat_is_enabled(): cancel_command(chat_id) - raise Exception("Redis is required") + raise Exception("Stateful chat is required to be enabled") client = yandex_oauth.YandexOAuthConsoleClient() result = None diff --git a/src/blueprints/telegram_bot/webhook/dispatcher.py b/src/blueprints/telegram_bot/webhook/dispatcher.py index b284c0f..48e81d0 100644 --- a/src/blueprints/telegram_bot/webhook/dispatcher.py +++ b/src/blueprints/telegram_bot/webhook/dispatcher.py @@ -7,8 +7,8 @@ import traceback from flask import current_app -from src.extensions import redis_client from src.blueprints.telegram_bot._common.stateful_chat import ( + stateful_chat_is_enabled, get_disposable_handler, delete_disposable_handler, get_subscribed_handlers, @@ -73,11 +73,11 @@ def intellectual_dispatch( user_id = message.get_user().id chat_id = message.get_chat().id message_date = message.get_date() - stateful_chat_is_enabled = redis_client.is_enabled + is_stateful_chat = stateful_chat_is_enabled() disposable_handler = None subscribed_handlers = None - if stateful_chat_is_enabled: + if is_stateful_chat: disposable_handler = get_disposable_handler(user_id, chat_id) subscribed_handlers = get_subscribed_handlers(user_id, chat_id) @@ -123,7 +123,7 @@ def intellectual_dispatch( handler_names.append(command) should_bind_command_to_date = ( - stateful_chat_is_enabled and + is_stateful_chat and ( route_source in ( @@ -133,7 +133,7 @@ def intellectual_dispatch( ) ) should_get_command_by_date = ( - stateful_chat_is_enabled and + is_stateful_chat and (not handler_names) ) From 0781619dd2178b8fdeff7c85b81126e1d6ea8d30 Mon Sep 17 00:00:00 2001 From: Sergey Kuznetsov Date: Fri, 11 Dec 2020 19:24:00 +0300 Subject: [PATCH 096/103] /element_info: preview will be sended within 10 seconds or never --- .../telegram_bot/webhook/commands/element_info.py | 6 +++++- src/configs/flask.py | 12 +++++++++++- 2 files changed, 16 insertions(+), 2 deletions(-) diff --git a/src/blueprints/telegram_bot/webhook/commands/element_info.py b/src/blueprints/telegram_bot/webhook/commands/element_info.py index 987a3af..aa71d94 100644 --- a/src/blueprints/telegram_bot/webhook/commands/element_info.py +++ b/src/blueprints/telegram_bot/webhook/commands/element_info.py @@ -131,7 +131,10 @@ def handle(*args, **kwargs): if task_queue.is_enabled: job_timeout = current_app.config[ - "RUNTIME_ELEMENT_INFO_WORKER_TIMEOUT" + "RUNTIME_ELEMENT_INFO_WORKER_JOB_TIMEOUT" + ] + ttl = current_app.config[ + "RUNTIME_ELEMENT_INFO_WORKER_TTL" ] task_queue.enqueue( @@ -139,6 +142,7 @@ def handle(*args, **kwargs): args=arguments, description=CommandName.ELEMENT_INFO.value, job_timeout=job_timeout, + ttl=ttl, result_ttl=0, failure_ttl=0 ) diff --git a/src/configs/flask.py b/src/configs/flask.py index 4f59003..e383c63 100644 --- a/src/configs/flask.py +++ b/src/configs/flask.py @@ -136,7 +136,17 @@ class Config: # See `RUNTIME_UPLOAD_WORKER_JOB_TIMEOUT` documentation. # This value is for `/element_info` worker. - RUNTIME_ELEMENT_INFO_WORKER_TIMEOUT = 5 + RUNTIME_ELEMENT_INFO_WORKER_JOB_TIMEOUT = 5 + + # See `RUNTIME_UPLOAD_WORKER_UPLOAD_TTL` documentation. + # This value is for `/element_info` worker. + # Because worker is used only to send preview, + # we will use small TTL, because if all workers + # are busy, most likely it can take a long time + # before preview will be sended. There is no point + # to send preview after 5 minutes - preview should + # be sended either now or never. + RUNTIME_ELEMENT_INFO_WORKER_TTL = 10 # See `RUNTIME_UPLOAD_WORKER_JOB_TIMEOUT` documentation. # This value is for `/space_info` worker. From f023cab4a0c9b59783203ec782eacbb52ddbb7cd Mon Sep 17 00:00:00 2001 From: Sergey Kuznetsov Date: Fri, 11 Dec 2020 21:02:12 +0300 Subject: [PATCH 097/103] /space_info: will send status message at beginning --- src/api/telegram/__init__.py | 3 +- src/api/telegram/methods.py | 9 +++++ .../webhook/commands/space_info.py | 33 +++++++++++++++++-- 3 files changed, 41 insertions(+), 4 deletions(-) diff --git a/src/api/telegram/__init__.py b/src/api/telegram/__init__.py index b88af85..a7507b7 100644 --- a/src/api/telegram/__init__.py +++ b/src/api/telegram/__init__.py @@ -3,7 +3,8 @@ get_file, send_chat_action, edit_message_text, - send_photo + send_photo, + delete_message ) from .requests import ( create_file_download_url diff --git a/src/api/telegram/methods.py b/src/api/telegram/methods.py index f649e1c..58b5c1b 100644 --- a/src/api/telegram/methods.py +++ b/src/api/telegram/methods.py @@ -37,6 +37,15 @@ def edit_message_text(**kwargs): return make_request("editMessageText", kwargs) +def delete_message(**kwargs): + """ + https://core.telegram.org/bots/api#deletemessage + + - see `api/request.py` documentation for more. + """ + return make_request("deleteMessage", kwargs) + + def send_photo(**kwargs): """ https://core.telegram.org/bots/api#sendphoto diff --git a/src/blueprints/telegram_bot/webhook/commands/space_info.py b/src/blueprints/telegram_bot/webhook/commands/space_info.py index db4f9c6..26678ef 100644 --- a/src/blueprints/telegram_bot/webhook/commands/space_info.py +++ b/src/blueprints/telegram_bot/webhook/commands/space_info.py @@ -37,10 +37,25 @@ def handle(*args, **kwargs): access_token = user.yandex_disk_token.get_access_token() disk_info = None + # If all task queue workers are busy, + # it can take a long time before they + # execute `send_photo()` function. + # We will indicate to user that everything + # is fine and result will be sent soon + sended_message = telegram.send_message( + chat_id=chat_id, + text="Generating..." + ) + sended_message_id = sended_message["content"]["message_id"] + try: disk_info = get_disk_info(access_token) except YandexAPIRequestError as error: - cancel_command(chat_id) + cancel_command( + chat_telegram_id=chat_id, + edit_message=sended_message_id + ) + raise error current_utc_date = get_current_utc_datetime() @@ -57,7 +72,8 @@ def handle(*args, **kwargs): jpeg_image, filename, file_caption, - chat_id + chat_id, + sended_message_id ) if task_queue.is_enabled: @@ -206,7 +222,8 @@ def send_photo( content: bytes, filename: str, file_caption: str, - chat_id: str + chat_id: int, + sended_message_id: int ): """ Sends photo to user. @@ -220,3 +237,13 @@ def send_photo( ), caption=file_caption ) + + try: + telegram.delete_message( + chat_id=chat_id, + message_id=sended_message_id + ) + except Exception: + # we can safely ignore if we can't delete + # sended message. Anyway we will send new one + pass From 59389a40e1dc3bbc5adc55179c1e7cc3c2ad639f Mon Sep 17 00:00:00 2001 From: Sergey Kuznetsov Date: Fri, 11 Dec 2020 21:09:39 +0300 Subject: [PATCH 098/103] Add missing checks for if stateful chat is enabled --- .../webhook/commands/create_folder.py | 28 ++++++++++--------- .../webhook/commands/element_info.py | 28 ++++++++++--------- .../telegram_bot/webhook/commands/publish.py | 28 ++++++++++--------- .../webhook/commands/unpublish.py | 28 ++++++++++--------- 4 files changed, 60 insertions(+), 52 deletions(-) diff --git a/src/blueprints/telegram_bot/webhook/commands/create_folder.py b/src/blueprints/telegram_bot/webhook/commands/create_folder.py index 09dbb0c..fafea32 100644 --- a/src/blueprints/telegram_bot/webhook/commands/create_folder.py +++ b/src/blueprints/telegram_bot/webhook/commands/create_folder.py @@ -10,6 +10,7 @@ CommandName ) from src.blueprints.telegram_bot._common.stateful_chat import ( + stateful_chat_is_enabled, set_disposable_handler ) from src.blueprints.telegram_bot.webhook.dispatcher_events import ( @@ -49,19 +50,20 @@ def handle(*args, **kwargs): ) if not folder_name: - set_disposable_handler( - user_id, - chat_id, - CommandName.CREATE_FOLDER.value, - [ - DispatcherEvent.PLAIN_TEXT.value, - DispatcherEvent.BOT_COMMAND.value, - DispatcherEvent.EMAIL.value, - DispatcherEvent.HASHTAG.value, - DispatcherEvent.URL.value - ], - current_app.config["RUNTIME_DISPOSABLE_HANDLER_EXPIRE"] - ) + if stateful_chat_is_enabled(): + set_disposable_handler( + user_id, + chat_id, + CommandName.CREATE_FOLDER.value, + [ + DispatcherEvent.PLAIN_TEXT.value, + DispatcherEvent.BOT_COMMAND.value, + DispatcherEvent.EMAIL.value, + DispatcherEvent.HASHTAG.value, + DispatcherEvent.URL.value + ], + current_app.config["RUNTIME_DISPOSABLE_HANDLER_EXPIRE"] + ) return request_absolute_folder_name(chat_id) diff --git a/src/blueprints/telegram_bot/webhook/commands/element_info.py b/src/blueprints/telegram_bot/webhook/commands/element_info.py index aa71d94..8d95317 100644 --- a/src/blueprints/telegram_bot/webhook/commands/element_info.py +++ b/src/blueprints/telegram_bot/webhook/commands/element_info.py @@ -9,6 +9,7 @@ YandexAPIRequestError ) from src.blueprints.telegram_bot._common.stateful_chat import ( + stateful_chat_is_enabled, set_disposable_handler ) from src.blueprints.telegram_bot._common.command_names import ( @@ -57,19 +58,20 @@ def handle(*args, **kwargs): ) if not path: - set_disposable_handler( - user_id, - chat_id, - CommandName.ELEMENT_INFO.value, - [ - DispatcherEvent.PLAIN_TEXT.value, - DispatcherEvent.BOT_COMMAND.value, - DispatcherEvent.EMAIL.value, - DispatcherEvent.HASHTAG.value, - DispatcherEvent.URL.value - ], - current_app.config["RUNTIME_DISPOSABLE_HANDLER_EXPIRE"] - ) + if stateful_chat_is_enabled(): + set_disposable_handler( + user_id, + chat_id, + CommandName.ELEMENT_INFO.value, + [ + DispatcherEvent.PLAIN_TEXT.value, + DispatcherEvent.BOT_COMMAND.value, + DispatcherEvent.EMAIL.value, + DispatcherEvent.HASHTAG.value, + DispatcherEvent.URL.value + ], + current_app.config["RUNTIME_DISPOSABLE_HANDLER_EXPIRE"] + ) return request_absolute_path(chat_id) diff --git a/src/blueprints/telegram_bot/webhook/commands/publish.py b/src/blueprints/telegram_bot/webhook/commands/publish.py index 9e891f1..a237d3d 100644 --- a/src/blueprints/telegram_bot/webhook/commands/publish.py +++ b/src/blueprints/telegram_bot/webhook/commands/publish.py @@ -9,6 +9,7 @@ YandexAPIRequestError ) from src.blueprints.telegram_bot._common.stateful_chat import ( + stateful_chat_is_enabled, set_disposable_handler ) from src.blueprints.telegram_bot._common.command_names import ( @@ -57,19 +58,20 @@ def handle(*args, **kwargs): ) if not path: - set_disposable_handler( - user_id, - chat_id, - CommandName.PUBLISH.value, - [ - DispatcherEvent.PLAIN_TEXT.value, - DispatcherEvent.BOT_COMMAND.value, - DispatcherEvent.EMAIL.value, - DispatcherEvent.HASHTAG.value, - DispatcherEvent.URL.value - ], - current_app.config["RUNTIME_DISPOSABLE_HANDLER_EXPIRE"] - ) + if stateful_chat_is_enabled(): + set_disposable_handler( + user_id, + chat_id, + CommandName.PUBLISH.value, + [ + DispatcherEvent.PLAIN_TEXT.value, + DispatcherEvent.BOT_COMMAND.value, + DispatcherEvent.EMAIL.value, + DispatcherEvent.HASHTAG.value, + DispatcherEvent.URL.value + ], + current_app.config["RUNTIME_DISPOSABLE_HANDLER_EXPIRE"] + ) return request_absolute_path(chat_id) diff --git a/src/blueprints/telegram_bot/webhook/commands/unpublish.py b/src/blueprints/telegram_bot/webhook/commands/unpublish.py index 0250ba9..fbe9be5 100644 --- a/src/blueprints/telegram_bot/webhook/commands/unpublish.py +++ b/src/blueprints/telegram_bot/webhook/commands/unpublish.py @@ -7,6 +7,7 @@ YandexAPIRequestError ) from src.blueprints.telegram_bot._common.stateful_chat import ( + stateful_chat_is_enabled, set_disposable_handler ) from src.blueprints.telegram_bot._common.command_names import ( @@ -52,19 +53,20 @@ def handle(*args, **kwargs): ) if not path: - set_disposable_handler( - user_id, - chat_id, - CommandName.UNPUBLISH.value, - [ - DispatcherEvent.PLAIN_TEXT.value, - DispatcherEvent.BOT_COMMAND.value, - DispatcherEvent.EMAIL.value, - DispatcherEvent.HASHTAG.value, - DispatcherEvent.URL.value - ], - current_app.config["RUNTIME_DISPOSABLE_HANDLER_EXPIRE"] - ) + if stateful_chat_is_enabled(): + set_disposable_handler( + user_id, + chat_id, + CommandName.UNPUBLISH.value, + [ + DispatcherEvent.PLAIN_TEXT.value, + DispatcherEvent.BOT_COMMAND.value, + DispatcherEvent.EMAIL.value, + DispatcherEvent.HASHTAG.value, + DispatcherEvent.URL.value + ], + current_app.config["RUNTIME_DISPOSABLE_HANDLER_EXPIRE"] + ) return request_absolute_path(chat_id) From 9d9bdbf8e71f7ab49f713603ceaa75c1a31295e4 Mon Sep 17 00:00:00 2001 From: Sergey Kuznetsov Date: Fri, 11 Dec 2020 21:13:42 +0300 Subject: [PATCH 099/103] Upgrade youtube-dl to latest version --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 2996156..17e40b7 100644 --- a/requirements.txt +++ b/requirements.txt @@ -44,5 +44,5 @@ text-unidecode==1.3 toml==0.10.2 urllib3==1.25.11 Werkzeug==1.0.1 -youtube-dl==2020.11.24 +youtube-dl==2020.12.9 -e . From 2554a0f625d704a1ceec96a6e0e6ba5f1368e5d1 Mon Sep 17 00:00:00 2001 From: Sergey Kuznetsov Date: Sat, 12 Dec 2020 20:15:51 +0300 Subject: [PATCH 100/103] Add REDIS_URL in .env.example --- .env.example | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/.env.example b/.env.example index c08b934..bba6afc 100644 --- a/.env.example +++ b/.env.example @@ -12,6 +12,13 @@ FLASK_SECRET_KEY= # PostgreSQL is expected in production, but not required DATABASE_URL=postgresql+psycopg2://:@:/ +# Address of Redis server. Optional. +# If address will be specified, then the app will assume +# that valid instance of Redis server is running, and the app +# will not make any checks (like `PING`). So, make sure you +# pointing to valid Redis instance. +REDIS_URL= + # API token received from @BotFather for Telegram bot TELEGRAM_API_BOT_TOKEN= From bd1a4aeecea2e6dab0b032647e4b02d9bd9bec8c Mon Sep 17 00:00:00 2001 From: Sergey Kuznetsov Date: Mon, 14 Dec 2020 13:04:02 +0300 Subject: [PATCH 101/103] Refactoring of README --- README.md | 235 ++++++++++++++++++++++++++++++++++++++++++++---------- 1 file changed, 195 insertions(+), 40 deletions(-) diff --git a/README.md b/README.md index 66888cf..dd4e1e2 100644 --- a/README.md +++ b/README.md @@ -17,28 +17,41 @@ - [Telegram](#telegram) - [Yandex.Disk](#yandexdisk) - [Local usage](#local-usage) + - [Environment variables](#environment-variables) - [Server](#server) + - [What the app uses](#what-the-app-uses) + - [What you should use](#what-you-should-use) - [Database](#database) + - [What the app uses](#what-the-app-uses-1) + - [What you should use](#what-you-should-use-1) + - [Background tasks](#background-tasks) + - [What the app uses](#what-the-app-uses-2) + - [How to use that](#how-to-use-that) + - [Expose local server](#expose-local-server) + - [What the app uses](#what-the-app-uses-3) + - [What you should use](#what-you-should-use-2) - [Deployment](#deployment) - [Before](#before) - [Heroku](#heroku) + - [First time](#first-time) + - [What's next](#whats-next) - [Contribution](#contribution) - [License](#license) ## Features -- uploading of photos (limit is 20 MB). -- uploading of files (limit is 20 MB). -- uploading of audio (limit is 20 MB). -- uploading of video (limit is 20 MB). -- uploading of voice (limit is 20 MB). -- uploading of files using direct URL. -- uploading of various resources (YouTube, for example) with help of `youtube-dl`. -- uploading for public access. -- publishing and unpublishing of files or folders. -- creating of folders. -- getting of information about file, folder or disk. +- uploading of photos (limit is 20 MB), +- uploading of files (limit is 20 MB), +- uploading of audio (limit is 20 MB), +- uploading of video (limit is 20 MB), +- uploading of voice (limit is 20 MB), +- uploading of files using direct URL, +- uploading of various resources (YouTube, for example) with help of `youtube-dl`, +- uploading for public access, +- publishing and unpublishing of files or folders, +- creating of folders, +- getting information about a file, folder or disk. ## Requirements @@ -50,9 +63,11 @@ - [curl](https://curl.haxx.se/) (optional) - [nginx 1.16+](https://nginx.org/) (optional) - [postgreSQL 10+](https://www.postgresql.org/) (optional) +- [redis 6.0+](https://redis.io/) (optional) - [heroku 7.39+](https://www.heroku.com/) (optional) +- [ngrok 2.3+](https://ngrok.com/) (optional) -It is expected that all of the above software is available as a global variable: `python3`, `python3 -m pip`, `python3 -m venv`, `git`, `curl`, `nginx`, `psql`, `heroku`. See [this](https://github.com/pypa/pip/issues/5599#issuecomment-597042338) why you should use such syntax: `python3 -m `. +It is expected that all of the above software is available as a global variables: `python3`, `python3 -m pip`, `python3 -m venv`, `git`, `curl`, `nginx`, `psql`, `heroku`, `ngrok`. See [this](https://github.com/pypa/pip/issues/5599#issuecomment-597042338) why you should use such syntax: `python3 -m `. All subsequent instructions is for Unix-like systems, primarily for Linux. You may need to make some changes on your own if you work on non-Linux operating system. @@ -72,15 +87,20 @@ cd yandex-disk-telegram-bot ```shell python3 -m venv venv +``` + +And activate it: + +```shell source ./venv/bin/activate ``` -Run `deactivate` when you end in order to exit from virtual environment. +Run `deactivate` when you end in order to exit from virtual environment or just close terminal window. After that step we will use `python` instead of `python3` and `pip` instead of `python3 -m pip`. If for some reason you don't want create virtual environment, then: -- use `python3` and `python3 -m pip` -- edit executable paths in `.vscode/settings.json` -- edit names in `./scripts` files +- use `python3` and `python3 -m pip`, +- edit executable paths in `.vscode/settings.json`, +- edit names in `./scripts` files. You probably need to upgrade `pip`, because [you may have](https://github.com/pypa/pip/issues/5221) an old version (9.0.1) instead of new one. Run `pip install --upgrade pip`. @@ -90,22 +110,39 @@ You probably need to upgrade `pip`, because [you may have](https://github.com/py ./scripts/requirements/install.sh ``` -4. If you want to use database locally, then make DB upgrade: +4. Set environment variables. + +```shell +source ./scripts/env/.sh +``` + +Where `` is either `production`, `development` or `testing`. Start with `development` if you don't know what to use. + +These environment variables are required to create right app configuration, which is unique for specific environemts. For example, you can have different `DATABASE_URL` variable for `production` and `development`. + +You need these environment variables every time when you implicitly interact with app configuration. For example, DB upgrade and background workers implicitly create app and use it configuration. + +If you forgot to set environment variables but they are required to configure the app, you will get following error: `Unable to map configuration name and .env.* files`. -```shel -source ./scripts/env/development.sh +5. If you want to use database locally, then make DB upgrade: + +```shell flask db upgrade ``` -5. Run this to see more actions: +6. Run this to see more available actions: + +```shell +python manage.py --help +``` -`python manage.py --help` +7. Every time when you open this project again, don't forget to activate virtual environment. -That's all you need for development. If you want create production-ready server, then: +That's all you need to set up this project. If you want to set up fully working app, then: 1. Perform [integration with external API's](#integration-with-external-apis). -2. See [Local usage](#local-usage) or [Deployment](#deployment). +2. See [Local usage](#local-usage) for `development` and [Deployment](#deployment) for `production`. ## Integration with external API's @@ -125,31 +162,37 @@ Russian users may need a proxy: ./scripts/telegram/set_webhook.sh "--proxy " ``` -For parameter `MAX_CONNECTIONS` it is recommended to use maxium number of simultaneous connections to the selected database. For example, "Heroku Postgres" extension at "Hobby Dev" plan have connection limit of 20. So, you should use `20` as value for `MAX_CONNECTIONS` parameter in order to avoid possible `Too many connections` error. +For parameter `MAX_CONNECTIONS` it is recommended to use maxium number of simultaneous connections to the selected database. For example, "Heroku Postgres" extension at "Hobby Dev" plan have connection limit of 20. So, you should use `20` as a value for `MAX_CONNECTIONS` parameter in order to avoid possible `Too many connections` error. From Telegram documentation: > If you'd like to make sure that the Webhook request comes from Telegram, we recommend using a secret path in the URL, e.g. `https://www.example.com/`. Since nobody else knows your bot‘s token, you can be pretty sure it’s us. -So, instead of `/telegram_bot/webhook` you can use something like this: `/telegram_bot/webhook_fd1k3Bfa01WQl5S`. +So, instead of `/telegram_bot/webhook` you can use something like this: `/telegram_bot/webhook_fd1k3Bfa01WQl5S`. Don't forget to edit route in `./src/blueprints/telegram_bot/webhook/views.py` if you decide to use it. ### Yandex.Disk -1. Register your app in [Yandex](https://yandex.ru/dev/oauth/). Most likely it will take a while for Yandex moderators to check your app. +1. Register your app in [Yandex](https://yandex.ru/dev/oauth/). Sometimes it can take a while for Yandex moderators to check your app. 2. Get your app ID and password at special Yandex page for your app. -3. At special Yandex page for your app find "Callback URI" setting and add this URI: `https:///telegram_bot/yandex_disk_authorization`. +3. At special Yandex page for your app find "Callback URI" setting and add this URI: `https:///telegram_bot/yandex_disk_authorization`. It is required if you want to use `AUTO_CODE_CLIENT` Yandex.OAuth method, which is configured by default. ## Local usage +### Environment variables + +In a root directory create `.env.development` file and fill it based on `.env.example` file. + ### Server -This WSGI App uses `gunicorn` as WSGI HTTP Server and `nginx` as HTTP Reverse Proxy Server. For development purposes `flask` built-in WSGI HTTP Server is used. +#### What the app uses + +This WSGI App uses `gunicorn` as WSGI HTTP Server and `nginx` as HTTP Reverse Proxy Server. For development purposes only `flask` built-in WSGI HTTP Server is used. `flask` uses `http://localhost:8000`, `gunicorn` uses `unix:/tmp/nginx-gunicorn.socket`, `nginx` uses `http://localhost:80`. Make sure these addresses is free for usage, or change specific server configuration. -`nginx` will not start until `gunicorn` creates `/tmp/gunicorn-ready` file. Make sure you have access to create this file. +`nginx` will not start until `gunicorn` creates `/tmp/gunicorn-ready` file. Make sure you have access rights to create this file. Open terminal and move in project root. Run `./scripts/wsgi/.sh ` where `` is either `prodction`, `development` or `testing`, and `` is either `flask`, `gunicorn` or `nginx`. Example: `./scripts/wsgi/production.sh gunicorn`. @@ -157,13 +200,101 @@ Usually you will want to run both `gunicorn` and `nginx`. To do so run scripts i Run `./scripts/server/stop_nginx.sh` in order to stop nginx. -nginx uses simple configuration from `./src/configs/nginx.conf`. You can ignore this and use any configuration for nginx that is appropriate to you. However, it is recommended to use exact configurations as in app for both `flask` and `gunicorn`. If you think these configurations is not right, then make PR instead. +nginx uses simple configuration from `./src/configs/nginx.conf`. You can ignore this and use any configuration for nginx that is appropriate to you. However, it is recommended to use exact configurations as in app for both `flask` and `gunicorn`. If you think these configurations is not good, then make PR instead. + +#### What you should use + +For active development it will be better to use only `flask` WSGI HTTP Server. + +```shell +source ./scripts/wsgi/development.sh flask +``` + +That command will automatically set environment variables and run `flask` WSGI server. And your app will be fully ready for incoming requests. + +If you want to test more stable and reliable configuration which will be used in production, then run these commands in separate terminal window. + +```shell +source ./scripts/wsgi/development.sh gunicorn +``` + +```shell +source ./scripts/server/stop_nginx.sh +source ./scripts/wsgi/development.sh nginx +``` ### Database +#### What the app uses + In both development and testing environments `SQLite` is used. For production `PostgreSQL` is recommended, but you can use any of [supported databases](https://docs.sqlalchemy.org/en/13/core/engines.html#supported-databases). App already configured for both `SQLite` and `PostgreSQL`, for another database you may have to install additional Python packages. -Development and testing databases will be located at `src/development.sqlite` and `src/testing.sqlite` respectively. +By default both development and testing databases will be located at `src/temp.sqlite`. If you want to use different name for DB, then specify value for `DATABASE_URL` in `.env.development` and `.env.testing` files. + +`Redis` database is supported and expected, but not required. However, it is highly recommended to enable it, because many useful features of the app depends on `Redis` functionality and they will be disabled in case if `Redis` is not configured using `REDIS_URL`. + +#### What you should use + +Usually it will be better to manually specify DB name in `.env.development` and specify `REDIS_URL`. Try to always use `Redis`, including development environment. If you decide not to enable `Redis`, it is fine and you still can use the app with basic functionality which don't depends on `Redis`. + +### Background tasks + +#### What the app uses + +`RQ` is used as a task queue. `Redis` is required. + +Examples of jobs that will be enqueued: monitoring of uploading status and uploading of files. + +The app is not ready to support other task queues. You may need to make changes on your own if you decide to use another task queue. + +#### How to use that + +It is highly recommended that you run at least one worker. + +1. Make sure `REDIS_URL` is specified. +2. Open separate terminal window. +3. Activate `venv` and set environment variables. +4. Run: `python worker.py` + +These steps will run one worker instance. Count of workers depends on your expected server load. For `development` environment recommend count is 2. + +Note that `RQ` will not automatically reload your running workers when source code of any job function's changes. So, you should restart workers manually. + +### Expose local server + +#### What the app uses + +`ngrok` is used to expose local server. It is free and suitable for development server. + +#### What you should use + +You can use whatever you want. But if you decide to use `ngrok`, the app provides fews utils to make it easier. + +Before: +- requests will be routed to `/telegram_bot/webhook`, so, make sure you didn't change this route, +- you also should have [jq](https://stedolan.github.io/jq/) on your system. + +Then: + +1. Run `flask` server: + +```shell +source ./scripts/wsgi/development.sh flask +``` + +2. In separate terminal window run `ngrok`: + +```shell +source ./scripts/ngrok/run.sh +``` + +3. In separate terminal window set a webhook: + +```shell +source ./scripts/ngrok/set_webhook.sh +``` + +Where `` is your Telegram bot API token for specific environment (you can have different bots for different environments). ## Deployment @@ -172,10 +303,14 @@ Regardless of any platform you choose for hosting, it is recommended to manually ### Before -It is recommended to run linters with `./scripts/linters/all.sh` before deployment and resolve all errors and warnings. +It is recommended to run linters with `source ./scripts/linters/all.sh` before deployment and resolve all errors and warnings. ### Heroku +It is a way to host this app for free. And that will be more than enough until you have hundreds of active users. + +#### First time + 1. If you don't have [Heroku](https://heroku.com/) installed, then it is a time to do that. 2. If you don't have Heroku remote, then add it: @@ -205,7 +340,13 @@ heroku addons:create heroku-postgresql:hobby-dev Later you can view the DB content by using `heroku pg:psql`. -5. Set required environment variables: +5. We need Heroku `Redis` addon in order to use that database. + +``` +heroku addons:create heroku-redis:hobby-dev +``` + +6. Set required environment variables: ``` heroku config:set SERVER_NAME= @@ -220,19 +361,25 @@ heroku config:set GUNICORN_WORKERS= heroku config:set GUNICORN_WORKER_CONNECTIONS= ``` -6. Switch to new branch special for Heroku (don't ever push it!): +7. Switch to a new branch that is special for Heroku (don't ever push it!): ```git git checkout -b heroku ``` -7. Make sure `.env.production` file is created and filled. Remove it from `.gitignore`. Don't forget: don't ever push it anywhere but Heroku. +If that branch already created, then just type: + +``` +git checkout heroku +``` + +8. Make sure `.env.production` file is created and filled. Remove it from `.gitignore`. Don't forget: don't ever push it anywhere but Heroku. -8. Add changes for pushing to Heroku: +9. Add changes for pushing to Heroku: -- if you edited files on heroku branch: +- if you edited files on `heroku` branch: ```git -git add . +git add git commit -m ``` @@ -241,13 +388,21 @@ git commit -m git merge -m ``` -9. Upload files to Heroku: +10. Upload files to Heroku: ```git git push heroku heroku:master ``` -You should do № 8 and № 9 every time you want to push changes. +11. Set number of workers for background tasks. On free plan you cannot use more than 1 worker. + +``` +heroku scale worker=1 +``` + +#### What's next + +You should do steps № 7, 9 and 10 every time when you want to push changes. ## Contribution From 2fe06eca14aff12dc169c6dad5260d3c6aad90f3 Mon Sep 17 00:00:00 2001 From: Sergey Kuznetsov Date: Mon, 14 Dec 2020 13:06:01 +0300 Subject: [PATCH 102/103] Upgrade youtube-dl to latest version --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 17e40b7..2b4afa8 100644 --- a/requirements.txt +++ b/requirements.txt @@ -44,5 +44,5 @@ text-unidecode==1.3 toml==0.10.2 urllib3==1.25.11 Werkzeug==1.0.1 -youtube-dl==2020.12.9 +youtube-dl==2020.12.14 -e . From 3bb441839194763746b76719a84da646927daee0 Mon Sep 17 00:00:00 2001 From: Sergey Kuznetsov Date: Mon, 14 Dec 2020 13:13:12 +0300 Subject: [PATCH 103/103] Add release date for 1.2.0 --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 33cc18d..3e002cc 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,4 +1,4 @@ -# 1.2.0 +# 1.2.0 (December 14, 2020) ## Telegram Bot