diff --git a/.env.example b/.env.example index c08b934..bba6afc 100644 --- a/.env.example +++ b/.env.example @@ -12,6 +12,13 @@ FLASK_SECRET_KEY= # PostgreSQL is expected in production, but not required DATABASE_URL=postgresql+psycopg2://:@:/ +# Address of Redis server. Optional. +# If address will be specified, then the app will assume +# that valid instance of Redis server is running, and the app +# will not make any checks (like `PING`). So, make sure you +# pointing to valid Redis instance. +REDIS_URL= + # API token received from @BotFather for Telegram bot TELEGRAM_API_BOT_TOKEN= diff --git a/.flake8 b/.flake8 index 7f41022..e03c87d 100644 --- a/.flake8 +++ b/.flake8 @@ -1,5 +1,8 @@ [flake8] -ignore = F401, W504 +ignore = + F401, + W504, + E261 exclude = # Folders __pycache__ diff --git a/.gitignore b/.gitignore index 82a2cff..84ffc37 100644 --- a/.gitignore +++ b/.gitignore @@ -13,4 +13,5 @@ venv # Secrets instance -.env +.env* +!.env.example diff --git a/.vscode/launch.json b/.vscode/launch.json new file mode 100644 index 0000000..c048bea --- /dev/null +++ b/.vscode/launch.json @@ -0,0 +1,26 @@ +{ + "version": "0.2.0", + "configurations": [ + { + "name": "Python: Flask", + "type": "python", + "request": "launch", + "module": "flask", + "env": { + "FLASK_APP": "wsgi.py", + "FLASK_ENV": "development", + "FLASK_DEBUG": "0", + "CONFIG_NAME": "development" + }, + "args": [ + "run", + "--no-debugger", + "--no-reload", + "--port", + "8000" + ], + "jinja": true, + "console": "integratedTerminal" + } + ] +} diff --git a/.vscode/settings.json b/.vscode/settings.json index 51dbe5c..cde25bb 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -1,9 +1,9 @@ { - "python.pythonPath": "./venv/bin/python", + "python.pythonPath": "venv/bin/python", "python.autoComplete.extraPaths": [ "./src/*" ], - "cornflakes.linter.executablePath": "./venv/bin/flake8", + "cornflakes.linter.executablePath": "venv/bin/flake8", "files.exclude": { "venv": true, "**/__pycache__": true, diff --git a/CHANGELOG.md b/CHANGELOG.md index de1eb89..3e002cc 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,72 @@ +# 1.2.0 (December 14, 2020) + +## Telegram Bot + +### Improved + +- Text of some bot responses. +- Formatting of help message. +- `/upload`: multiple items will be handled at a same time, not one by one. + +### Added + +- `/public_upload_photo`, `/public_upload_file`, `/public_upload_audio`, `/public_upload_video`, `/public_upload_voice`, `/public_upload_url`: uploading of files and then publishing them. +- `/publish`: publishing of files or folders. +- `/unpublish`: unpublishing of files or folders. +- `/space_info`: getting of information about remaining Yandex.Disk space. +- `/element_info`: getting of information about file or folder. +- `/disk_info`: getting of information about Yandex.Disk. +- `/commands`: full list of available commands without help message. +- `/upload_url`: added `youtube-dl` support. +- Handling of files that exceed file size limit in 20 MB. At the moment the bot will asnwer with warning message, not trying to upload that file. [#3](https://github.com/Amaimersion/yandex-disk-telegram-bot/issues/3) +- Now you can forward many attachments and add single command. This command will be applied to all forwarders attachments. +- Now you can send many attachments as a one group and add single command. This command will be applied to all attachments of that group. +- Help messages for each upload command will be sended when there are no suitable input data. + +### Changed + +- `/create_folder`: now it will wait for folder name if you send empty command, not deny operation. +- `/upload`: on success it will return information about uploaded file, not plain status. +- `/upload`: increase maxium time of checking of operation status from 10 seconds to 16. +- `/upload_url`: result name will not contain parameters, queries and fragments. +- `/upload_voice`: result name will be ISO 8601 date, but without `T` separator (for example, `2020-11-24 09:57:46+00:00`), not ID from Telegram. + +### Fixed + +- `/create_folder`: fixed a bug when bot could remove `/create_folder` occurrences from folder name. +- `/create_folder`: fixed a bug when bot don't send any responses on invalid folder name. +- Wrong information in help message for `/upload_video`. +- A bug when paths with `:` in name (for example, `Telegram Bot/folder:test`) led to `DiskPathFormatError` from Yandex. + +## Project + +### Improved + +- Upgrade `python` to 3.8.5. +- All requirements upgraded to latest version. +- Big refactoring. + +### Added + +- Stateful chat support. Now bot can store custom user data (in different namespaces: user, chat, user in chat); determine Telegram message types; register single use handler (call once for message) with optional timeout for types of message; subscribe handlers with optional timeout for types of messages. +- [Console Client](https://yandex.ru/dev/oauth/doc/dg/reference/console-client.html) Yandex.OAuth method. By default it is disabled, and default one is [Auto Code Client](https://yandex.ru/dev/oauth/doc/dg/reference/auto-code-client.html/). +- RQ (job queue). It requires Redis to be enabled, and as Redis it is also optional. However, it is highly recommended to use it. +- Support for different env-type files (based on current environment). Initially it was only for production. +- Web Site: 302 (redirect to Telegram) will be returned instead of 404 (not found page), but only in production mode. +- Debug configuration for VSCode. +- DB: add indexes for frequent using columns. + +### Changed + +- Redirect to favicon will be handled by nginx. +- Biggest photo (from single photo file) will be selected based on total pixels count, not based on height. + +### Fixed + +- A bug when new user (didn't use any command before) used `/revoke_access` command and it led to request crash (500). +- Situation: Telegram send an update, the server sent back a 500; Telegram will send same update again and again until it get 200 from a server, but server always returns 500. Such sitations can occur, for example, when user initiated a command and blocked the bot - bot can't send message to user in this case (it gets 403 from Telegram API, so, server raises error because it is an unexpected error and should be logged). Now it is fixed and the bot always send back 200, even for such error situations. + + # 1.1.0 (May 9, 2020) ## Telegram Bot diff --git a/Procfile b/Procfile index bbc0b8c..bcc0089 100644 --- a/Procfile +++ b/Procfile @@ -1,2 +1,3 @@ release: . ./scripts/env/production.sh; flask db upgrade web: bin/start-nginx bash ./scripts/wsgi/production.sh gunicorn +worker: . ./scripts/env/production.sh; python worker.py diff --git a/README.md b/README.md index d89f9b1..dd4e1e2 100644 --- a/README.md +++ b/README.md @@ -17,24 +17,41 @@ - [Telegram](#telegram) - [Yandex.Disk](#yandexdisk) - [Local usage](#local-usage) + - [Environment variables](#environment-variables) - [Server](#server) + - [What the app uses](#what-the-app-uses) + - [What you should use](#what-you-should-use) - [Database](#database) + - [What the app uses](#what-the-app-uses-1) + - [What you should use](#what-you-should-use-1) + - [Background tasks](#background-tasks) + - [What the app uses](#what-the-app-uses-2) + - [How to use that](#how-to-use-that) + - [Expose local server](#expose-local-server) + - [What the app uses](#what-the-app-uses-3) + - [What you should use](#what-you-should-use-2) - [Deployment](#deployment) - [Before](#before) - [Heroku](#heroku) + - [First time](#first-time) + - [What's next](#whats-next) - [Contribution](#contribution) - [License](#license) ## Features -- uploading of photos. -- uploading of files. -- uploading of files using direct URL. -- uploading of audio. -- uploading of video. -- uploading of voice. -- creating of folders. +- uploading of photos (limit is 20 MB), +- uploading of files (limit is 20 MB), +- uploading of audio (limit is 20 MB), +- uploading of video (limit is 20 MB), +- uploading of voice (limit is 20 MB), +- uploading of files using direct URL, +- uploading of various resources (YouTube, for example) with help of `youtube-dl`, +- uploading for public access, +- publishing and unpublishing of files or folders, +- creating of folders, +- getting information about a file, folder or disk. ## Requirements @@ -46,9 +63,11 @@ - [curl](https://curl.haxx.se/) (optional) - [nginx 1.16+](https://nginx.org/) (optional) - [postgreSQL 10+](https://www.postgresql.org/) (optional) +- [redis 6.0+](https://redis.io/) (optional) - [heroku 7.39+](https://www.heroku.com/) (optional) +- [ngrok 2.3+](https://ngrok.com/) (optional) -It is expected that all of the above software is available as a global variable: `python3`, `python3 -m pip`, `python3 -m venv`, `git`, `curl`, `nginx`, `psql`, `heroku`. See [this](https://github.com/pypa/pip/issues/5599#issuecomment-597042338) why you should use such syntax: `python3 -m `. +It is expected that all of the above software is available as a global variables: `python3`, `python3 -m pip`, `python3 -m venv`, `git`, `curl`, `nginx`, `psql`, `heroku`, `ngrok`. See [this](https://github.com/pypa/pip/issues/5599#issuecomment-597042338) why you should use such syntax: `python3 -m `. All subsequent instructions is for Unix-like systems, primarily for Linux. You may need to make some changes on your own if you work on non-Linux operating system. @@ -68,15 +87,20 @@ cd yandex-disk-telegram-bot ```shell python3 -m venv venv +``` + +And activate it: + +```shell source ./venv/bin/activate ``` -Run `deactivate` when you end in order to exit from virtual environment. +Run `deactivate` when you end in order to exit from virtual environment or just close terminal window. After that step we will use `python` instead of `python3` and `pip` instead of `python3 -m pip`. If for some reason you don't want create virtual environment, then: -- use `python3` and `python3 -m pip` -- edit executable paths in `.vscode/settings.json` -- edit names in `./scripts` files +- use `python3` and `python3 -m pip`, +- edit executable paths in `.vscode/settings.json`, +- edit names in `./scripts` files. You probably need to upgrade `pip`, because [you may have](https://github.com/pypa/pip/issues/5221) an old version (9.0.1) instead of new one. Run `pip install --upgrade pip`. @@ -86,22 +110,39 @@ You probably need to upgrade `pip`, because [you may have](https://github.com/py ./scripts/requirements/install.sh ``` -4. If you want to use database locally, then make DB upgrade: +4. Set environment variables. + +```shell +source ./scripts/env/.sh +``` + +Where `` is either `production`, `development` or `testing`. Start with `development` if you don't know what to use. + +These environment variables are required to create right app configuration, which is unique for specific environemts. For example, you can have different `DATABASE_URL` variable for `production` and `development`. + +You need these environment variables every time when you implicitly interact with app configuration. For example, DB upgrade and background workers implicitly create app and use it configuration. + +If you forgot to set environment variables but they are required to configure the app, you will get following error: `Unable to map configuration name and .env.* files`. -```shel -source ./scripts/env/development.sh +5. If you want to use database locally, then make DB upgrade: + +```shell flask db upgrade ``` -5. Run this to see more actions: +6. Run this to see more available actions: + +```shell +python manage.py --help +``` -`python manage.py --help` +7. Every time when you open this project again, don't forget to activate virtual environment. -That's all you need for development. If you want create production-ready server, then: +That's all you need to set up this project. If you want to set up fully working app, then: 1. Perform [integration with external API's](#integration-with-external-apis). -2. See [Local usage](#local-usage) or [Deployment](#deployment). +2. See [Local usage](#local-usage) for `development` and [Deployment](#deployment) for `production`. ## Integration with external API's @@ -121,31 +162,37 @@ Russian users may need a proxy: ./scripts/telegram/set_webhook.sh "--proxy " ``` -For parameter `MAX_CONNECTIONS` it is recommended to use maxium number of simultaneous connections to the selected database. For example, "Heroku Postgres" extension at "Hobby Dev" plan have connection limit of 20. So, you should use `20` as value for `MAX_CONNECTIONS` parameter in order to avoid possible `Too many connections` error. +For parameter `MAX_CONNECTIONS` it is recommended to use maxium number of simultaneous connections to the selected database. For example, "Heroku Postgres" extension at "Hobby Dev" plan have connection limit of 20. So, you should use `20` as a value for `MAX_CONNECTIONS` parameter in order to avoid possible `Too many connections` error. From Telegram documentation: > If you'd like to make sure that the Webhook request comes from Telegram, we recommend using a secret path in the URL, e.g. `https://www.example.com/`. Since nobody else knows your bot‘s token, you can be pretty sure it’s us. -So, instead of `/telegram_bot/webhook` you can use something like this: `/telegram_bot/webhook_fd1k3Bfa01WQl5S`. +So, instead of `/telegram_bot/webhook` you can use something like this: `/telegram_bot/webhook_fd1k3Bfa01WQl5S`. Don't forget to edit route in `./src/blueprints/telegram_bot/webhook/views.py` if you decide to use it. ### Yandex.Disk -1. Register your app in [Yandex](https://yandex.ru/dev/oauth/). Most likely it will take a while for Yandex moderators to check your app. +1. Register your app in [Yandex](https://yandex.ru/dev/oauth/). Sometimes it can take a while for Yandex moderators to check your app. 2. Get your app ID and password at special Yandex page for your app. -3. At special Yandex page for your app find "Callback URI" setting and add this URI: `https:///telegram_bot/yandex_disk_authorization`. +3. At special Yandex page for your app find "Callback URI" setting and add this URI: `https:///telegram_bot/yandex_disk_authorization`. It is required if you want to use `AUTO_CODE_CLIENT` Yandex.OAuth method, which is configured by default. ## Local usage +### Environment variables + +In a root directory create `.env.development` file and fill it based on `.env.example` file. + ### Server -This WSGI App uses `gunicorn` as WSGI HTTP Server and `nginx` as HTTP Reverse Proxy Server. For development purposes `flask` built-in WSGI HTTP Server is used. +#### What the app uses + +This WSGI App uses `gunicorn` as WSGI HTTP Server and `nginx` as HTTP Reverse Proxy Server. For development purposes only `flask` built-in WSGI HTTP Server is used. `flask` uses `http://localhost:8000`, `gunicorn` uses `unix:/tmp/nginx-gunicorn.socket`, `nginx` uses `http://localhost:80`. Make sure these addresses is free for usage, or change specific server configuration. -`nginx` will not start until `gunicorn` creates `/tmp/gunicorn-ready` file. Make sure you have access to create this file. +`nginx` will not start until `gunicorn` creates `/tmp/gunicorn-ready` file. Make sure you have access rights to create this file. Open terminal and move in project root. Run `./scripts/wsgi/.sh ` where `` is either `prodction`, `development` or `testing`, and `` is either `flask`, `gunicorn` or `nginx`. Example: `./scripts/wsgi/production.sh gunicorn`. @@ -153,13 +200,101 @@ Usually you will want to run both `gunicorn` and `nginx`. To do so run scripts i Run `./scripts/server/stop_nginx.sh` in order to stop nginx. -nginx uses simple configuration from `./src/configs/nginx.conf`. You can ignore this and use any configuration for nginx that is appropriate to you. However, it is recommended to use exact configurations as in app for both `flask` and `gunicorn`. If you think these configurations is not right, then make PR instead. +nginx uses simple configuration from `./src/configs/nginx.conf`. You can ignore this and use any configuration for nginx that is appropriate to you. However, it is recommended to use exact configurations as in app for both `flask` and `gunicorn`. If you think these configurations is not good, then make PR instead. + +#### What you should use + +For active development it will be better to use only `flask` WSGI HTTP Server. + +```shell +source ./scripts/wsgi/development.sh flask +``` + +That command will automatically set environment variables and run `flask` WSGI server. And your app will be fully ready for incoming requests. + +If you want to test more stable and reliable configuration which will be used in production, then run these commands in separate terminal window. + +```shell +source ./scripts/wsgi/development.sh gunicorn +``` + +```shell +source ./scripts/server/stop_nginx.sh +source ./scripts/wsgi/development.sh nginx +``` ### Database +#### What the app uses + In both development and testing environments `SQLite` is used. For production `PostgreSQL` is recommended, but you can use any of [supported databases](https://docs.sqlalchemy.org/en/13/core/engines.html#supported-databases). App already configured for both `SQLite` and `PostgreSQL`, for another database you may have to install additional Python packages. -Development and testing databases will be located at `src/development.sqlite` and `src/testing.sqlite` respectively. +By default both development and testing databases will be located at `src/temp.sqlite`. If you want to use different name for DB, then specify value for `DATABASE_URL` in `.env.development` and `.env.testing` files. + +`Redis` database is supported and expected, but not required. However, it is highly recommended to enable it, because many useful features of the app depends on `Redis` functionality and they will be disabled in case if `Redis` is not configured using `REDIS_URL`. + +#### What you should use + +Usually it will be better to manually specify DB name in `.env.development` and specify `REDIS_URL`. Try to always use `Redis`, including development environment. If you decide not to enable `Redis`, it is fine and you still can use the app with basic functionality which don't depends on `Redis`. + +### Background tasks + +#### What the app uses + +`RQ` is used as a task queue. `Redis` is required. + +Examples of jobs that will be enqueued: monitoring of uploading status and uploading of files. + +The app is not ready to support other task queues. You may need to make changes on your own if you decide to use another task queue. + +#### How to use that + +It is highly recommended that you run at least one worker. + +1. Make sure `REDIS_URL` is specified. +2. Open separate terminal window. +3. Activate `venv` and set environment variables. +4. Run: `python worker.py` + +These steps will run one worker instance. Count of workers depends on your expected server load. For `development` environment recommend count is 2. + +Note that `RQ` will not automatically reload your running workers when source code of any job function's changes. So, you should restart workers manually. + +### Expose local server + +#### What the app uses + +`ngrok` is used to expose local server. It is free and suitable for development server. + +#### What you should use + +You can use whatever you want. But if you decide to use `ngrok`, the app provides fews utils to make it easier. + +Before: +- requests will be routed to `/telegram_bot/webhook`, so, make sure you didn't change this route, +- you also should have [jq](https://stedolan.github.io/jq/) on your system. + +Then: + +1. Run `flask` server: + +```shell +source ./scripts/wsgi/development.sh flask +``` + +2. In separate terminal window run `ngrok`: + +```shell +source ./scripts/ngrok/run.sh +``` + +3. In separate terminal window set a webhook: + +```shell +source ./scripts/ngrok/set_webhook.sh +``` + +Where `` is your Telegram bot API token for specific environment (you can have different bots for different environments). ## Deployment @@ -168,10 +303,14 @@ Regardless of any platform you choose for hosting, it is recommended to manually ### Before -It is recommended to run linters with `./scripts/linters/all.sh` before deployment and resolve all errors and warnings. +It is recommended to run linters with `source ./scripts/linters/all.sh` before deployment and resolve all errors and warnings. ### Heroku +It is a way to host this app for free. And that will be more than enough until you have hundreds of active users. + +#### First time + 1. If you don't have [Heroku](https://heroku.com/) installed, then it is a time to do that. 2. If you don't have Heroku remote, then add it: @@ -201,7 +340,13 @@ heroku addons:create heroku-postgresql:hobby-dev Later you can view the DB content by using `heroku pg:psql`. -5. Set required environment variables: +5. We need Heroku `Redis` addon in order to use that database. + +``` +heroku addons:create heroku-redis:hobby-dev +``` + +6. Set required environment variables: ``` heroku config:set SERVER_NAME= @@ -216,19 +361,25 @@ heroku config:set GUNICORN_WORKERS= heroku config:set GUNICORN_WORKER_CONNECTIONS= ``` -6. Switch to new branch special for Heroku (don't ever push it!): +7. Switch to a new branch that is special for Heroku (don't ever push it!): ```git git checkout -b heroku ``` -7. Make sure `.env` file is created and filled. Remove it from `.gitignore`. Don't forget: don't ever push it anywhere but Heroku. +If that branch already created, then just type: + +``` +git checkout heroku +``` + +8. Make sure `.env.production` file is created and filled. Remove it from `.gitignore`. Don't forget: don't ever push it anywhere but Heroku. -8. Add changes for pushing to Heroku: +9. Add changes for pushing to Heroku: -- if you edited files on heroku branch: +- if you edited files on `heroku` branch: ```git -git add . +git add git commit -m ``` @@ -237,13 +388,21 @@ git commit -m git merge -m ``` -9. Upload files to Heroku: +10. Upload files to Heroku: ```git git push heroku heroku:master ``` -You should do № 8 and № 9 every time you want to push changes. +11. Set number of workers for background tasks. On free plan you cannot use more than 1 worker. + +``` +heroku scale worker=1 +``` + +#### What's next + +You should do steps № 7, 9 and 10 every time when you want to push changes. ## Contribution diff --git a/info/info.json b/info/info.json index 4823fa9..d6dae65 100644 --- a/info/info.json +++ b/info/info.json @@ -1,6 +1,6 @@ { "name": "Yandex.Disk Bot", - "version": "1.1.0", + "version": "1.2.0", "description": "This bot integrates Yandex.Disk into Telegram.", "about": "Work with Yandex.Disk.", "telegram": "Ya_Disk_Bot", @@ -21,34 +21,82 @@ "command": "about", "description": "Read about me" }, + { + "command": "commands", + "description": "See full list of commands" + }, { "command": "upload_photo", "description": "Upload a photo with quality loss" }, + { + "command": "public_upload_photo", + "description": "Upload a photo and publish it" + }, { "command": "upload_file", "description": "Upload a file with original state" }, + { + "command": "public_upload_file", + "description": "Upload a file and publish it" + }, { "command": "upload_audio", "description": "Upload an audio with original quality" }, + { + "command": "public_upload_audio", + "description": "Upload an audio and publish it" + }, { "command": "upload_video", "description": "Upload a video with original quality" }, + { + "command": "public_upload_video", + "description": "Upload a video and publish it" + }, { "command": "upload_voice", "description": "Upload a voice message" }, + { + "command": "public_upload_voice", + "description": "Upload a voice and publish it" + }, { "command": "upload_url", - "description": "Upload a file using direct URL" + "description": "Upload a resource using URL" + }, + { + "command": "public_upload_url", + "description": "Upload a resource using URL and publish it" + }, + { + "command": "publish", + "description": "Publish a file or folder using OS path" + }, + { + "command": "unpublish", + "description": "Unpublish a file or folder using OS path" }, { "command": "create_folder", "description": "Create a folder using OS path" }, + { + "command": "element_info", + "description": "Information about file or folder" + }, + { + "command": "space_info", + "description": "Information about remaining space" + }, + { + "command": "disk_info", + "description": "Information about your Yandex.Disk" + }, { "command": "grant_access", "description": "Grant me an access to your Yandex.Disk" diff --git a/manage.py b/manage.py index 79e5a96..b085ba3 100644 --- a/manage.py +++ b/manage.py @@ -4,8 +4,8 @@ from sqlalchemy.exc import IntegrityError from src.app import create_app +from src.extensions import db from src.database import ( - db, User, Chat, YandexDiskToken, diff --git a/migrations/versions/c6fa89936c27_init.py b/migrations/versions/c6fa89936c27_init.py index c676c89..cc70e63 100644 --- a/migrations/versions/c6fa89936c27_init.py +++ b/migrations/versions/c6fa89936c27_init.py @@ -1,7 +1,7 @@ """Init Revision ID: c6fa89936c27 -Revises: +Revises: Create Date: 2020-03-29 13:07:39.579009 """ @@ -24,7 +24,7 @@ def upgrade(): sa.Column('last_update_date', sa.DateTime(), server_default=sa.text('(CURRENT_TIMESTAMP)'), nullable=False), sa.Column('telegram_id', sa.Integer(), nullable=False, comment='Unique ID to identificate user in Telegram'), sa.Column('is_bot', sa.Boolean(), nullable=False, comment='User is bot in Telegram'), - sa.Column('language', sa.Enum('EN', name='supportedlanguages'), nullable=False, comment='Preferred language of user'), + sa.Column('language', sa.Enum('EN', name='supportedlanguage'), nullable=False, comment='Preferred language of user'), sa.Column('group', sa.Enum('USER', 'TESTER', 'ADMIN', name='usergroup'), nullable=False, comment='User rights group'), sa.PrimaryKeyConstraint('id'), sa.UniqueConstraint('telegram_id') diff --git a/migrations/versions/c8db92e01cf4_add_index_property_for_frequent_columns.py b/migrations/versions/c8db92e01cf4_add_index_property_for_frequent_columns.py new file mode 100644 index 0000000..5c6735a --- /dev/null +++ b/migrations/versions/c8db92e01cf4_add_index_property_for_frequent_columns.py @@ -0,0 +1,44 @@ +"""Add index property for frequent columns + +Revision ID: c8db92e01cf4 +Revises: 67ffbddd6efe +Create Date: 2020-11-24 22:27:44.498208 + +""" +from alembic import op +import sqlalchemy as sa + + +# revision identifiers, used by Alembic. +revision = 'c8db92e01cf4' +down_revision = '67ffbddd6efe' +branch_labels = None +depends_on = None + + +def upgrade(): + # ### commands auto generated by Alembic - please adjust! ### + with op.batch_alter_table('chats', schema=None) as batch_op: + batch_op.create_index(batch_op.f('ix_chats_telegram_id'), ['telegram_id'], unique=True) + + with op.batch_alter_table('users', schema=None) as batch_op: + batch_op.create_index(batch_op.f('ix_users_telegram_id'), ['telegram_id'], unique=True) + + with op.batch_alter_table('yandex_disk_tokens', schema=None) as batch_op: + batch_op.create_index(batch_op.f('ix_yandex_disk_tokens_user_id'), ['user_id'], unique=True) + + # ### end Alembic commands ### + + +def downgrade(): + # ### commands auto generated by Alembic - please adjust! ### + with op.batch_alter_table('yandex_disk_tokens', schema=None) as batch_op: + batch_op.drop_index(batch_op.f('ix_yandex_disk_tokens_user_id')) + + with op.batch_alter_table('users', schema=None) as batch_op: + batch_op.drop_index(batch_op.f('ix_users_telegram_id')) + + with op.batch_alter_table('chats', schema=None) as batch_op: + batch_op.drop_index(batch_op.f('ix_chats_telegram_id')) + + # ### end Alembic commands ### diff --git a/requirements.txt b/requirements.txt index f225ee4..2b4afa8 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,37 +1,48 @@ -alembic==1.4.2 -autopep8==1.5 -certifi==2019.11.28 -cffi==1.14.0 +alembic==1.4.3 +autopep8==1.5.4 +certifi==2020.6.20 +cffi==1.14.3 chardet==3.0.4 -click==7.1.1 -cryptography==2.8 +click==7.1.2 +cryptography==3.2.1 entrypoints==0.3 -Faker==4.0.2 -flake8==3.7.9 -Flask==1.1.1 +Faker==4.14.0 +flake8==3.8.4 +Flask==1.1.2 Flask-Migrate==2.5.3 -Flask-SQLAlchemy==2.4.1 +Flask-SQLAlchemy==2.4.4 gevent==1.5.0 greenlet==0.4.15 gunicorn==20.0.4 -idna==2.9 +hiredis==1.1.0 +idna==2.10 itsdangerous==1.1.0 -Jinja2==2.11.1 -Mako==1.1.2 +Jinja2==2.11.2 +kaleido==0.1.0a3 +Mako==1.1.3 MarkupSafe==1.1.1 mccabe==0.6.1 -psycopg2-binary==2.8.4 -pycodestyle==2.5.0 +numpy==1.19.3 +pandas==1.1.4 +plotly==4.12.0 +psycopg2-binary==2.8.6 +pycodestyle==2.6.0 pycparser==2.20 -pyflakes==2.1.1 +pyflakes==2.2.0 PyJWT==1.7.1 python-dateutil==2.8.1 -python-dotenv==0.12.0 +python-dotenv==0.15.0 python-editor==1.0.4 -requests==2.23.0 -six==1.14.0 -SQLAlchemy==1.3.15 +pytz==2020.1 +redis==3.5.3 +requests==2.24.0 +retrying==1.3.3 +rq==1.7.0 +six==1.15.0 +SQLAlchemy==1.3.20 text-unidecode==1.3 -urllib3==1.25.8 -Werkzeug==1.0.0 +toml==0.10.2 +urllib3==1.25.11 +Werkzeug==1.0.1 +youtube-dl==2020.12.14 -e . diff --git a/runtime.txt b/runtime.txt index 73b1cf8..43b47fb 100644 --- a/runtime.txt +++ b/runtime.txt @@ -1 +1 @@ -python-3.8.0 +python-3.8.5 diff --git a/scripts/ngrok/run.sh b/scripts/ngrok/run.sh new file mode 100644 index 0000000..205d26b --- /dev/null +++ b/scripts/ngrok/run.sh @@ -0,0 +1,3 @@ +#!/bin/bash + +ngrok http 8000 -bind-tls=true diff --git a/scripts/ngrok/set_webhook.sh b/scripts/ngrok/set_webhook.sh new file mode 100644 index 0000000..25cc7e7 --- /dev/null +++ b/scripts/ngrok/set_webhook.sh @@ -0,0 +1,8 @@ +#!/bin/bash + +ngrok_url=`curl http://localhost:4040/api/tunnels/command_line --silent --show-error | jq '.public_url' --raw-output` +webhook_url="$ngrok_url/telegram_bot/webhook" +bot_token=$1 +max_connections=30 + +source ./scripts/telegram/set_webhook.sh $bot_token $webhook_url $max_connections diff --git a/src/api/request.py b/src/api/request.py index a8ee217..9768244 100644 --- a/src/api/request.py +++ b/src/api/request.py @@ -23,7 +23,7 @@ class RequestResult(typing.TypedDict): def request( - raise_for_status=True, + raise_for_status=False, content_type: CONTENT_TYPE = "none", **kwargs ) -> RequestResult: diff --git a/src/api/telegram/__init__.py b/src/api/telegram/__init__.py index edce8c5..a7507b7 100644 --- a/src/api/telegram/__init__.py +++ b/src/api/telegram/__init__.py @@ -2,7 +2,9 @@ send_message, get_file, send_chat_action, - edit_message_text + edit_message_text, + send_photo, + delete_message ) from .requests import ( create_file_download_url diff --git a/src/api/telegram/methods.py b/src/api/telegram/methods.py index 7a3152e..58b5c1b 100644 --- a/src/api/telegram/methods.py +++ b/src/api/telegram/methods.py @@ -35,3 +35,33 @@ def edit_message_text(**kwargs): - see `api/request.py` documentation for more. """ return make_request("editMessageText", kwargs) + + +def delete_message(**kwargs): + """ + https://core.telegram.org/bots/api#deletemessage + + - see `api/request.py` documentation for more. + """ + return make_request("deleteMessage", kwargs) + + +def send_photo(**kwargs): + """ + https://core.telegram.org/bots/api#sendphoto + + - see `api/request.py` documentation for more. + - if you want to send bytes, then specify `photo` as + tuple from `files` in + https://requests.readthedocs.io/en/latest/api/#requests.request + """ + files = None + key = "photo" + value = kwargs.get(key) + + if not isinstance(value, str): + files = { + key: kwargs.pop(key) + } + + return make_request("sendPhoto", data=kwargs, files=files) diff --git a/src/api/telegram/requests.py b/src/api/telegram/requests.py index 5051c18..e05ecd8 100644 --- a/src/api/telegram/requests.py +++ b/src/api/telegram/requests.py @@ -43,36 +43,55 @@ def create_file_download_url(file_path: str) -> str: ) -def make_request(method_name: str, data: dict) -> dict: +def make_request( + method_name: str, + data: dict, + files: dict = None +) -> dict: """ Makes HTTP request to Telegram Bot API. - see `api/request.py` documentation for more. :param method_name: Name of API method in URL. - :param data: JSON data to send. + :param data: JSON data to send. It will be sent as + `application/json` payload. + :param files: Files data to send. If specified, then + `data` will be sent as query string. Files will be sent + as `multipart/form-data` payload. See `files` for more - + https://requests.readthedocs.io/en/latest/api/#requests.request :raises TelegramBotApiException: See `telegram/exceptions.py` documentation for more. """ url = create_bot_url(method_name) timeout = current_app.config["TELEGRAM_API_TIMEOUT"] + payload = { + "json": data + } + + if files: + payload = { + "files": files, + "params": data + } + result = request( content_type="json", method="POST", url=url, - json=data, timeout=timeout, allow_redirects=False, - verify=True + verify=True, + **payload ) - ok = result["content"]["ok"] + # 4xx or 5xx if not ok: raise RequestFailed( create_error_text( - result.content + result["content"] ) ) diff --git a/src/api/yandex/__init__.py b/src/api/yandex/__init__.py index e63ce3a..c66d8bd 100644 --- a/src/api/yandex/__init__.py +++ b/src/api/yandex/__init__.py @@ -1,9 +1,15 @@ from .methods import ( get_access_token, upload_file_with_url, - create_folder + create_folder, + publish, + unpublish, + get_disk_info, + get_element_info, + get_element_public_info ) from .requests import ( make_link_request, - create_user_oauth_url + create_user_oauth_url, + make_photo_preview_request ) diff --git a/src/api/yandex/methods.py b/src/api/yandex/methods.py index e1ce723..13105df 100644 --- a/src/api/yandex/methods.py +++ b/src/api/yandex/methods.py @@ -39,3 +39,73 @@ def create_folder(user_token: str, **kwargs): data=kwargs, user_token=user_token ) + + +def publish(user_token: str, **kwargs): + """ + https://yandex.ru/dev/disk/api/reference/publish-docpage/ + + - see `api/request.py` documentation for more. + """ + return make_disk_request( + http_method="PUT", + api_method="resources/publish", + data=kwargs, + user_token=user_token + ) + + +def unpublish(user_token: str, **kwargs): + """ + https://yandex.ru/dev/disk/api/reference/publish-docpage/ + + - see `api/request.py` documentation for more. + """ + return make_disk_request( + http_method="PUT", + api_method="resources/unpublish", + data=kwargs, + user_token=user_token + ) + + +def get_disk_info(user_token: str, **kwargs): + """ + https://yandex.ru/dev/disk/api/reference/capacity-docpage/ + + - see `api/request.py` documentation for more. + """ + return make_disk_request( + http_method="GET", + api_method="", + data=kwargs, + user_token=user_token + ) + + +def get_element_info(user_token: str, **kwargs): + """ + https://yandex.ru/dev/disk/api/reference/meta.html/ + + - see `api/request.py` documentation for more. + """ + return make_disk_request( + http_method="GET", + api_method="resources", + data=kwargs, + user_token=user_token + ) + + +def get_element_public_info(user_token: str, **kwargs): + """ + https://dev.yandex.net/disk-polygon/?lang=ru&tld=ru#!/v147disk47public47resources/GetPublicResource # noqa + + - see `api/request.py` documentation for more. + """ + return make_disk_request( + http_method="GET", + api_method="public/resources", + data=kwargs, + user_token=user_token + ) diff --git a/src/api/yandex/requests.py b/src/api/yandex/requests.py index e5de073..cf93679 100644 --- a/src/api/yandex/requests.py +++ b/src/api/yandex/requests.py @@ -1,5 +1,4 @@ from os import environ -import base64 from requests.auth import HTTPBasicAuth from flask import current_app @@ -29,12 +28,9 @@ def create_user_oauth_url(state: str) -> str: - https://yandex.ru/dev/oauth/doc/dg/concepts/about-docpage/ - :param state: `state` parameter. Will be encoded with base64. + :param state: urlsafe `state` parameter. """ client_id = environ["YANDEX_OAUTH_API_APP_ID"] - state = base64.urlsafe_b64encode( - state.encode() - ).decode() return ( "https://oauth.yandex.ru/authorize?" @@ -71,6 +67,7 @@ def make_oauth_request(method_name: str, data: dict): password = environ["YANDEX_OAUTH_API_APP_PASSWORD"] return request( + raise_for_status=False, content_type="json", method="POST", url=url, @@ -144,3 +141,38 @@ def make_link_request(data: dict, user_token: str): allow_redirects=False, verify=True ) + + +def make_photo_preview_request(photo_url: str, user_token: str): + """ + Makes request to URL in order to get bytes content of photo. + + Yandex requires user OAuth token in order to get + access to photo previews, so, it is why you should + use this method. + + - it will not raise in case of error HTTP code. + - see `api/request.py` documentation for more. + + :param photo_url: + URL of photo. + :param user_token: + User OAuth token to access this URL. + + :returns: + See `api/request.py`. + In case of `ok = True` under `content` will be bytes content + of requested photo. + """ + timeout = current_app.config["YANDEX_DISK_API_TIMEOUT"] + + return request( + raise_for_status=False, + content_type="bytes", + method="GET", + url=photo_url, + timeout=timeout, + auth=HTTPOAuthAuth(user_token), + allow_redirects=False, + verify=True + ) diff --git a/src/app.py b/src/app.py index c18d839..05b7ff6 100644 --- a/src/app.py +++ b/src/app.py @@ -7,18 +7,26 @@ from flask import ( Flask, redirect, - url_for + url_for, + current_app ) from .configs import flask_config -from .database import db, migrate +from .extensions import ( + db, + migrate, + redis_client, + task_queue +) from .blueprints import ( telegram_bot_blueprint, legal_blueprint ) -from .blueprints.utils import ( +from .blueprints._common.utils import ( absolute_url_for ) +# we need to import every model in order Migrate knows them +from .database.models import * # noqa: F403 def create_app(config_name: str = None) -> Flask: @@ -28,9 +36,10 @@ def create_app(config_name: str = None) -> Flask: app = Flask(__name__) configure_app(app, config_name) - configure_db(app) + configure_extensions(app) configure_blueprints(app) configure_redirects(app) + configure_error_handlers(app) return app @@ -47,13 +56,23 @@ def configure_app(app: Flask, config_name: str = None) -> None: app.config.from_object(config) -def configure_db(app: Flask) -> None: +def configure_extensions(app: Flask) -> None: """ - Configures database. + Configures Flask extensions. """ + # Database db.init_app(app) + + # Migration migrate.init_app(app, db) + # Redis + redis_client.init_app(app) + + # RQ + if redis_client.is_enabled: + task_queue.init_app(app, redis_client.connection) + def configure_blueprints(app: Flask) -> None: """ @@ -72,6 +91,12 @@ def configure_blueprints(app: Flask) -> None: def configure_redirects(app: Flask) -> None: """ Configures redirects. + + Note: all redirects to static content should be handled by + HTTP Reverse Proxy Server, not by WSGI HTTP Server. + We are keeping this redirect to static favicon only for + development builds where usually only Flask itself is used + (we want to see favicon at development stage - it is only the reason). """ @app.route("/favicon.ico") def favicon(): @@ -81,3 +106,23 @@ def favicon(): filename="favicons/favicon.ico" ) ) + + +def configure_error_handlers(app: Flask) -> None: + """ + Configures error handlers. + """ + if not app.config["DEBUG"]: + @app.errorhandler(404) + def not_found(error): + """ + We will redirect all requests rather than send "Not Found" + error, because we using web pages only for exceptional cases. + It is expected that all interaction with user should go + through Telegram when possible. + """ + return redirect( + location=app.config["PROJECT_URL_FOR_BOT"], + # temporary, in case if some routes will be added in future + code=302 + ) diff --git a/src/blueprints/_common/utils.py b/src/blueprints/_common/utils.py new file mode 100644 index 0000000..c1a832d --- /dev/null +++ b/src/blueprints/_common/utils.py @@ -0,0 +1,107 @@ +from datetime import datetime, timezone + +from flask import ( + current_app, + url_for +) + + +def absolute_url_for(endpoint: str, **kwargs) -> str: + """ + Implements Flask `url_for`, but by default + creates absolute URL (`_external` and `_scheme`) with + `PREFERRED_URL_SCHEME` scheme. + + - you can specify these parameters to change behavior. + + https://flask.palletsprojects.com/en/1.1.x/api/#flask.url_for + """ + if ("_external" not in kwargs): + kwargs["_external"] = True + + if ("_scheme" not in kwargs): + kwargs["_scheme"] = current_app.config["PREFERRED_URL_SCHEME"] + + return url_for(endpoint, **kwargs) + + +def get_current_datetime() -> dict: + """ + :returns: Information about current date and time. + """ + current_datetime = datetime.now(timezone.utc) + current_date = current_datetime.strftime("%d.%m.%Y") + current_time = current_datetime.strftime("%H:%M:%S") + current_timezone = current_datetime.strftime("%Z") + + return { + "date": current_date, + "time": current_time, + "timezone": current_timezone + } + + +def get_current_iso_datetime(sep="T", timespec="seconds") -> str: + """ + See https://docs.python.org/3.8/library/datetime.html#datetime.datetime.isoformat # noqa + """ + return datetime.now(timezone.utc).isoformat(sep, timespec) + + +def convert_iso_datetime(date_string: str) -> dict: + """ + :returns: + Pretty-print information about ISO 8601 `date_string`. + """ + value = datetime.fromisoformat(date_string) + value_date = value.strftime("%d.%m.%Y") + value_time = value.strftime("%H:%M:%S") + value_timezone = value.strftime("%Z") + + return { + "date": value_date, + "time": value_time, + "timezone": value_timezone + } + + +def bytes_to_human_unit( + bytes_count: int, + factor: float, + suffix: str +) -> str: + """ + Converts bytes to human readable string. + + - function source: https://stackoverflow.com/a/1094933/8445442 + - https://en.wikipedia.org/wiki/Binary_prefix + - https://man7.org/linux/man-pages/man7/units.7.html + + :param bytes_count: + Count of bytes to convert. + :param factor: + Use `1024.0` for binary and `1000.0` for decimal. + :param suffix: + Use `iB` for binary and `B` for decimal. + """ + for unit in ["", "K", "M", "G", "T", "P", "E", "Z"]: + if abs(bytes_count) < factor: + return "%3.1f %s%s" % (bytes_count, unit, suffix) + + bytes_count /= factor + + return "%.1f %s%s" % (bytes_count, "Y", suffix) + + +def bytes_to_human_binary(bytes_count: int) -> str: + """ + Bytes -> binary representation. + """ + return bytes_to_human_unit(bytes_count, 1024.0, "iB") + + +def bytes_to_human_decimal(bytes_count: int) -> str: + """ + Bytes -> decimal representation. + """ + return bytes_to_human_unit(bytes_count, 1000.0, "B") diff --git a/src/blueprints/legal/views.py b/src/blueprints/legal/views.py index 3e04e0d..67245ea 100644 --- a/src/blueprints/legal/views.py +++ b/src/blueprints/legal/views.py @@ -1,6 +1,6 @@ from flask import redirect, url_for -from src.blueprints.utils import absolute_url_for +from src.blueprints._common.utils import absolute_url_for from src.blueprints.legal import legal_blueprint as bp diff --git a/src/blueprints/telegram_bot/webhook/commands/common/__init__.py b/src/blueprints/telegram_bot/_common/__init__.py similarity index 100% rename from src/blueprints/telegram_bot/webhook/commands/common/__init__.py rename to src/blueprints/telegram_bot/_common/__init__.py diff --git a/src/blueprints/telegram_bot/_common/command_names.py b/src/blueprints/telegram_bot/_common/command_names.py new file mode 100644 index 0000000..5c91be8 --- /dev/null +++ b/src/blueprints/telegram_bot/_common/command_names.py @@ -0,0 +1,33 @@ +from enum import Enum, unique + + +@unique +class CommandName(Enum): + """ + Command supported by bot. + """ + START = "/start" + HELP = "/help" + ABOUT = "/about" + SETTINGS = "/settings" + YD_AUTH = "/grant_access" + YD_REVOKE = "/revoke_access" + UPLOAD_PHOTO = "/upload_photo" + UPLOAD_FILE = "/upload_file" + UPLOAD_AUDIO = "/upload_audio" + UPLOAD_VIDEO = "/upload_video" + UPLOAD_VOICE = "/upload_voice" + UPLOAD_URL = "/upload_url" + PUBLIC_UPLOAD_PHOTO = "/public_upload_photo" + PUBLIC_UPLOAD_FILE = "/public_upload_file" + PUBLIC_UPLOAD_AUDIO = "/public_upload_audio" + PUBLIC_UPLOAD_VIDEO = "/public_upload_video" + PUBLIC_UPLOAD_VOICE = "/public_upload_voice" + PUBLIC_UPLOAD_URL = "/public_upload_url" + CREATE_FOLDER = "/create_folder" + PUBLISH = "/publish" + UNPUBLISH = "/unpublish" + SPACE_INFO = "/space_info" + ELEMENT_INFO = "/element_info" + DISK_INFO = "/disk_info" + COMMANDS_LIST = "/commands" diff --git a/src/blueprints/telegram_bot/_common/stateful_chat.py b/src/blueprints/telegram_bot/_common/stateful_chat.py new file mode 100644 index 0000000..6395493 --- /dev/null +++ b/src/blueprints/telegram_bot/_common/stateful_chat.py @@ -0,0 +1,512 @@ +""" +Used for implementing of stateful dialog between the bot and user +on Telegram chats. It is just manages the state and data, so, this +module should work in pair with dispatcher that will route messages +based on current state, data and some custom conditions. In short, +this module manages the state and dispatcher manages the behavior, +dispatcher should be implemented independently. + +- requires Redis to be enabled. Use `stateful_chat_is_enabled()` +to check if stateful chat is enabled and can be used. +- you shouldn't import things that starts with `_`, +because they are intended for internal usage only + +Documentation for this entire module will be provided here, not in every +function documentation. This documentation tells the dispatcher how this +should be implemented, however, dispatcher can change final realization, +so, also take a look at dispatcher documentation for how to use this module. + +- each function can raise an error if it occurs + +"Custom Data" functions. + +Using set/get/delete_ you can manage your own data in +specific namespace. Instead of `update` operation use `set` operation. +For example, `set_user_data` sets/updates your custom data in user namespace, +that potentially can be used to store data about specific user across +multiple chats. `user` argument provides unique user identificator, as +id or name. `expire` argument tells if this data should be deletde after +given number of seconds, `0` is used for permanent storing. +Documentation for different namespaces (user, user in chat, chat, etc.) +is similar. + +"Event Handling" functions. + +"Disposable handler" means that only one handler will exists +at a time, and that handler should handle an event only one time. +For example, you can set handler for `TEXT` event, and when message with +text will arrive, this handler will be called for that message. Next time, +when text messages arrives, this handler will be not called, because +it already was called for previous message. +There can be only one handler, so, calling `set_disposable_handler` at +first in function № 1, then in function № 2, and after in function № 3 +will set only handler from function № 3, because last call from № 3 will +override handler from № 2, and call from № 2 will override handler from № 1. +You should specify set of events, not single event. Dispatcher should +implement the logic for how events of both handler and message are compared. +Dispatcher also should implement deleting of handler when it about to call. +It is recommended to stick with documeneted logic. +`user`, `chat`, `handler` - unique identifiers of these objects, such as +id's or names. +`expire` - means handler will be automatically deleted after given number +of seconds. `0` means it is permanent handler that will wait for it call. +If you call register function again for same handler, then old timeout +will be removed, and new one with this value will be setted. +`events` - iterable of unique events for that dispatcher. +Note: if you want to use `Enum`, then pass values of that enum, not +objects itself. Redis will return back strings even if you will pass +int values, so, be aware of it when comparing these values. +Note: return result is an unordered. So, you shouldn't rely on order. + +"Subscribed handlers" means you can register any amount of handlers for +any events, these handlers will be called every time until you remove them. +For example, you can register handler № 1 for TEXT and URL event, and +register handler № 2 for TEXT and PHOTO event. Realization of routing +is up to dispatcher, but recommend way is to route message (TEXT) to both +handler № 1 and handler № 2, route message (TEXT, PHOTO) to both +handler № 1 and handler № 2, and route message (URL) to handler № 1. +Documentation of function arguments is same as for "disposable handler". +Note: these handlers stored in a set, so, you can safely call register +function which registers same handler from different functions, and result +will be one registered handler. +""" + + +from collections import deque +from typing import Union, Set + +from src.extensions import redis_client + + +# region Common + + +# Namespaces +_SEPARATOR = ":" +_NAMESPACE_KEY = "stateful_chat" +_USER_KEY = "user" +_CHAT_KEY = "chat" +_DATA_KEY = "custom_data" +_DISPOSABLE_HANDLER_KEY = "disposable_handler" +_NAME_KEY = "name" +_EVENTS_KEY = "events" +_SUBSCRIBED_HANDLERS_KEY = "subscribed_handlers" + + +def _create_key(*args) -> str: + return _SEPARATOR.join(map(str, args)) + + +def stateful_chat_is_enabled() -> bool: + return redis_client.is_enabled + + +# endregion + + +# region Custom Data + + +def _set_data( + key: str, + field: str, + value: str, + expire: int +) -> None: + key = _create_key(_NAMESPACE_KEY, key, _DATA_KEY, field) + pipeline = redis_client.pipeline() + + pipeline.set(key, value) + + if (expire > 0): + pipeline.expire(key, expire) + + pipeline.execute(raise_on_error=True) + + +def _get_data( + key: str, + field: str +) -> Union[str, None]: + return redis_client.get( + _create_key(_NAMESPACE_KEY, key, _DATA_KEY, field) + ) + + +def _delete_data( + key: str, + field: str +) -> None: + redis_client.delete( + _create_key(_NAMESPACE_KEY, key, _DATA_KEY, field) + ) + + +def set_user_data( + user: str, + key: str, + value: str, + expire: int = 0 +) -> None: + _set_data( + _create_key(_USER_KEY, user), + key, + value, + expire + ) + + +def get_user_data( + user: str, + key: str +): + return _get_data( + _create_key(_USER_KEY, user), + key + ) + + +def delete_user_data( + user: str, + key: str +) -> None: + _delete_data( + _create_key(_USER_KEY, user), + key + ) + + +def set_user_chat_data( + user: str, + chat: str, + key: str, + value: str, + expire: int = 0 +) -> None: + _set_data( + _create_key(_USER_KEY, user, _CHAT_KEY, chat), + key, + value, + expire + ) + + +def get_user_chat_data( + user: str, + chat: str, + key: str +): + return _get_data( + _create_key(_USER_KEY, user, _CHAT_KEY, chat), + key + ) + + +def delete_user_chat_data( + user: str, + chat: str, + key: str +) -> None: + _delete_data( + _create_key(_USER_KEY, user, _CHAT_KEY, chat), + key + ) + + +def set_chat_data( + chat: str, + key: str, + value: str, + expire: int = 0 +) -> None: + _set_data( + _create_key(_CHAT_KEY, chat), + key, + value, + expire + ) + + +def get_chat_data( + chat: str, + key: str +): + return _get_data( + _create_key(_CHAT_KEY, chat), + key + ) + + +def delete_chat_data( + chat: str, + key: str +) -> None: + _delete_data( + _create_key(_CHAT_KEY, chat), + key + ) + + +# endregion + + +# region Event Handling + + +def set_disposable_handler( + user: str, + chat: str, + handler: str, + events: Set[str], + expire: int = 0 +) -> None: + name_key = _create_key( + _NAMESPACE_KEY, + _USER_KEY, + user, + _CHAT_KEY, + chat, + _DISPOSABLE_HANDLER_KEY, + _NAME_KEY + ) + events_key = _create_key( + _NAMESPACE_KEY, + _USER_KEY, + user, + _CHAT_KEY, + chat, + _DISPOSABLE_HANDLER_KEY, + _EVENTS_KEY + ) + pipeline = redis_client.pipeline() + + pipeline.set(name_key, handler) + # in case of update (same name for already + # existing handler) we need to delete old events + # in order to not merge them with new ones. + # also, `sadd` don't clears expire, but + # `delete` does + pipeline.delete(events_key) + pipeline.sadd(events_key, *events) + + if (expire > 0): + pipeline.expire(name_key, expire) + pipeline.expire(events_key, expire) + + pipeline.execute(raise_on_error=True) + + +def get_disposable_handler( + user: str, + chat: str +) -> Union[dict, None]: + name_key = _create_key( + _NAMESPACE_KEY, + _USER_KEY, + user, + _CHAT_KEY, + chat, + _DISPOSABLE_HANDLER_KEY, + _NAME_KEY + ) + events_key = _create_key( + _NAMESPACE_KEY, + _USER_KEY, + user, + _CHAT_KEY, + chat, + _DISPOSABLE_HANDLER_KEY, + _EVENTS_KEY + ) + pipeline = redis_client.pipeline() + + pipeline.get(name_key) + pipeline.smembers(events_key) + + name, events = pipeline.execute(raise_on_error=True) + result = None + + if name: + result = { + "name": name, + "events": events + } + + return result + + +def delete_disposable_handler( + user: str, + chat: str +) -> None: + name_key = _create_key( + _NAMESPACE_KEY, + _USER_KEY, + user, + _CHAT_KEY, + chat, + _DISPOSABLE_HANDLER_KEY, + _NAME_KEY + ) + events_key = _create_key( + _NAMESPACE_KEY, + _USER_KEY, + user, + _CHAT_KEY, + chat, + _DISPOSABLE_HANDLER_KEY, + _EVENTS_KEY + ) + pipeline = redis_client.pipeline() + + pipeline.delete(name_key) + pipeline.delete(events_key) + + pipeline.execute(raise_on_error=True) + + +def subscribe_handler( + user: str, + chat: str, + handler: str, + events: Set[str], + expire: int = 0 +) -> None: + # In this set stored all handler names + # that were registered. It is not indicator + # if handler is registered at the moment of check. + subscribed_handlers_key = _create_key( + _NAMESPACE_KEY, + _USER_KEY, + user, + _CHAT_KEY, + chat, + _SUBSCRIBED_HANDLERS_KEY + ) + # Example - `...::events`. + # In this set stored all events for this handler. + # If `events_key` doesn't exists or empty, then it is + # indicates that handler not registered anymore and + # should be removed from `subscribed_handlers_key`. + events_key = _create_key( + _NAMESPACE_KEY, + _USER_KEY, + user, + _CHAT_KEY, + chat, + _SUBSCRIBED_HANDLERS_KEY, + handler, + _EVENTS_KEY + ) + pipeline = redis_client.pipeline() + + pipeline.sadd(subscribed_handlers_key, handler) + # in case of update (same name for already + # existing handler) we need to delete old events + # in order to not merge them with new ones. + # also, `sadd` don't clears expire, but + # `delete` does + pipeline.delete(events_key) + pipeline.sadd(events_key, *events) + + if (expire > 0): + pipeline.expire(events_key) + + pipeline.execute(raise_on_error=True) + + +def unsubcribe_handler( + user: str, + chat: str, + handler: str +) -> None: + subscribed_handlers_key = _create_key( + _NAMESPACE_KEY, + _USER_KEY, + user, + _CHAT_KEY, + chat, + _SUBSCRIBED_HANDLERS_KEY + ) + events_key = _create_key( + _NAMESPACE_KEY, + _USER_KEY, + user, + _CHAT_KEY, + chat, + _SUBSCRIBED_HANDLERS_KEY, + handler, + _EVENTS_KEY + ) + pipeline = redis_client.pipeline() + + pipeline.srem(subscribed_handlers_key, handler) + pipeline.delete(events_key) + + pipeline.execute(raise_on_error=True) + + +def get_subscribed_handlers( + user: str, + chat: str +) -> deque: + subscribed_handlers_key = _create_key( + _NAMESPACE_KEY, + _USER_KEY, + user, + _CHAT_KEY, + chat, + _SUBSCRIBED_HANDLERS_KEY + ) + possible_handlers = redis_client.smembers( + subscribed_handlers_key + ) + pipeline = redis_client.pipeline() + + for possible_handler in possible_handlers: + pipeline.smembers( + _create_key( + _NAMESPACE_KEY, + _USER_KEY, + user, + _CHAT_KEY, + chat, + _SUBSCRIBED_HANDLERS_KEY, + possible_handler, + _EVENTS_KEY + ) + ) + + events = pipeline.execute(raise_on_error=True) + subscribed_handlers = deque() + i = 0 + + # `possible_handlers` is a set. + # Set it is an unordered structure, so, order + # of iteration not guaranted from time to time + # (for example, from first script execution to second + # script execution). + # However, in Python order of set iteration in + # single run is a same (if set wasn't modified), + # so, we can safely iterate this set one more time + # and associate it values with another values + # through `i` counter (`events` is an array). + # See for more: https://stackoverflow.com/q/3848091/8445442 + for handler_name in possible_handlers: + handler_events = events[i] + i += 1 + + # see `subscribe_handler` documentation for + # why this check works so + if handler_events: + subscribed_handlers.append({ + "name": handler_name, + "events": handler_events + }) + else: + unsubcribe_handler(user, chat, handler_name) + + return subscribed_handlers + + +# endregion diff --git a/src/blueprints/telegram_bot/webhook/telegram_interface.py b/src/blueprints/telegram_bot/_common/telegram_interface.py similarity index 75% rename from src/blueprints/telegram_bot/webhook/telegram_interface.py rename to src/blueprints/telegram_bot/_common/telegram_interface.py index 3495267..d36091b 100644 --- a/src/blueprints/telegram_bot/webhook/telegram_interface.py +++ b/src/blueprints/telegram_bot/_common/telegram_interface.py @@ -4,8 +4,6 @@ Any ) -from .commands import CommandsNames - class User: """ @@ -90,6 +88,7 @@ class Message: def __init__(self, raw_data: dict) -> None: self.raw_data = raw_data self.entities: Union[List[Entity], None] = None + self.plain_text: Union[str, None] = None @property def message_id(self) -> int: @@ -132,6 +131,73 @@ def get_text(self) -> str: "" ) + def get_date(self) -> int: + """ + :returns: + "Date the message was sent in Unix time" + """ + return self.raw_data["date"] + + def get_text_without_entities( + self, + without: List[str] + ) -> str: + """ + :param without: + Types of entities which should be removed + from message text. See + https://core.telegram.org/bots/api#messageentity + + :returns: + Text of message without specified entities. + Empty string in case if there are no initial + text or nothing left after removing. + """ + original_text = self.get_text() + result_text = original_text + entities = self.get_entities() + + if not original_text: + return "" + + for entity in entities: + if entity.type not in without: + continue + + offset = entity.offset + length = entity.length + value = original_text[offset:offset + length] + + result_text = result_text.replace(value, "") + + return result_text.strip() + + def get_plain_text(self) -> str: + """ + :returns: + `get_text_without_entities([mention, hashtag, + cashtag, bot_command, url, email, phone_number, + code, pre, text_link, text_mention]` + """ + if self.plain_text is not None: + return self.plain_text + + self.plain_text = self.get_text_without_entities([ + "mention", + "hashtag", + "cashtag", + "bot_command", + "url", + "email", + "phone_number", + "code", + "pre", + "text_link", + "text_mention" + ]) + + return self.plain_text + def get_entities(self) -> List[Entity]: """ :returns: Entities from a message. @@ -197,33 +263,6 @@ def get_entity_value( return value - def guess_bot_command(self, default=CommandsNames.HELP) -> str: - """ - Tries to guess which bot command - user assumed based on a message. - - :param default: Default command which will be - returned if unable to guess. - - :returns: Guessed bot command based on a message. - """ - command = default - - if ("photo" in self.raw_data): - command = CommandsNames.UPLOAD_PHOTO - elif ("document" in self.raw_data): - command = CommandsNames.UPLOAD_FILE - elif ("audio" in self.raw_data): - command = CommandsNames.UPLOAD_AUDIO - elif ("video" in self.raw_data): - command = CommandsNames.UPLOAD_VIDEO - elif ("voice" in self.raw_data): - command = CommandsNames.UPLOAD_VOICE - elif (self.get_entity_value("url") is not None): - command = CommandsNames.UPLOAD_URL - - return command - class Request: """ diff --git a/src/blueprints/telegram_bot/_common/yandex_disk.py b/src/blueprints/telegram_bot/_common/yandex_disk.py new file mode 100644 index 0000000..2454302 --- /dev/null +++ b/src/blueprints/telegram_bot/_common/yandex_disk.py @@ -0,0 +1,618 @@ +from time import sleep +from typing import Generator, Deque +from collections import deque + +from flask import current_app + +from src.api import yandex + + +# region Exceptions + + +class YandexAPIRequestError(Exception): + """ + Unknown error occurred during Yandex.Disk API request + (not necessarily from Yandex). + """ + pass + + +class YandexAPIError(Exception): + """ + Error response from Yandex.Disk API. + + - may contain human-readable error message. + """ + pass + + +class YandexAPICreateFolderError(Exception): + """ + Unable to create folder on Yandex.Disk. + + - may contain human-readable error message. + """ + pass + + +class YandexAPIUploadFileError(Exception): + """ + Unable to upload file on Yandex.Disk. + + - may contain human-readable error message. + """ + pass + + +class YandexAPIPublishItemError(Exception): + """ + Unable to pubish an item from Yandex.Disk. + + - may contain human-readable error message. + """ + pass + + +class YandexAPIUnpublishItemError(Exception): + """ + Unable to unpublish an item from Yandex.Disk. + + - may contain human-readable error message. + """ + pass + + +class YandexAPIGetElementInfoError(Exception): + """ + Unable to get information about Yandex.Disk + element (folder or file). + + - may contain human-readable error message. + """ + pass + + +class YandexAPIExceededNumberOfStatusChecksError(Exception): + """ + There was too much attempts to check status + of Yandex.Disk operation. Yandex didn't give any + acceptable status, i.e. operation still in progress + (most probably) and function cannot do any status + checks because of check limit. So, operation status + becoming unknown. + """ + pass + + +# endregion + + +# region Helpers + + +class YandexDiskPath: + """ + Yandex.Disk path to resource. + + - you should use this class, not raw strings from user! + """ + def __init__(self, *args): + """ + :param *args: + List of raw paths from user. + """ + self.separator = "/" + self.disk_namespace = "disk:" + self.trash_namespace = "trash:" + self.raw_paths = args + + def get_absolute_path(self) -> Deque[str]: + """ + :returns: + Iterable of resource names without separator. + Join result with separator will be a valid Yandex.Disk path. + + :examples: + 1) `self.raw_paths = ["Telegram Bot/test", "name.jpg"]` -> + `["disk:", "Telegram Bot", "test", "name.jpg"]`. + 2) `self.raw_paths = ["disk:/Telegram Bot//test", "/name.jpg/"]` -> + `["disk:", "Telegram Bot", "test", "name.jpg"]`. + """ + data = deque() + + for raw_path in self.raw_paths: + data.extend( + [x for x in raw_path.split(self.separator) if x] + ) + + if not data: + data.append(self.disk_namespace) + + namespace = data[0] + is_valid_namepsace = namespace in ( + self.disk_namespace, + self.trash_namespace + ) + + # Yandex.Disk path must starts with some namespace + # (Disk, Trash, etc.). Paths without valid namespace + # are invalid! However, they can work without namespace. + # But paths without namespace can lead to unexpected + # error at any time. So, you always should add namespace. + # For example, `Telegram Bot/12` will work, but + # `Telegram Bot/12:12` will lead to `DiskPathFormatError` + # from Yandex because Yandex assumes `12` as namespace. + # `disk:/Telegram Bot/12:12` will work fine. + if not is_valid_namepsace: + # Let's use Disk namespace by default + data.appendleft(self.disk_namespace) + + return data + + def create_absolute_path(self) -> str: + """ + :returns: + Valid absolute path. + """ + data = self.get_absolute_path() + + # if path is only namespace, then + # it should end with separator, + # otherwise there should be no + # separator at the end + if (len(data) == 1): + return f"{data.pop()}{self.separator}" + else: + return self.separator.join(data) + + def generate_absolute_path( + self, + include_namespace=True + ) -> Generator[str, None, None]: + """ + :yields: + Valid absolute path piece by piece. + + :examples: + 1) `create_absolute_path()` -> `disk:/Telegram Bot/folder/file.jpg` + `generate_absolute_path(True)` -> `[disk:/, disk:/Telegram Bot, + disk:/Telegram Bot/folder, disk:/Telegram Bot/folder/file.jpg]`. + """ + data = self.get_absolute_path() + absolute_path = data.popleft() + + if include_namespace: + yield f"{absolute_path}{self.separator}" + + for element in data: + absolute_path += f"{self.separator}{element}" + yield absolute_path + + +def create_yandex_error_text(data: dict) -> str: + """ + :returns: + Human error message from Yandex error response. + """ + error_name = data.get("error", "?") + error_description = ( + data.get("message") or + data.get("description") or + "?" + ) + + return (f"{error_name}: {error_description}") + + +def is_error_yandex_response(data: dict) -> bool: + """ + :returns: Yandex response contains error or not. + """ + return ("error" in data) + + +def yandex_operation_is_success(data: dict) -> bool: + """ + :returns: + Yandex response contains status which + indicates that operation is successfully ended. + """ + return ( + ("status" in data) and + (data["status"] == "success") + ) + + +def yandex_operation_is_failed(data: dict) -> bool: + """ + :returns: + Yandex response contains status which + indicates that operation is failed. + """ + return ( + ("status" in data) and + (data["status"] in ( + # Yandex documentation is different in some places + "failure", + "failed" + )) + ) + + +# endregion + + +# region API + + +def create_folder( + user_access_token: str, + folder_name: str +) -> int: + """ + Creates folder using Yandex API. + + Yandex not able to create folder if some of + middle folders not exists. This method will try to create + each folder one by one, and ignore safe errors (if + already exists, for example) from all folder names + except last one. + + :returns: + Last (for last folder name) HTTP Status code. + + :raises: + `YandexAPIRequestError`, + `YandexAPICreateFolderError`. + """ + path = YandexDiskPath(folder_name) + resources = path.generate_absolute_path(True) + last_status_code = 201 # namespace always created + allowed_errors = [409] + + for resource in resources: + result = None + + try: + result = yandex.create_folder( + user_access_token, + path=resource + ) + except Exception as error: + raise YandexAPIRequestError(error) + + response = result["content"] + last_status_code = result["status_code"] + + if ( + (last_status_code == 201) or + (last_status_code in allowed_errors) or + not is_error_yandex_response(response) + ): + continue + + raise YandexAPICreateFolderError( + create_yandex_error_text(response) + ) + + return last_status_code + + +def publish_item( + user_access_token: str, + absolute_item_path: str +) -> None: + """ + Publish an item that already exists on Yandex.Disk. + + :raises: + `YandexAPIRequestError`, + `YandexAPIPublishItemError`. + """ + path = YandexDiskPath(absolute_item_path) + absolute_path = path.create_absolute_path() + + try: + response = yandex.publish( + user_access_token, + path=absolute_path + ) + except Exception as error: + raise YandexAPIRequestError(error) + + response = response["content"] + is_error = is_error_yandex_response(response) + + if is_error: + raise YandexAPIPublishItemError( + create_yandex_error_text(response) + ) + + +def unpublish_item( + user_access_token: str, + absolute_item_path: str +) -> None: + """ + Unpublish an item that already exists on Yandex.Disk. + + :raises: + `YandexAPIRequestError`, + `YandexAPIUnpublishItemError`. + """ + path = YandexDiskPath(absolute_item_path) + absolute_path = path.create_absolute_path() + + try: + response = yandex.unpublish( + user_access_token, + path=absolute_path + ) + except Exception as error: + raise YandexAPIRequestError(error) + + response = response["content"] + is_error = is_error_yandex_response(response) + + if is_error: + raise YandexAPIUnpublishItemError( + create_yandex_error_text(response) + ) + + +def upload_file_with_url( + user_access_token: str, + folder_path: str, + file_name: str, + download_url: str +) -> Generator[dict, None, None]: + """ + Uploads a file to Yandex.Disk using file download url. + + - before uploading creates a folder. + - after uploading will monitor operation status according + to app configuration. Because it is synchronous, it may + take significant time to end this function! + + :yields: + `dict` with `success`, `failed`, `completed`, `status`. + It will yields with some interval (according + to app configuration). Order is an order in which + Yandex sent an operation status, so, `status` can + be safely logged to user. + `status` - currenet string status of uploading + (for example, `in progress`). + `completed` - uploading is completed. + `success` - uploading is successfully ended. + `failed` - uploading is failed (unknown error, known + error will be throwed with `YandexAPIUploadFileError`). + + :raises: + `YandexAPIRequestError`, + `YandexAPICreateFolderError`, + `YandexAPIUploadFileError`, + `YandexAPIExceededNumberOfStatusChecksError` + """ + create_folder( + user_access_token=user_access_token, + folder_name=folder_path + ) + + path = YandexDiskPath(folder_path, file_name) + absolute_path = path.create_absolute_path() + response = None + + try: + response = yandex.upload_file_with_url( + user_access_token, + url=download_url, + path=absolute_path + ) + except Exception as error: + raise YandexAPIRequestError(error) + + operation_status_link = response["content"] + is_error = is_error_yandex_response(operation_status_link) + + if is_error: + raise YandexAPIUploadFileError( + create_yandex_error_text(operation_status_link) + ) + + operation_status = None + is_error = False + is_success = False + is_failed = False + is_completed = False + attempt = 0 + max_attempts = current_app.config[ + "YANDEX_DISK_API_CHECK_OPERATION_STATUS_MAX_ATTEMPTS" + ] + interval = current_app.config[ + "YANDEX_DISK_API_CHECK_OPERATION_STATUS_INTERVAL" + ] + too_many_attempts = (attempt >= max_attempts) + + while not ( + is_error or + is_completed or + too_many_attempts + ): + sleep(interval) + + try: + response = yandex.make_link_request( + data=operation_status_link, + user_token=user_access_token + ) + except Exception as error: + raise YandexAPIRequestError(error) + + operation_status = response["content"] + is_error = is_error_yandex_response(operation_status) + is_success = yandex_operation_is_success(operation_status) + is_failed = yandex_operation_is_failed(operation_status) + is_completed = (is_success or is_failed) + attempt += 1 + too_many_attempts = (attempt >= max_attempts) + + if not is_error: + yield { + "success": is_success, + "failed": is_failed, + "completed": (is_success or is_failed), + "status": operation_status.get( + "status", + "unknown" + ) + } + + if is_error: + raise YandexAPIUploadFileError( + create_yandex_error_text(operation_status) + ) + elif ( + too_many_attempts and + not is_completed + ): + raise YandexAPIExceededNumberOfStatusChecksError() + + +def get_disk_info(user_access_token: str) -> dict: + """ + See for interface: + - https://yandex.ru/dev/disk/api/reference/capacity-docpage/ + - https://dev.yandex.net/disk-polygon/#!/v147disk + + :returns: + Information about user Yandex.Disk. + + :raises: + `YandexAPIRequestError`. + """ + try: + response = yandex.get_disk_info(user_access_token) + except Exception as error: + raise YandexAPIRequestError(error) + + response = response["content"] + is_error = is_error_yandex_response(response) + + if is_error: + raise YandexAPIUploadFileError( + create_yandex_error_text(response) + ) + + return response + + +def get_element_info( + user_access_token: str, + absolute_element_path: str, + get_public_info=False, + preview_size="L", + preview_crop=False, + embedded_elements_limit=0, + embedded_elements_offset=0, + embedded_elements_sort="name" +) -> dict: + """ + - https://yandex.ru/dev/disk/api/reference/meta.html + - https://dev.yandex.net/disk-polygon/?lang=ru&tld=ru#!/v147disk47public47resources/GetPublicResource # noqa + + :param user_access_token: + User access token. + :param absolute_element_path: + Absolute path of element. + :param get_public_info: + Make another HTTP request to get public info. + If `True`, then these fields will be added in + normal info: `views_count`, `owner`. + Set to `False` if you don't need this information - + this will improve speed. + :param preview_size: + Size of preview (if available). + :param preview_crop: + Allow cropping of preview. + :param embedded_elements_limit: + How many embedded elements (elements inside + folder) should be included in response. + Set to `0` if you don't need this information - + this will improve speed. + :param embedded_elements_offset: + Offset for embedded elements. + Note `sort` parameter. + :param embedded_elements_sort: + How to sort embedded elements. + Possible values: `name`, `path`, `created`, + `modified`, `size`. Append `-` for reverse + order (example: `-name`). + + :returns: + Information about object. + Check for keys before using them! + + :raises: + `YandexAPIRequestError`, `YandexAPIGetElementInfoError`. + """ + path = YandexDiskPath(absolute_element_path) + absolute_path = path.create_absolute_path() + + try: + response = yandex.get_element_info( + user_access_token, + path=absolute_path, + preview_crop=preview_crop, + preview_size=preview_size, + limit=embedded_elements_limit, + offset=embedded_elements_offset, + sort=embedded_elements_sort + ) + except Exception as error: + raise YandexAPIRequestError(error) + + response = response["content"] + is_error = is_error_yandex_response(response) + + if is_error: + raise YandexAPIGetElementInfoError( + create_yandex_error_text(response) + ) + + if ( + get_public_info and + ("public_key" in response) + ): + public_info_response = None + + try: + public_info_response = yandex.get_element_public_info( + user_access_token, + public_key=response["public_key"], + # we need only these fields, because they + # are missing in normal info + fields="views_count,owner", + preview_crop=preview_crop, + preview_size=preview_size, + limit=embedded_elements_limit, + offset=embedded_elements_offset, + sort=embedded_elements_sort + ) + except Exception as error: + raise YandexAPIRequestError(error) + + public_info_response = public_info_response["content"] + is_error = is_error_yandex_response(public_info_response) + + if is_error: + raise YandexAPIGetElementInfoError( + create_yandex_error_text(response) + ) + + response = {**response, **public_info_response} + + return response + + +# endregion diff --git a/src/blueprints/telegram_bot/_common/yandex_oauth.py b/src/blueprints/telegram_bot/_common/yandex_oauth.py new file mode 100644 index 0000000..c116ad5 --- /dev/null +++ b/src/blueprints/telegram_bot/_common/yandex_oauth.py @@ -0,0 +1,517 @@ +import base64 +import secrets +from enum import Enum, auto +from typing import Union + +from flask import current_app +import jwt + +from src.api import yandex +from src.extensions import db +from src.database import ( + User, + YandexDiskToken, + UserQuery, + ChatQuery +) + + +class InvalidState(Exception): + """ + Provided state is invalid + (invalid Base64, missing data, wrong data, etc.). + For security reasons there is no exact reason. + """ + pass + + +class ExpiredInsertToken(Exception): + """ + Provided insert token is expired. + """ + pass + + +class InvalidInsertToken(Exception): + """ + Provided insert token (extracted from state) is invalid. + Most probably new state was requested and old one + was passed for handling. + """ + + +class YandexRequestError(Exception): + """ + Unexpected error occurred during Yandex.OAuth HTTP request. + """ + pass + + +class MissingData(Exception): + """ + Requested data is missing. + """ + pass + + +class YandexOAuthClient: + """ + Base class for all Yandex.OAuth clients. + """ + def encode_state(self, user_id: int, insert_token: str) -> str: + """ + :returns: + JWT which should be used as a value for `state` + Yandex.OAuth key. It is urlsafe base64 string. + """ + return base64.urlsafe_b64encode( + jwt.encode( + { + "user_id": user_id, + "insert_token": insert_token + }, + current_app.secret_key.encode(), + algorithm="HS256" + ) + ).decode() + + def decode_state(self, state: str) -> dict: + """ + :param state: + A state from `create_state()`. + + :returns: + A dict of arguments that were passed into `create_state()`. + + :raises: + `InvalidState`. + """ + encoded_state = None + + try: + encoded_state = base64.urlsafe_b64decode( + state.encode() + ).decode() + except Exception: + raise InvalidState() + + decoded_state = None + + try: + decoded_state = jwt.decode( + encoded_state, + current_app.secret_key.encode(), + algorithm="HS256" + ) + except Exception: + raise InvalidState() + + user_id = decoded_state.get("user_id") + insert_token = decoded_state.get("insert_token") + + if not any((user_id, insert_token)): + raise InvalidState() + + return { + "user_id": user_id, + "insert_token": insert_token + } + + def get_user(self, user_id: int, insert_token: str) -> User: + """ + :param user_id: + DB id of needed user. + :param insert_token: + User will be returned only in case when provided + insert token matchs with one from DB. This means + you are allowed to modify this DB user. + Insert token of that user can be modified in futher by + some another operation, so, you should call this function + once and reuse returned result. + + :returns: + DB user. + + :raises: + `MissingData`, `ExpiredInsertToken`, `InvalidInsertToken`. + """ + user = UserQuery.get_user_by_id(user_id) + + if ( + user is None or + # for some reason `yandex_disk_token` not created, + # it is not intended behavior. + user.yandex_disk_token is None + ): + raise MissingData() + + db_insert_token = None + + try: + db_insert_token = user.yandex_disk_token.get_insert_token() + except Exception: + raise ExpiredInsertToken() + + if (insert_token != db_insert_token): + raise InvalidInsertToken() + + return user + + def request_access_token(self, code="", refresh_token="") -> dict: + """ + Makes HTTP request to Yandex.OAuth API to get access token. + + - you should specify only one parameter: + `code` or `refresh_token`. If specified both, then `code` + will be selected. If nothing is specified, then an error + will be thrown. + + :returns: + `ok` indicates status of operation. + If `ok = False`, then `error` will contain + `title` and optional `description`. + if `ok = True`, then `access_token`, `token_type`, + `expires_in`, `refresh_token` will be provided. + + :raises: + `YandexRequestError`. + """ + response = None + kwargs = {} + + if code: + kwargs["grant_type"] = "authorization_code" + kwargs["code"] = code + elif refresh_token: + kwargs["grant_type"] = "refresh_token" + kwargs["refresh_token"] = refresh_token + else: + raise Exception("Invalid arguments") + + try: + response = yandex.get_access_token( + **kwargs + )["content"] + except Exception as error: + raise YandexRequestError(str(error)) + + if "error" in response: + return { + "ok": False, + "error": { + "title": response["error"], + "description": response.get("error_description") + } + } + + return { + "ok": True, + "access_token": response["access_token"], + "token_type": response["token_type"], + "expires_in": response["expires_in"], + "refresh_token": response["refresh_token"], + } + + def set_access_token(self, user: User, code: str) -> dict: + """ + Makes request to Yandex.OAuth server, gets access + token and saves it. + + - on success clears insert token. + - perform a DB commit in order to save changes! + + :param user: + DB user. + :param code: + Code from Yandex which was given to user. + + :returns: + `ok` which contains status of operation. + `error` from Yandex in case of `ok = False`, + `error` contains `title` and optional `description`. + + :raises: + `YandexRequestError`. + """ + response = self.request_access_token(code=code) + + if not response["ok"]: + return response + + user.yandex_disk_token.clear_insert_token() + user.yandex_disk_token.set_access_token( + response["access_token"] + ) + user.yandex_disk_token.access_token_type = ( + response["token_type"] + ) + user.yandex_disk_token.access_token_expires_in = ( + response["expires_in"] + ) + user.yandex_disk_token.set_refresh_token( + response["refresh_token"] + ) + + return { + "ok": True + } + + def refresh_access_token(self, user: User) -> dict: + """ + Similar to `set_access_token()`, but uses user + refresh token from DB. + + - perform DB commit in order to save changes! + - `error` not always presented in case of `ok = False`. + + :raises: + `YandexRequestError`. + """ + refresh_token = user.yandex_disk_token.get_refresh_token() + + if refresh_token is None: + return { + "ok": False + } + + response = self.request_access_token(refresh_token=refresh_token) + + if not response["ok"]: + return response + + user.yandex_disk_token.clear_insert_token() + user.yandex_disk_token.set_access_token( + response["access_token"] + ) + user.yandex_disk_token.access_token_type = ( + response["token_type"] + ) + user.yandex_disk_token.access_token_expires_in = ( + response["expires_in"] + ) + user.yandex_disk_token.set_refresh_token( + response["refresh_token"] + ) + + return { + "ok": True + } + + def clear_access_token(self, user: User) -> None: + """ + Clears access token. + + - perform DB commit in order to save changes! + """ + user.yandex_disk_token.clear_access_token() + user.yandex_disk_token.clear_refresh_token() + + def have_valid_access_token(self, user: User) -> bool: + """ + :returns: + User have valid (not expired) access token. + """ + token = user.yandex_disk_token + + if not token: + return False + + if not token.have_access_token(): + return False + + try: + # there will be errors in case of + # expired or invalid token + token.get_access_token() + except Exception: + return False + + return True + + def create_insert_token(self, user: User) -> str: + """ + Creates insert token (used to insert new data). + + WARNING: it clears all previous data + (access token, refresh token, etc)! + + - perform DB commit in order to save changes! + + :returns: + Created insert token. + + :raises: + `MissingData` (DB data is corrupted or problems with DB). + """ + user.yandex_disk_token.clear_all_tokens() + user.yandex_disk_token.set_insert_token( + secrets.token_hex( + current_app.config[ + "YANDEX_OAUTH_API_INSERT_TOKEN_BYTES" + ] + ) + ) + user.yandex_disk_token.insert_token_expires_in = ( + current_app.config[ + "YANDEX_OAUTH_API_INSERT_TOKEN_LIFETIME" + ] + ) + + # it is necessary to check if we able to get + # valid token after inseting + insert_token = user.yandex_disk_token.get_insert_token() + + if insert_token is None: + raise MissingData("Insert token is NULL") + + return insert_token + + +class YandexOAuthAutoCodeClient(YandexOAuthClient): + """ + Implements https://yandex.ru/dev/oauth/doc/dg/reference/auto-code-client.html # noqa + """ + def before_user_interaction(self, user: User) -> dict: + """ + This function should be executed before user interation. + + :returns: + `status` that contains operation status. See `OperationStatus` + documentation for more. In case of `status = CONTINUE_TO_URL` + there will be both `url` and `lifetime`. User should open this + url, after `lifetime` seconds this url will be expired. + `state` is used to avoid handling of url, but you should + already have valid code from Yandex. + In case of any other `status` further user actions not needed + because this user already have valid access token. + + :raises: + `YandexRequestError`, `MissingData`. + """ + # it can be not created if it is a new user + if not user.yandex_disk_token: + db.session.add( + YandexDiskToken(user=user) + ) + elif self.have_valid_access_token(user): + return { + "status": OperationStatus.HAVE_ACCESS_TOKEN + } + + refresh_result = self.refresh_access_token(user) + + # if `ok = False`, then there can be useful error message + # from Yandex with some description. At the moment + # we will do nothing with it and just continue + # with need of user interaction + if refresh_result["ok"]: + db.session.commit() + + return { + "status": OperationStatus.ACCESS_TOKEN_REFRESHED + } + + insert_token = self.create_insert_token(user) + state = self.encode_state(user.id, insert_token) + url = yandex.create_user_oauth_url(state) + lifetime_in_seconds = ( + user.yandex_disk_token.insert_token_expires_in + ) + + db.session.commit() + + return { + "status": OperationStatus.CONTINUE_TO_URL, + "url": url, + "lifetime": lifetime_in_seconds, + "state": state + } + + def after_success_redirect(self, state: str, code: str) -> dict: + """ + Should be called after Yandex successful redirect + (when there is both `code` and `state` parameters). + Performs needed operations to end user authorization. + + :returns: + `ok` which contains status of setting of access token. + `error` from Yandex in case of `ok = False`, + `error` contains `title` and optional `description`. + If `ok = False`, you should notify user about occured error + and user should request new authorization link because old + one will become invalid. + `user` is DB user. + + :raises: + - `InvalidState`, `ExpiredInsertToken`, `MissingData`, + `InvalidInsertToken`, `YandexRequestError`. + - Other errors (`Exception`) should be considered as + internal server error. + """ + data = self.decode_state(state) + user = self.get_user( + data["user_id"], + data["insert_token"] + ) + result = self.set_access_token(user, code) + result["user"] = user + + if not result["ok"]: + user.yandex_disk_token.clear_insert_token() + + db.session.commit() + + return result + + def after_error_redirect(self, state: str) -> None: + """ + Should be called after Yandex error redirect + (when there is both `error` and `state` parameters). + + - if function successfully ends, then old user authorization + link will become invalid. + + :raises: + `InvalidState`, `ExpiredInsertToken`, + `InvalidInsertToken`, `MissingData`. + """ + data = self.decode_state(state) + user = self.get_user( + data["user_id"], + data["insert_token"] + ) + + user.yandex_disk_token.clear_insert_token() + db.session.commit() + + +# It inherits `YandexOAuthAutoCodeClient`, not base `YandexOAuthClient`, +# because we will use it exactly like `YandexOAuthAutoCodeClient`. +# We doing so because `YandexOAuthConsoleClient` intended mostly +# for usage at development process, so, UX not the key. +# However, it is better to write pure code for that module later +class YandexOAuthConsoleClient(YandexOAuthAutoCodeClient): + """ + Implements https://yandex.ru/dev/oauth/doc/dg/reference/console-client.html # noqa + """ + def handle_code(self, state: str, code: str) -> dict: + """ + See `YandexOAuthAutoCodeClient.after_success_redirect` documentation. + """ + return self.after_success_redirect(state, code) + + +class OperationStatus(Enum): + """ + Status of requested operation. + """ + # User already have valid access token. + # No further actions is required + HAVE_ACCESS_TOKEN = auto() + + # Access token was successfully refreshed. + # No further actions is required + ACCESS_TOKEN_REFRESHED = auto() + + # User should manually open an URL + CONTINUE_TO_URL = auto() diff --git a/src/blueprints/telegram_bot/_common/youtube_dl.py b/src/blueprints/telegram_bot/_common/youtube_dl.py new file mode 100644 index 0000000..7ecd6eb --- /dev/null +++ b/src/blueprints/telegram_bot/_common/youtube_dl.py @@ -0,0 +1,149 @@ +import youtube_dl + + +# region Exceptions + + +class CustomYoutubeDLError(Exception): + """ + Base exception for all custom `youtube_dl` errors. + """ + pass + + +class UnsupportedURLError(CustomYoutubeDLError): + """ + Provided URL is not supported by `youtube_dl` or + not allowed by custom rules to be handled. + """ + pass + + +class UnexpectedError(CustomYoutubeDLError): + """ + Some unexpected error occured. + """ + pass + + +# endregion + + +# region Utils + + +class Logger: + """ + Custom logger for `youtube_dl`. + """ + def debug(self, message: str) -> None: + pass + + def warning(self, message: str) -> None: + pass + + def error(self, message: str) -> None: + pass + + +# endregion + + +# region youtube_dl + + +def extract_info(url: str) -> dict: + """ + Extracts info from URL. + This info can be used to download this resource. + + - see this for supported sites - + https://github.com/ytdl-org/youtube-dl/blob/master/docs/supportedsites.md + + :param url: + URL to resource. + You can safely pass any URL + (YouTube video, direct URL to JPG, plain page, etc.) + + :returns: + `direct_url` - can be used to download resource, + `filename` - recommended filename. + If provided URL not supported, then error will be + raised. So, if result is successfully returned, + then it is 100% valid result which can be used to + download resource. + + :raises: + `UnsupportedURLError`, + `UnexpectedError`. + """ + result = { + "direct_url": None, + "filename": None + } + info = None + + # TODO: implement execution timeout, + # because long requests blocks server requests + try: + info = ydl.extract_info(url) + except youtube_dl.DownloadError as error: + raise UnsupportedURLError(str(error)) + except Exception as error: + raise UnexpectedError(str(error)) + + direct_url = info.get("url") + + if not direct_url: + raise UnsupportedURLError( + "youtube_dl didn't return direct URL" + ) + + result["direct_url"] = direct_url + + if info.get("direct"): + result["filename"] = info.get("webpage_url_basename") + else: + result["filename"] = ydl.prepare_filename(info) + + return result + + +# endregion + + +options = { + # We using `youtube_dl` to get information + # for download and pass further to another service + # (Yandex.Disk, for example). + # We don't downloading anything. + # Almost all videos with > Full HD + # don't provide direct MP4, they are + # using MPEG-DASH streaming. + # It is means there is two streams: + # one for video and one for audio. + # These two streams should be converted + # into one to get video with audio. + # It is really expensive for public server. + # Also, videos with high resolution have + # large size, which affects at download time + # by another service (Yandex.Disk, for example). + # Probably it can even break the download. + # So, passing Full HD (maxium) videos is a golden mean. + # `best` is fallback if there is no + # needed things (they almost always presented), + # and in that case final result can be a + # not that user expected. + "format": "[height <= 1080]/best", + "youtube_include_dash_manifest": False, + "logger": Logger(), + # disable downloading at all + "simulate": True, + "skip_download": True, + # Ignore YouTube playlists, because large + # playlists can take long time to parse + # (affects at server response time) + "extract_flat": "in_playlist", + "noplaylist": True +} +ydl = youtube_dl.YoutubeDL(options) diff --git a/src/blueprints/telegram_bot/webhook/commands/__init__.py b/src/blueprints/telegram_bot/webhook/commands/__init__.py index 6eb71fe..dfb8bba 100644 --- a/src/blueprints/telegram_bot/webhook/commands/__init__.py +++ b/src/blueprints/telegram_bot/webhook/commands/__init__.py @@ -1,14 +1,51 @@ -from .common.names import CommandsNames +""" +How to create handler for the dispatcher: +- you should provide one function. This function +will be called to handle an incoming Telegram message +- dispatcher handles any handler exceptions, so, if +any unexpected error occurs you can raise the exception. +Sure, you can raise your own exception for nice debug +- edit dispatcher module in order to register your handler +- handler should accept only `(*args, **kwargs)` arguments +- dispatcher passes in `**kwargs`: `route_source: RouteSource`, +`message_events: Set[str]` (`str` it is `DispatcherEvent` values), +`user_id: int`, `chat_id: int`, `message: TelegramMessage`. +These values can have `None` value, so, check for it before using. +For another values in `*args` and `**kwargs` see documentation of +functions from call stack of dispatcher function. + +How to create stateful chat: +- use `set_disposable_handler`, `subscribe_handler`, +`unsubcribe_handler`, `set/get/delete_user/user_chat/chat_data` +- if you want to provide Enum, then provide value of that enum, +not object directly +""" + + +from .upload import ( + handle_photo as upload_photo_handler, + handle_file as upload_file_handler, + handle_audio as upload_audio_handler, + handle_video as upload_video_handler, + handle_voice as upload_voice_handler, + handle_url as upload_url_handler, + handle_public_photo as public_upload_photo_handler, + handle_public_file as public_upload_file_handler, + handle_public_audio as public_upload_audio_handler, + handle_public_video as public_upload_video_handler, + handle_public_voice as public_upload_voice_handler, + handle_public_url as public_upload_url_handler, +) from .unknown import handle as unknown_handler from .help import handle as help_handler from .about import handle as about_handler from .settings import handle as settings_handler from .yd_auth import handle as yd_auth_handler from .yd_revoke import handle as yd_revoke_handler -from .upload import handle_photo as upload_photo_handler -from .upload import handle_file as upload_file_handler -from .upload import handle_audio as upload_audio_handler -from .upload import handle_video as upload_video_handler -from .upload import handle_voice as upload_voice_handler -from .upload import handle_url as upload_url_handler from .create_folder import handle as create_folder_handler +from .publish import handle as publish_handler +from .unpublish import handle as unpublish_handler +from .space_info import handle as space_info_handler +from .element_info import handle as element_info_handler +from .disk_info import handle as disk_info_handler +from .commands_list import handle as commands_list_handler diff --git a/src/blueprints/telegram_bot/webhook/commands/_common/__init__.py b/src/blueprints/telegram_bot/webhook/commands/_common/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/blueprints/telegram_bot/webhook/commands/_common/commands_content.py b/src/blueprints/telegram_bot/webhook/commands/_common/commands_content.py new file mode 100644 index 0000000..4d12a5f --- /dev/null +++ b/src/blueprints/telegram_bot/webhook/commands/_common/commands_content.py @@ -0,0 +1,185 @@ +# flake8: noqa +# Long lines are allowed here, but try to avoid them + + +from flask import current_app + +from src.blueprints.telegram_bot._common.command_names import CommandName + + +def to_code(text: str) -> str: + return f"{text}" + + +commands_html_content = ( + { + "name": "Yandex.Disk", + "commands": ( + { + "name": CommandName.UPLOAD_PHOTO.value, + "help": ( + "upload a photo. Original name will be " + "not saved, quality of photo will be decreased. " + "You can send photo without this command. " + f"Use {CommandName.PUBLIC_UPLOAD_PHOTO.value} " + "for public uploading" + ) + }, + { + "name": CommandName.PUBLIC_UPLOAD_PHOTO.value + }, + { + "name": CommandName.UPLOAD_FILE.value, + "help": ( + "upload a file. Original name will be saved. " + "For photos, original quality will be saved. " + "You can send file without this command. " + f"Use {CommandName.PUBLIC_UPLOAD_FILE.value} " + "for public uploading" + ) + }, + { + "name": CommandName.PUBLIC_UPLOAD_FILE.value + }, + { + "name": CommandName.UPLOAD_AUDIO.value, + "help": ( + "upload an audio. Original name will be saved, " + "original type may be changed. " + "You can send audio file without this command. " + f"Use {CommandName.PUBLIC_UPLOAD_AUDIO.value} " + "for public uploading" + ) + }, + { + "name": CommandName.PUBLIC_UPLOAD_AUDIO.value + }, + { + "name": CommandName.UPLOAD_VIDEO.value, + "help": ( + "upload a video. Original name will be saved, " + "original type may be changed. " + "You can send video file without this command. " + f"Use {CommandName.PUBLIC_UPLOAD_VIDEO.value} " + "for public uploading" + ) + }, + { + "name": CommandName.PUBLIC_UPLOAD_VIDEO.value + }, + { + "name": CommandName.UPLOAD_VOICE.value, + "help": ( + "upload a voice. " + "You can send voice file without this command. " + f"Use {CommandName.PUBLIC_UPLOAD_VOICE.value} " + "for public uploading" + ) + }, + { + "name": CommandName.PUBLIC_UPLOAD_VOICE.value + }, + { + "name": CommandName.UPLOAD_URL.value, + "help": ( + "upload a some resource using URL. " + "For direct URL's to file original name, " + "quality and size will be saved. " + "For URL's to some resource best name and " + "best possible quality will be selected. " + '"Resource" means anything: YouTube video, ' + "Twitch clip, music track, etc. " + "Not everything will work as you expect, " + "but some URL's will. " + 'See this ' + "for all supported sites. " + f"Use {CommandName.PUBLIC_UPLOAD_URL.value} " + "for public uploading" + ) + }, + { + "name": CommandName.PUBLIC_UPLOAD_URL.value + }, + { + "name": CommandName.PUBLISH.value, + "help": ( + "publish a file or folder that " + "already exists. Send full name of " + "element to publish with this command. " + f'Example: {to_code(f"Telegram Bot/files/photo.jpeg")}' + ) + }, + { + "name": CommandName.UNPUBLISH.value, + "help": ( + "unpublish a file or folder that " + "already exists. Send full name of " + "element to unpublish with this command. " + f'Example: {to_code(f"Telegram Bot/files/photo.jpeg")}' + ) + }, + { + "name": CommandName.CREATE_FOLDER.value, + "help": ( + "create a folder. Send folder name to " + "create with this command. Folder name " + "should starts from root, nested folders should be " + f'separated with "{to_code("/")}" character' + ) + }, + { + "name": CommandName.ELEMENT_INFO.value, + "help": ( + "get information about file or folder. " + "Send full path of element with this command" + ) + }, + { + "name": CommandName.SPACE_INFO.value, + "help": "get information about remaining space" + }, + { + "name": CommandName.DISK_INFO.value, + "help": "get information about your Yandex.Disk" + } + ) + }, + { + "name": "Yandex.Disk Access", + "commands": ( + { + "name": CommandName.YD_AUTH.value, + "help": "grant me access to your Yandex.Disk" + }, + { + "name": CommandName.YD_REVOKE.value, + "help": "revoke my access to your Yandex.Disk" + } + ) + }, + { + "name": "Settings", + "commands": ( + { + "name": CommandName.SETTINGS.value, + "help": "edit your settings" + }, + ) + }, + { + "name": "Information", + "commands": ( + { + "name": CommandName.ABOUT.value, + "help": "read about me" + }, + { + "name": CommandName.COMMANDS_LIST.value, + "help": ( + "see full list of available " + "commands without help message" + ) + } + ) + } +) diff --git a/src/blueprints/telegram_bot/webhook/commands/common/decorators.py b/src/blueprints/telegram_bot/webhook/commands/_common/decorators.py similarity index 89% rename from src/blueprints/telegram_bot/webhook/commands/common/decorators.py rename to src/blueprints/telegram_bot/webhook/commands/_common/decorators.py index a0916e8..a18fc3a 100644 --- a/src/blueprints/telegram_bot/webhook/commands/common/decorators.py +++ b/src/blueprints/telegram_bot/webhook/commands/_common/decorators.py @@ -2,8 +2,8 @@ from flask import g +from src.extensions import db from src.database import ( - db, User, UserQuery, Chat, @@ -12,9 +12,9 @@ from src.database.models import ( ChatType ) -from src.localization import SupportedLanguages +from src.localization import SupportedLanguage +from src.blueprints.telegram_bot._common.command_names import CommandName from .responses import cancel_command -from .names import CommandsNames def register_guest(func): @@ -37,7 +37,7 @@ def wrapper(*args, **kwargs): new_user = User( telegram_id=tg_user.id, is_bot=tg_user.is_bot, - language=SupportedLanguages.get(tg_user.language_code) + language=SupportedLanguage.get(tg_user.language_code) ) Chat( telegram_id=tg_chat.id, @@ -102,7 +102,7 @@ def wrapper(*args, **kwargs): (user.yandex_disk_token is None) or (not user.yandex_disk_token.have_access_token()) ): - return g.route_to(CommandsNames.YD_AUTH) + return g.direct_dispatch(CommandName.YD_AUTH)() return func(*args, **kwargs) diff --git a/src/blueprints/telegram_bot/webhook/commands/_common/responses.py b/src/blueprints/telegram_bot/webhook/commands/_common/responses.py new file mode 100644 index 0000000..9f98236 --- /dev/null +++ b/src/blueprints/telegram_bot/webhook/commands/_common/responses.py @@ -0,0 +1,194 @@ +from enum import IntEnum, unique + +from flask import current_app + +from src.api import telegram + + +@unique +class AbortReason(IntEnum): + """ + Reason for `abort_command`. + """ + UNKNOWN = 1 + NO_SUITABLE_DATA = 2 + EXCEED_FILE_SIZE_LIMIT = 3 + + +def abort_command( + chat_telegram_id: int, + reason: AbortReason, + edit_message: int = None, + reply_to_message: int = None +) -> None: + """ + Aborts command execution due to invalid message data. + + - don't confuse with `cancel_command()`. + - if `edit_message` Telegram ID specified, then + that message will be edited. + - if `reply_to_message` Telegram ID specified, then + that message will be used for reply message. + """ + texts = { + AbortReason.UNKNOWN: ( + "I can't handle this because something is wrong." + ), + AbortReason.NO_SUITABLE_DATA: ( + "I can't handle this because " + "you didn't send any suitable data " + "for that command." + ), + AbortReason.EXCEED_FILE_SIZE_LIMIT: ( + "I can't handle file of such a large size. " + "At the moment my limit is " + f"{current_app.config['TELEGRAM_API_MAX_FILE_SIZE'] / 1024 / 1024} MB." # noqa + ) + } + text = texts[reason] + + if (edit_message is not None): + telegram.edit_message_text( + chat_id=chat_telegram_id, + message_id=edit_message, + text=text + ) + elif (reply_to_message is not None): + telegram.send_message( + chat_id=chat_telegram_id, + reply_to_message_id=reply_to_message, + text=text + ) + else: + telegram.send_message( + chat_id=chat_telegram_id, + text=text + ) + + +def cancel_command( + chat_telegram_id: int, + edit_message: int = None, + reply_to_message: int = None +) -> None: + """ + Cancels command execution due to internal server error. + + - don't confuse with `abort_command()`. + - if `edit_message` Telegram ID specified, then + that message will be edited. + - if `reply_to_message` Telegram ID specified, then + that message will be used for reply message. + """ + text = ( + "At the moment i can't process this " + "because of my internal error. " + "Try later please." + ) + + if (edit_message is not None): + telegram.edit_message_text( + chat_id=chat_telegram_id, + message_id=edit_message, + text=text + ) + elif (reply_to_message is not None): + telegram.send_message( + chat_id=chat_telegram_id, + reply_to_message_id=reply_to_message, + text=text + ) + else: + telegram.send_message( + chat_id=chat_telegram_id, + text=text + ) + + +def request_private_chat(chat_telegram_id: int) -> None: + """ + Aborts command execution due to lack of private chat with user. + """ + telegram.send_message( + chat_id=chat_telegram_id, + text=( + "I need to send you your secret information, " + "but i don't know any private chat with you. " + "First, contact me through private chat (direct message). " + "After that repeat your request." + ) + ) + + +def send_yandex_disk_error( + chat_telegram_id: int, + error_text: str, + reply_to_message_id: int = None +) -> None: + """ + Sends a message that indicates that Yandex.Disk threw an error. + + :param error_text: + Text of error that will be printed. + Can be empty. + :param reply_to_message_id: + If specified, then sended message will be a reply message. + """ + kwargs = { + "chat_id": chat_telegram_id, + "parse_mode": "HTML", + "text": ( + "Yandex.Disk Error" + "\n\n" + f"{error_text or 'Unknown'}" + ) + } + + if reply_to_message_id is not None: + kwargs["reply_to_message_id"] = reply_to_message_id + + telegram.send_message(**kwargs) + + +def request_absolute_path(chat_telegram_id: int) -> None: + """ + Sends a message that asks a user to send an + absolute path (folder or file). + """ + telegram.send_message( + chat_id=chat_telegram_id, + parse_mode="HTML", + text=( + "Send a full path." + "\n\n" + "It should starts from root directory, " + "nested folders should be separated with " + '"/" character. ' + "In short, i expect an absolute path to the item." + "\n\n" + "Example: Telegram Bot/kittens and raccoons" + "\n" + "Example: /Telegram Bot/kittens and raccoons/musya.jpg" # noqa + ) + ) + + +def request_absolute_folder_name(chat_telegram_id: int) -> None: + """ + Sends a message that asks a user to send an + absolute path of folder. + """ + telegram.send_message( + chat_id=chat_telegram_id, + parse_mode="HTML", + text=( + "Send a folder name." + "\n\n" + "It should starts from root directory, " + "nested folders should be separated with " + '"/" character. ' + "In short, i expect a full path." + "\n\n" + "Example: Telegram Bot/kittens and raccoons" + ) + ) diff --git a/src/blueprints/telegram_bot/webhook/commands/_common/utils.py b/src/blueprints/telegram_bot/webhook/commands/_common/utils.py new file mode 100644 index 0000000..bec1d5b --- /dev/null +++ b/src/blueprints/telegram_bot/webhook/commands/_common/utils.py @@ -0,0 +1,252 @@ +from typing import Union +from collections import deque +import json + +from src.blueprints._common.utils import ( + convert_iso_datetime, + bytes_to_human_decimal +) +from src.blueprints.telegram_bot._common.telegram_interface import ( + Message as TelegramMessage +) +from src.blueprints.telegram_bot.webhook.dispatcher_events import ( + RouteSource +) + + +def extract_absolute_path( + message: TelegramMessage, + bot_command: str, + route_source: Union[RouteSource, None], +) -> str: + """ + Extracts absolute path from Telegram message. + It supports both: folders and files. + + :param message: + Incoming Telegram message. + :param bot_command: + Bot command which will be removed from message. + :param route_source: + It is dispatcher parameter, see it documentation. + You should always pass it, even if it is `None`. + Bot command will be deleted from start of a message + when it is equal to `DISPOSABLE_HANDLER`. + + :returns: + Extracted absolute path. Can be empty. + """ + path = message.get_text() + + # On "Disposable handler" route we expect pure text, + # in other cases we expect bot command as start of a message + if (route_source != RouteSource.DISPOSABLE_HANDLER): + path = path.replace( + bot_command, + "", + 1 + ).strip() + + return path + + +def create_element_info_html_text( + info: dict, + include_private_info: bool +) -> str: + """ + :param info: + - https://yandex.ru/dev/disk/api/reference/meta.html/ + - https://dev.yandex.net/disk-polygon/?lang=ru&tld=ru#!/v147disk47resources/GetResource # noqa + :param include_private_info: + Include private information in result. + "Private information" means information that + can compromise a user (for example, full path of element). + Use `True` when you want to display this text only to user, + use `False` when you can assume that user theoretically can + forward this text to some another user(s). + """ + text = deque() + + if "name" in info: + text.append( + f"Name: {info['name']}" + ) + + if "type" in info: + incoming_type = info["type"].lower() + value = "Unknown" + + if (incoming_type == "dir"): + value = "Folder" + elif (incoming_type == "file"): + if "media_type" in info: + value = info["media_type"] + else: + value = "File" + + if "mime_type" in info: + value = f"{value} ({info['mime_type']})" + + text.append( + f"Type: {value}" + ) + + if "size" in info: + bytes_count = info["size"] + value = f"{bytes_count:,} bytes" + + if (bytes_count >= 1000): + decimal = bytes_to_human_decimal(bytes_count) + value = f"{decimal} ({bytes_count:,} bytes)" + + text.append( + f"Size: {value}" + ) + + if ( + include_private_info and + ("created" in info) + ): + value = convert_iso_datetime(info["created"]) + text.append( + "Created: " + f"{value['date']} {value['time']} {value['timezone']}" + ) + + if ( + include_private_info and + ("modified" in info) + ): + value = convert_iso_datetime(info["modified"]) + text.append( + "Modified: " + f"{value['date']} {value['time']} {value['timezone']}" + ) + + if ( + include_private_info and + ("path" in info) + ): + text.append( + "Full path: " + f"{info['path']}" + ) + + if ( + include_private_info and + ("origin_path" in info) + ): + text.append( + "Origin path: " + f"{info['origin_path']}" + ) + + if ( + ("_embedded" in info) and + ("total" in info["_embedded"]) + ): + text.append( + f"Total elements: {info['_embedded']['total']}" + ) + + if "public_url" in info: + text.append( + f"Public URL: {info['public_url']}" + ) + + if ( + include_private_info and + ("views_count" in info) + ): + text.append( + f"Views: {info['views_count']}" + ) + + if ( + include_private_info and + ("owner" in info) + ): + data = info["owner"] + name = data.get("display_name") + login = data.get("login") + value = "?" + + if name: + value = name + + if ( + value and + login and + (value != login) + ): + value = f"{value} ({login})" + elif login: + value = login + + text.append( + f"Owner: {value}" + ) + + if ( + include_private_info and + ("share" in info) + ): + data = info["share"] + + if "is_owned" in data: + value = "No" + + if data["is_owned"]: + value = "Yes" + + text.append( + f"Shared access — Owner: {value}" + ) + + if "rights" in data: + value = data["rights"].lower() + + if (value == "rw"): + value = "Full access" + elif (value == "r"): + value = "Read" + elif (value == "w"): + value = "Write" + + text.append( + f"Shared access — Rights: {value}" + ) + + if "is_root" in data: + value = "No" + + if data["is_root"]: + value = "Yes" + + text.append( + f"Shared access — Root: {value}" + ) + + if ( + include_private_info and + ("exif" in info) and + info["exif"] + ): + exif = json.dumps(info["exif"], indent=4) + text.append( + "EXIF: " + f"{exif}" + ) + + if "sha256" in info: + text.append( + f"SHA-256: {info['sha256']}" + ) + + if "md5" in info: + text.append( + f"MD5: {info['md5']}" + ) + + return "\n".join(text) diff --git a/src/blueprints/telegram_bot/webhook/commands/about.py b/src/blueprints/telegram_bot/webhook/commands/about.py index ab7e8b4..e904cd0 100644 --- a/src/blueprints/telegram_bot/webhook/commands/about.py +++ b/src/blueprints/telegram_bot/webhook/commands/about.py @@ -1,15 +1,18 @@ from flask import g, current_app from src.api import telegram -from src.blueprints.utils import absolute_url_for +from src.blueprints._common.utils import absolute_url_for -def handle(): +def handle(*args, **kwargs): """ Handles `/about` command. """ telegram.send_message( - chat_id=g.telegram_chat.id, + chat_id=kwargs.get( + "chat_id", + g.telegram_chat.id + ), disable_web_page_preview=True, text=( "I'm free and open-source bot that allows " diff --git a/src/blueprints/telegram_bot/webhook/commands/commands_list.py b/src/blueprints/telegram_bot/webhook/commands/commands_list.py new file mode 100644 index 0000000..473ed9b --- /dev/null +++ b/src/blueprints/telegram_bot/webhook/commands/commands_list.py @@ -0,0 +1,39 @@ +from collections import deque + +from flask import g + +from src.api import telegram +from ._common.commands_content import commands_html_content + + +def handle(*args, **kwargs): + """ + Handles `/commands` command. + """ + chat_id = kwargs.get( + "chat_id", + g.telegram_chat.id + ) + text = create_commands_list_html_text() + + telegram.send_message( + chat_id=chat_id, + text=text, + parse_mode="HTML" + ) + + +def create_commands_list_html_text() -> str: + text = deque() + + for group in commands_html_content: + text.append( + f"{group['name']}" + ) + + for command in group["commands"]: + text.append(command["name"]) + + text.append("") + + return "\n".join(text) diff --git a/src/blueprints/telegram_bot/webhook/commands/common/names.py b/src/blueprints/telegram_bot/webhook/commands/common/names.py deleted file mode 100644 index ecf0500..0000000 --- a/src/blueprints/telegram_bot/webhook/commands/common/names.py +++ /dev/null @@ -1,21 +0,0 @@ -from enum import Enum, unique - - -@unique -class CommandsNames(Enum): - """ - Commands supported by bot. - """ - START = "/start" - HELP = "/help" - ABOUT = "/about" - SETTINGS = "/settings" - YD_AUTH = "/grant_access" - YD_REVOKE = "/revoke_access" - UPLOAD_PHOTO = "/upload_photo" - UPLOAD_FILE = "/upload_file" - UPLOAD_AUDIO = "/upload_audio" - UPLOAD_VIDEO = "/upload_video" - UPLOAD_VOICE = "/upload_voice" - UPLOAD_URL = "/upload_url" - CREATE_FOLDER = "/create_folder" diff --git a/src/blueprints/telegram_bot/webhook/commands/common/responses.py b/src/blueprints/telegram_bot/webhook/commands/common/responses.py deleted file mode 100644 index 7dffb69..0000000 --- a/src/blueprints/telegram_bot/webhook/commands/common/responses.py +++ /dev/null @@ -1,94 +0,0 @@ -from src.api import telegram - - -def abort_command( - chat_telegram_id: int, - edit_message: int = None, - reply_to_message: int = None -) -> None: - """ - Aborts command execution due to invalid message data. - - - don't confuse with `cancel_command()`. - - if `edit_message` Telegram ID specified, then - that message will be edited. - - if `reply_to_message` Telegram ID specified, then - that message will be used for reply message. - """ - text = ( - "I can't handle this because " - "you didn't send any suitable data " - "for that command." - ) - - if (edit_message is not None): - telegram.edit_message_text( - chat_id=chat_telegram_id, - message_id=edit_message, - text=text - ) - elif (reply_to_message is not None): - telegram.send_message( - chat_id=chat_telegram_id, - reply_to_message_id=reply_to_message, - text=text - ) - else: - telegram.send_message( - chat_id=chat_telegram_id, - text=text - ) - - -def cancel_command( - chat_telegram_id: int, - edit_message: int = None, - reply_to_message: int = None -) -> None: - """ - Cancels command execution due to internal server error. - - - don't confuse with `abort_command()`. - - if `edit_message` Telegram ID specified, then - that message will be edited. - - if `reply_to_message` Telegram ID specified, then - that message will be used for reply message. - """ - text = ( - "At the moment i can't process this " - "because of my internal error. " - "Try later please." - ) - - if (edit_message is not None): - telegram.edit_message_text( - chat_id=chat_telegram_id, - message_id=edit_message, - text=text - ) - elif (reply_to_message is not None): - telegram.send_message( - chat_id=chat_telegram_id, - reply_to_message_id=reply_to_message, - text=text - ) - else: - telegram.send_message( - chat_id=chat_telegram_id, - text=text - ) - - -def request_private_chat(chat_telegram_id: int) -> None: - """ - Aborts command execution due to lack of private chat with user. - """ - telegram.send_message( - chat_id=chat_telegram_id, - text=( - "I need to send you your secret information, " - "but i don't know any private chat with you. " - "First, contact me through private chat (direct message). " - "After that repeat your request." - ) - ) diff --git a/src/blueprints/telegram_bot/webhook/commands/common/yandex_api.py b/src/blueprints/telegram_bot/webhook/commands/common/yandex_api.py deleted file mode 100644 index edeba79..0000000 --- a/src/blueprints/telegram_bot/webhook/commands/common/yandex_api.py +++ /dev/null @@ -1,244 +0,0 @@ -from time import sleep -from typing import Generator - -from flask import current_app - -from src.api import yandex - - -class YandexAPIRequestError(Exception): - """ - Unknown error occurred during Yandex.Disk API request - (not necessarily from Yandex). - """ - pass - - -class YandexAPIError(Exception): - """ - Error response from Yandex.Disk API. - - - may contain human-readable error message. - """ - pass - - -class YandexAPICreateFolderError(Exception): - """ - Unable to create folder on Yandex.Disk. - - - may contain human-readable error message. - """ - pass - - -class YandexAPIUploadFileError(Exception): - """ - Unable to upload folder on Yandex.Disk. - - - may contain human-readable error message. - """ - pass - - -class YandexAPIExceededNumberOfStatusChecksError(Exception): - """ - There was too much attempts to check status - of Yandex.Disk operation. Yandex didn't give any - acceptable status, i.e. operation still in progress - (most probably) and function cannot do any status - checks because of check limit. So, operation status - becoming unknown. - """ - pass - - -def create_folder( - user_access_token: str, - folder_name: str -) -> int: - """ - Creates folder using Yandex API. - - Yandex not able to create folder if some of - middle folders not exists. This method will try to create - each folder one by one, and ignore safe errors (if - already exists, for example) from all folder names - except last one. - - :returns: Last (for last folder name) HTTP Status code. - - :raises: YandexAPIRequestError - :raises: YandexAPICreateFolderError - """ - folders = [x for x in folder_name.split("/") if x] - folder_path = "" - last_status_code = 201 # root always created - allowed_errors = [409] - - for folder in folders: - result = None - folder_path = f"{folder_path}/{folder}" - - try: - result = yandex.create_folder( - user_access_token, - path=folder_path - ) - except Exception as error: - raise YandexAPIRequestError(error) - - response = result["content"] - last_status_code = result["status_code"] - - if ( - (last_status_code == 201) or - (last_status_code in allowed_errors) or - (not is_error_yandex_response(response)) - ): - continue - - raise YandexAPICreateFolderError( - create_yandex_error_text(result) - ) - - return last_status_code - - -def upload_file_with_url( - user_access_token: str, - folder_path: str, - file_name: str, - download_url: str -) -> Generator[str, None, None]: - """ - Uploads a file to Yandex.Disk using file download url. - - - before uploading creates a folder. - - after uploading will monitor operation status according - to app configuration. Because it is synchronous, it may - take significant time to end this function! - - :yields: status of operation in Yandex format (for example, - `"in progress"`). It will yields with some interval (according - to app configuration). Order is an order in which Yandex - sends the operation status. - - :raises: YandexAPIRequestError - :raises: YandexAPICreateFolderError - :raises: YandexAPIUploadFileError - :raises: YandexAPIExceededNumberOfStatusChecksError - """ - create_folder( - user_access_token=user_access_token, - folder_name=folder_path - ) - - folders = [x for x in folder_path.split("/") if x] - full_path = "/".join(folders + [file_name]) - response = None - - try: - response = yandex.upload_file_with_url( - user_access_token, - url=download_url, - path=full_path - ) - except Exception as error: - raise YandexAPIRequestError(error) - - operation_status_link = response["content"] - is_error = is_error_yandex_response(operation_status_link) - - if (is_error): - raise YandexAPIUploadFileError( - create_yandex_error_text( - operation_status_link - ) - ) - - operation_status = {} - attempt = 0 - max_attempts = current_app.config[ - "YANDEX_DISK_API_CHECK_OPERATION_STATUS_MAX_ATTEMPTS" - ] - interval = current_app.config[ - "YANDEX_DISK_API_CHECK_OPERATION_STATUS_INTERVAL" - ] - - while not ( - is_error_yandex_response(operation_status) or - yandex_operation_is_completed(operation_status) or - attempt >= max_attempts - ): - sleep(interval) - - try: - response = yandex.make_link_request( - data=operation_status_link, - user_token=user_access_token - ) - except Exception as error: - raise YandexAPIRequestError(error) - - operation_status = response["content"] - - if ("status" in operation_status): - yield operation_status["status"] - - attempt += 1 - - is_error = is_error_yandex_response(operation_status) - is_completed = yandex_operation_is_completed(operation_status) - - if (is_error): - raise YandexAPIUploadFileError( - create_yandex_error_text( - operation_status - ) - ) - elif ( - (attempt >= max_attempts) and - not is_completed - ): - raise YandexAPIExceededNumberOfStatusChecksError() - - -def is_error_yandex_response(data: dict) -> bool: - """ - :returns: Yandex response contains error or not. - """ - return ("error" in data) - - -def create_yandex_error_text(data: dict) -> str: - """ - :returns: Human error message from Yandex error response. - """ - error_name = data["error"] - error_description = ( - data.get("message") or - data.get("description") or - "?" - ) - - return ( - "Yandex.Disk Error: " - f"{error_name} ({error_description})" - ) - - -def yandex_operation_is_completed(data: dict) -> bool: - """ - :returns: Yandex response contains completed - operation status or not. - """ - return ( - ("status" in data) and - ( - (data["status"] == "success") or - # Yandex documentation is different in some places - (data["status"] == "failure") or - (data["status"] == "failed") - ) - ) diff --git a/src/blueprints/telegram_bot/webhook/commands/create_folder.py b/src/blueprints/telegram_bot/webhook/commands/create_folder.py index 69bf7cd..fafea32 100644 --- a/src/blueprints/telegram_bot/webhook/commands/create_folder.py +++ b/src/blueprints/telegram_bot/webhook/commands/create_folder.py @@ -1,40 +1,73 @@ -from flask import g +from flask import g, current_app from src.api import telegram -from .common.responses import ( +from src.blueprints.telegram_bot._common.yandex_disk import ( + create_folder, + YandexAPICreateFolderError, + YandexAPIRequestError +) +from src.blueprints.telegram_bot._common.command_names import ( + CommandName +) +from src.blueprints.telegram_bot._common.stateful_chat import ( + stateful_chat_is_enabled, + set_disposable_handler +) +from src.blueprints.telegram_bot.webhook.dispatcher_events import ( + DispatcherEvent +) +from ._common.responses import ( cancel_command, - abort_command + send_yandex_disk_error, + request_absolute_folder_name ) -from .common.decorators import ( +from ._common.decorators import ( yd_access_token_required, get_db_data ) -from .common.yandex_api import ( - create_folder, - YandexAPICreateFolderError, - YandexAPIRequestError -) -from . import CommandsNames +from ._common.utils import extract_absolute_path @yd_access_token_required @get_db_data -def handle(): - """ - Handles `/create_folder` command. - """ - message = g.telegram_message - user = g.db_user - chat = g.db_chat - message_text = message.get_text() - folder_name = message_text.replace( - CommandsNames.CREATE_FOLDER.value, - "" - ).strip() +def handle(*args, **kwargs): + message = kwargs.get( + "message", + g.telegram_message + ) + user_id = kwargs.get( + "user_id", + g.telegram_user.id + ) + chat_id = kwargs.get( + "chat_id", + g.telegram_chat.id + ) + folder_name = extract_absolute_path( + message, + CommandName.CREATE_FOLDER.value, + kwargs.get("route_source") + ) - if not (folder_name): - return abort_command(chat.telegram_id) + if not folder_name: + if stateful_chat_is_enabled(): + set_disposable_handler( + user_id, + chat_id, + CommandName.CREATE_FOLDER.value, + [ + DispatcherEvent.PLAIN_TEXT.value, + DispatcherEvent.BOT_COMMAND.value, + DispatcherEvent.EMAIL.value, + DispatcherEvent.HASHTAG.value, + DispatcherEvent.URL.value + ], + current_app.config["RUNTIME_DISPOSABLE_HANDLER_EXPIRE"] + ) + return request_absolute_folder_name(chat_id) + + user = g.db_user access_token = user.yandex_disk_token.get_access_token() last_status_code = None @@ -44,19 +77,13 @@ def handle(): folder_name=folder_name ) except YandexAPIRequestError as error: - print(error) - return cancel_command(chat.telegram_id) + cancel_command(chat_id) + raise error except YandexAPICreateFolderError as error: - error_text = ( - str(error) or - "Unknown Yandex.Disk error" - ) - - telegram.send_message( - chat_id=chat.telegram_id, - text=error_text - ) + send_yandex_disk_error(chat_id, str(error)) + # it is expected error and should be + # logged only to user return text = None @@ -66,9 +93,9 @@ def handle(): elif (last_status_code == 409): text = "Already exists" else: - text = f"Unknown status code: {last_status_code}" + text = f"Unknown operation status: {last_status_code}" telegram.send_message( - chat_id=chat.telegram_id, + chat_id=chat_id, text=text ) diff --git a/src/blueprints/telegram_bot/webhook/commands/disk_info.py b/src/blueprints/telegram_bot/webhook/commands/disk_info.py new file mode 100644 index 0000000..094d863 --- /dev/null +++ b/src/blueprints/telegram_bot/webhook/commands/disk_info.py @@ -0,0 +1,139 @@ +from collections import deque + +from flask import g, current_app + +from src.api import telegram +from src.blueprints._common.utils import bytes_to_human_binary +from src.blueprints.telegram_bot._common.yandex_disk import ( + get_disk_info, + YandexAPIRequestError +) +from ._common.responses import cancel_command +from ._common.decorators import ( + yd_access_token_required, + get_db_data +) + + +@yd_access_token_required +@get_db_data +def handle(*args, **kwargs): + """ + Handles `/disk_info` command. + """ + chat_id = kwargs.get( + "chat_id", + g.telegram_chat.id + ) + user = g.db_user + access_token = user.yandex_disk_token.get_access_token() + info = None + + try: + info = get_disk_info(access_token) + except YandexAPIRequestError as error: + cancel_command(chat_id) + raise error + + text = create_disk_info_html_text(info) + + telegram.send_message( + chat_id=chat_id, + text=text, + parse_mode="HTML", + disable_web_page_preview=True + ) + + +def create_disk_info_html_text(info: dict) -> str: + """ + :param info: + https://dev.yandex.net/disk-polygon/?lang=ru&tld=ru#!/v147disk/GetDisk + """ + text = deque() + + if "user" in info: + data = info["user"] + + if "display_name" in data: + text.append( + f"User — Name: {data['display_name']}" + ) + + if "login" in data: + text.append( + f"User — Login: {data['login']}" + ) + + if "country" in data: + text.append( + f"User — Country: {data['country']}" + ) + + if "is_paid" in info: + value = "?" + + if info["is_paid"]: + value = "Yes" + else: + value = "No" + + text.append( + f"Paid: {value}" + ) + + if "total_space" in info: + value = bytes_to_string(info["total_space"]) + + text.append( + f"Total space: {value}" + ) + + if "used_space" in info: + value = bytes_to_string(info["used_space"]) + + text.append( + f"Used space: {value}" + ) + + if "trash_size" in info: + value = bytes_to_string(info["trash_size"]) + + text.append( + f"Trash size: {value}" + ) + + if ( + ("total_space" in info) and + ("used_space" in info) and + ("trash_size" in info) + ): + bytes_count = ( + info["total_space"] - + info["used_space"] - + info["trash_size"] + ) + value = bytes_to_string(bytes_count) + + text.append( + f"Free space: {value}" + ) + + if "max_file_size" in info: + value = bytes_to_string(info["max_file_size"]) + + text.append( + f"Maximum file size: {value}" + ) + + return "\n".join(text) + + +def bytes_to_string(bytes_count: int) -> str: + value = f"{bytes_count:,} bytes" + + if (bytes_count >= 1000): + decimal = bytes_to_human_binary(bytes_count) + value = f"{decimal} ({bytes_count:,} bytes)" + + return value diff --git a/src/blueprints/telegram_bot/webhook/commands/element_info.py b/src/blueprints/telegram_bot/webhook/commands/element_info.py new file mode 100644 index 0000000..8d95317 --- /dev/null +++ b/src/blueprints/telegram_bot/webhook/commands/element_info.py @@ -0,0 +1,185 @@ +from flask import g, current_app + +from src.extensions import task_queue +from src.api import telegram +from src.api.yandex import make_photo_preview_request +from src.blueprints.telegram_bot._common.yandex_disk import ( + get_element_info, + YandexAPIGetElementInfoError, + YandexAPIRequestError +) +from src.blueprints.telegram_bot._common.stateful_chat import ( + stateful_chat_is_enabled, + set_disposable_handler +) +from src.blueprints.telegram_bot._common.command_names import ( + CommandName +) +from src.blueprints.telegram_bot.webhook.dispatcher_events import ( + DispatcherEvent +) +from ._common.responses import ( + cancel_command, + request_absolute_path, + send_yandex_disk_error +) +from ._common.decorators import ( + yd_access_token_required, + get_db_data +) +from ._common.utils import ( + extract_absolute_path, + create_element_info_html_text +) + + +@yd_access_token_required +@get_db_data +def handle(*args, **kwargs): + """ + Handles `/element_info` command. + """ + message = kwargs.get( + "message", + g.telegram_message + ) + user_id = kwargs.get( + "user_id", + g.telegram_user.id + ) + chat_id = kwargs.get( + "chat_id", + g.telegram_chat.id + ) + path = extract_absolute_path( + message, + CommandName.ELEMENT_INFO.value, + kwargs.get("route_source") + ) + + if not path: + if stateful_chat_is_enabled(): + set_disposable_handler( + user_id, + chat_id, + CommandName.ELEMENT_INFO.value, + [ + DispatcherEvent.PLAIN_TEXT.value, + DispatcherEvent.BOT_COMMAND.value, + DispatcherEvent.EMAIL.value, + DispatcherEvent.HASHTAG.value, + DispatcherEvent.URL.value + ], + current_app.config["RUNTIME_DISPOSABLE_HANDLER_EXPIRE"] + ) + + return request_absolute_path(chat_id) + + user = g.db_user + access_token = user.yandex_disk_token.get_access_token() + info = None + + try: + info = get_element_info( + access_token, + path, + get_public_info=True + ) + except YandexAPIRequestError as error: + cancel_command(chat_id) + raise error + except YandexAPIGetElementInfoError as error: + send_yandex_disk_error(chat_id, str(error)) + + # it is expected error and should be + # logged only to user + return + + text = create_element_info_html_text(info, True) + params = { + "chat_id": chat_id, + "text": text, + "parse_mode": "HTML", + "disable_web_page_preview": True + } + download_url = info.get("file") + + if download_url: + params["reply_markup"] = { + "inline_keyboard": [[ + { + "text": "Download", + "url": download_url + } + ]] + } + + # We will send message without preview, + # because it can take a while to download + # preview file and send it. We will + # send it later if it is available. + telegram.send_message(**params) + + preview_url = info.get("preview") + + if preview_url: + filename = info.get("name", "preview.jpg") + arguments = ( + preview_url, + filename, + access_token, + chat_id + ) + + if task_queue.is_enabled: + job_timeout = current_app.config[ + "RUNTIME_ELEMENT_INFO_WORKER_JOB_TIMEOUT" + ] + ttl = current_app.config[ + "RUNTIME_ELEMENT_INFO_WORKER_TTL" + ] + + task_queue.enqueue( + send_preview, + args=arguments, + description=CommandName.ELEMENT_INFO.value, + job_timeout=job_timeout, + ttl=ttl, + result_ttl=0, + failure_ttl=0 + ) + else: + # NOTE: current thread will + # be blocked for a while + send_preview(*arguments) + + +def send_preview( + preview_url: str, + filename: str, + user_access_token: str, + chat_id: int +): + """ + Downloads preview from Yandex.Disk and sends it to user. + + - requires user Yandex.Disk access token to + download preview file. + """ + result = make_photo_preview_request( + preview_url, + user_access_token + ) + + if result["ok"]: + data = result["content"] + + telegram.send_photo( + chat_id=chat_id, + photo=( + filename, + data, + "image/jpeg" + ), + disable_notification=True + ) diff --git a/src/blueprints/telegram_bot/webhook/commands/help.py b/src/blueprints/telegram_bot/webhook/commands/help.py index 79548db..0f34435 100644 --- a/src/blueprints/telegram_bot/webhook/commands/help.py +++ b/src/blueprints/telegram_bot/webhook/commands/help.py @@ -1,76 +1,81 @@ -# flake8: noqa +from collections import deque from flask import g, current_app from src.api import telegram -from . import CommandsNames +from ._common.commands_content import ( + to_code, + commands_html_content +) -def handle(): +def handle(*args, **kwargs): """ Handles `/help` command. """ + chat_id = kwargs.get( + "chat_id", + g.telegram_chat.id + ) + text = create_help_html_text() + telegram.send_message( + chat_id=chat_id, + parse_mode="HTML", + text=text, + disable_web_page_preview=True + ) + + +def create_help_html_text() -> str: yd_upload_default_folder = current_app.config[ "YANDEX_DISK_API_DEFAULT_UPLOAD_FOLDER" ] + file_size_limit_in_mb = int(current_app.config[ + "TELEGRAM_API_MAX_FILE_SIZE" + ] / 1024 / 1024) + bullet_char = "•" + text = deque() - text = ( - "You can control me by sending these commands:" - "\n\n" - "Yandex.Disk" - "\n" - f'For uploading "{to_code(yd_upload_default_folder)}" folder is used by default.' - "\n" - f"{CommandsNames.UPLOAD_PHOTO.value} — upload a photo. " - "Original name will be not saved, quality of photo will be decreased. " - "You can send photo without this command." - "\n" - f"{CommandsNames.UPLOAD_FILE.value} — upload a file. " - "Original name will be saved. " - "For photos, original quality will be saved. " - "You can send file without this command." - "\n" - f"{CommandsNames.UPLOAD_AUDIO.value} — upload an audio. " - "Original name will be saved, original type may be changed. " - "You can send audio file without this command." - "\n" - f"{CommandsNames.UPLOAD_VIDEO.value} — upload a video. " - "Original name will be not saved, original type may be changed. " - "You can send video file without this command." - "\n" - f"{CommandsNames.UPLOAD_VOICE.value} — upload a voice. " - "You can send voice file without this command." - "\n" - f"{CommandsNames.UPLOAD_URL.value} — upload a file using direct URL. " - "Original name will be saved. " - "You can send direct URL to a file without this command." - "\n" - f"{CommandsNames.CREATE_FOLDER.value}— create a folder. " - "Send folder name to create with this command. " - "Folder name should starts from root, " - f'nested folders should be separated with "{to_code("/")}" character.' + text.append( + "You can interact with " + 'Yandex.Disk ' + "by using me. To control me send following commands." "\n\n" - "Yandex.Disk Access" + "Note:" "\n" - f"{CommandsNames.YD_AUTH.value} — grant me access to your Yandex.Disk" + f"{bullet_char} for uploading " + f'"{to_code(yd_upload_default_folder)}" ' + "folder is used by default," "\n" - f"{CommandsNames.YD_REVOKE.value} — revoke my access to your Yandex.Disk" - "\n\n" - "Settings" - "\n" - f"{CommandsNames.SETTINGS.value} — edit your settings" - "\n\n" - "Information" + f"{bullet_char} maximum size of every upload " + f"(except URL) is {file_size_limit_in_mb} MB." "\n" - f"{CommandsNames.ABOUT.value} — read about me" ) - telegram.send_message( - chat_id=g.telegram_chat.id, - parse_mode="HTML", - text=text - ) + for group in commands_html_content: + group_name = group["name"] + commands = group["commands"] + group_added = False + + text.append( + f"{group_name}" + ) + + for command in commands: + command_name = command["name"] + help_message = command.get("help") + + if help_message: + text.append( + f"{bullet_char} {command_name} — {help_message}." + ) + group_added = True + if group_added: + # extra line + text.append("") + else: + # we don't want empty group name + text.pop() -def to_code(text: str) -> str: - return f"{text}" + return "\n".join(text) diff --git a/src/blueprints/telegram_bot/webhook/commands/publish.py b/src/blueprints/telegram_bot/webhook/commands/publish.py new file mode 100644 index 0000000..a237d3d --- /dev/null +++ b/src/blueprints/telegram_bot/webhook/commands/publish.py @@ -0,0 +1,114 @@ +from flask import g, current_app + +from src.api import telegram +from src.blueprints.telegram_bot._common.yandex_disk import ( + publish_item, + get_element_info, + YandexAPIGetElementInfoError, + YandexAPIPublishItemError, + YandexAPIRequestError +) +from src.blueprints.telegram_bot._common.stateful_chat import ( + stateful_chat_is_enabled, + set_disposable_handler +) +from src.blueprints.telegram_bot._common.command_names import ( + CommandName +) +from src.blueprints.telegram_bot.webhook.dispatcher_events import ( + DispatcherEvent +) +from ._common.responses import ( + cancel_command, + request_absolute_path, + send_yandex_disk_error +) +from ._common.decorators import ( + yd_access_token_required, + get_db_data +) +from ._common.utils import ( + extract_absolute_path, + create_element_info_html_text +) + + +@yd_access_token_required +@get_db_data +def handle(*args, **kwargs): + """ + Handles `/publish` command. + """ + message = kwargs.get( + "message", + g.telegram_message + ) + user_id = kwargs.get( + "user_id", + g.telegram_user.id + ) + chat_id = kwargs.get( + "chat_id", + g.telegram_chat.id + ) + path = extract_absolute_path( + message, + CommandName.PUBLISH.value, + kwargs.get("route_source") + ) + + if not path: + if stateful_chat_is_enabled(): + set_disposable_handler( + user_id, + chat_id, + CommandName.PUBLISH.value, + [ + DispatcherEvent.PLAIN_TEXT.value, + DispatcherEvent.BOT_COMMAND.value, + DispatcherEvent.EMAIL.value, + DispatcherEvent.HASHTAG.value, + DispatcherEvent.URL.value + ], + current_app.config["RUNTIME_DISPOSABLE_HANDLER_EXPIRE"] + ) + + return request_absolute_path(chat_id) + + user = g.db_user + access_token = user.yandex_disk_token.get_access_token() + + try: + publish_item(access_token, path) + except YandexAPIRequestError as error: + cancel_command(chat_id) + raise error + except YandexAPIPublishItemError as error: + send_yandex_disk_error(chat_id, str(error)) + + # it is expected error and should be + # logged only to user + return + + info = None + + try: + info = get_element_info(access_token, path) + except YandexAPIRequestError as error: + cancel_command(chat_id) + raise error + except YandexAPIGetElementInfoError as error: + send_yandex_disk_error(chat_id, str(error)) + + # it is expected error and should be + # logged only to user + return + + text = create_element_info_html_text(info, False) + + telegram.send_message( + chat_id=chat_id, + text=text, + parse_mode="HTML", + disable_web_page_preview=True + ) diff --git a/src/blueprints/telegram_bot/webhook/commands/settings.py b/src/blueprints/telegram_bot/webhook/commands/settings.py index a9d0f66..b4c6a9f 100644 --- a/src/blueprints/telegram_bot/webhook/commands/settings.py +++ b/src/blueprints/telegram_bot/webhook/commands/settings.py @@ -1,34 +1,39 @@ from flask import g from src.api import telegram -from .common.decorators import ( +from src.blueprints.telegram_bot._common.yandex_oauth import ( + YandexOAuthClient +) +from ._common.decorators import ( register_guest, get_db_data ) -from .common.responses import ( +from ._common.responses import ( request_private_chat ) @register_guest @get_db_data -def handle(): +def handle(*args, **kwargs): """ Handles `/settings` command. """ - user = g.db_user - incoming_chat = g.db_chat private_chat = g.db_private_chat - if (private_chat is None): - return request_private_chat(incoming_chat.telegram_id) + if private_chat is None: + incoming_chat_id = kwargs.get( + "chat_id", + g.db_chat.telegram_id + ) + return request_private_chat(incoming_chat_id) + + user = g.db_user + yo_client = YandexOAuthClient() yd_access = False - if ( - (user.yandex_disk_token) and - (user.yandex_disk_token.have_access_token()) - ): + if yo_client.have_valid_access_token(user): yd_access = True text = ( diff --git a/src/blueprints/telegram_bot/webhook/commands/space_info.py b/src/blueprints/telegram_bot/webhook/commands/space_info.py new file mode 100644 index 0000000..26678ef --- /dev/null +++ b/src/blueprints/telegram_bot/webhook/commands/space_info.py @@ -0,0 +1,249 @@ +from string import ascii_letters, digits +from datetime import datetime, timezone + +from flask import g, current_app +from plotly.graph_objects import Pie, Figure +from plotly.express import colors +from plotly.io import to_image + +from src.extensions import task_queue +from src.api import telegram +from src.blueprints._common.utils import get_current_iso_datetime +from src.blueprints.telegram_bot._common.yandex_disk import ( + get_disk_info, + YandexAPIRequestError +) +from src.blueprints.telegram_bot._common.command_names import ( + CommandName +) +from ._common.responses import cancel_command +from ._common.decorators import ( + yd_access_token_required, + get_db_data +) + + +@yd_access_token_required +@get_db_data +def handle(*args, **kwargs): + """ + Handles `/space_info` command. + """ + user = g.db_user + chat_id = kwargs.get( + "chat_id", + g.telegram_chat.id + ) + access_token = user.yandex_disk_token.get_access_token() + disk_info = None + + # If all task queue workers are busy, + # it can take a long time before they + # execute `send_photo()` function. + # We will indicate to user that everything + # is fine and result will be sent soon + sended_message = telegram.send_message( + chat_id=chat_id, + text="Generating..." + ) + sended_message_id = sended_message["content"]["message_id"] + + try: + disk_info = get_disk_info(access_token) + except YandexAPIRequestError as error: + cancel_command( + chat_telegram_id=chat_id, + edit_message=sended_message_id + ) + + raise error + + current_utc_date = get_current_utc_datetime() + current_iso_date = get_current_iso_datetime() + jpeg_image = create_space_chart( + total_space=disk_info["total_space"], + used_space=disk_info["used_space"], + trash_size=disk_info["trash_size"], + caption=current_utc_date + ) + filename = f"{to_filename(current_iso_date)}.jpg" + file_caption = f"Yandex.Disk space at {current_utc_date}" + arguments = ( + jpeg_image, + filename, + file_caption, + chat_id, + sended_message_id + ) + + if task_queue.is_enabled: + job_timeout = current_app.config[ + "RUNTIME_SPACE_INFO_WORKER_TIMEOUT" + ] + + task_queue.enqueue( + send_photo, + args=arguments, + description=CommandName.SPACE_INFO.value, + job_timeout=job_timeout, + result_ttl=0, + failure_ttl=0 + ) + else: + send_photo(*arguments) + + +def create_space_chart( + total_space: int, + used_space: int, + trash_size: int, + caption: str = None +) -> bytes: + """ + Creates Yandex.Disk space chart. + + - all sizes (total, used, trash) should be + specified in binary bytes. They will be + converted to binary gigabytes. + + :returns: JPEG image as bytes. + """ + free_space = b_to_gb(total_space - used_space - trash_size) + total_space = b_to_gb(total_space) + used_space = b_to_gb(used_space) + trash_size = b_to_gb(trash_size) + + chart = Pie( + labels=[ + "Used Space", + "Trash Size", + "Free Space" + ], + values=[ + used_space, + trash_size, + free_space + ], + text=[ + "Used", + "Trash", + "Free" + ], + marker={ + "colors": [ + colors.sequential.Rainbow[3], + colors.sequential.Rainbow[8], + colors.sequential.Rainbow[5] + ], + "line": { + "width": 0.2 + } + }, + sort=False, + direction="clockwise", + texttemplate=( + "%{text}
" + "%{value:.2f} GB
" + "%{percent}" + ), + textposition="outside", + hole=0.5 + ) + figure = Figure( + data=chart, + layout={ + "title": { + "text": caption, + "font": { + "size": 20 + } + }, + "annotations": [ + { + "align": "center", + "showarrow": False, + "text": ( + "Total
" + f"{total_space:.2f} GB
" + "100%" + ) + } + ], + "width": 1000, + "height": 800, + "font": { + "size": 27 + }, + "margin": { + "t": 140, + "b": 40, + "r": 230, + "l": 165 + } + } + ) + + return to_image(figure, format="jpeg") + + +def b_to_gb(value: int) -> int: + """ + Converts binary bytes to binary gigabytes. + """ + return (value / 1024 / 1024 / 1024) + + +def get_current_utc_datetime() -> str: + """ + :returns: Current date as string representation. + """ + now = datetime.now(timezone.utc) + + return now.strftime("%d.%m.%Y %H:%M %Z") + + +def to_filename(value: str) -> str: + """ + :returns: Valid filename. + """ + valid_chars = f"-_.{ascii_letters}{digits}" + filename = value.lower() + filename = ( + filename + .replace(" ", "_") + .replace(":", "_") + ) + filename = "".join(x for x in filename if x in valid_chars) + + return filename + + +def send_photo( + content: bytes, + filename: str, + file_caption: str, + chat_id: int, + sended_message_id: int +): + """ + Sends photo to user. + """ + telegram.send_photo( + chat_id=chat_id, + photo=( + filename, + content, + "image/jpeg" + ), + caption=file_caption + ) + + try: + telegram.delete_message( + chat_id=chat_id, + message_id=sended_message_id + ) + except Exception: + # we can safely ignore if we can't delete + # sended message. Anyway we will send new one + pass diff --git a/src/blueprints/telegram_bot/webhook/commands/unknown.py b/src/blueprints/telegram_bot/webhook/commands/unknown.py index f1d70b0..2feb829 100644 --- a/src/blueprints/telegram_bot/webhook/commands/unknown.py +++ b/src/blueprints/telegram_bot/webhook/commands/unknown.py @@ -1,17 +1,20 @@ from flask import g from src.api import telegram -from . import CommandsNames +from src.blueprints.telegram_bot._common.command_names import CommandName -def handle(): +def handle(*args, **kwargs): """ Handles unknown command. """ telegram.send_message( - chat_id=g.telegram_chat.id, + chat_id=kwargs.get( + "chat_id", + g.telegram_chat.id + ), text=( "I don't know this command. " - f"See commands list or type {CommandsNames.HELP.value}" + f"See commands list or type {CommandName.HELP.value}" ) ) diff --git a/src/blueprints/telegram_bot/webhook/commands/unpublish.py b/src/blueprints/telegram_bot/webhook/commands/unpublish.py new file mode 100644 index 0000000..fbe9be5 --- /dev/null +++ b/src/blueprints/telegram_bot/webhook/commands/unpublish.py @@ -0,0 +1,94 @@ +from flask import g, current_app + +from src.api import telegram +from src.blueprints.telegram_bot._common.yandex_disk import ( + unpublish_item, + YandexAPIUnpublishItemError, + YandexAPIRequestError +) +from src.blueprints.telegram_bot._common.stateful_chat import ( + stateful_chat_is_enabled, + set_disposable_handler +) +from src.blueprints.telegram_bot._common.command_names import ( + CommandName +) +from src.blueprints.telegram_bot.webhook.dispatcher_events import ( + DispatcherEvent +) +from ._common.responses import ( + cancel_command, + request_absolute_path, + send_yandex_disk_error +) +from ._common.decorators import ( + yd_access_token_required, + get_db_data +) +from ._common.utils import extract_absolute_path + + +@yd_access_token_required +@get_db_data +def handle(*args, **kwargs): + """ + Handles `/unpublish` command. + """ + message = kwargs.get( + "message", + g.telegram_message + ) + user_id = kwargs.get( + "user_id", + g.telegram_user.id + ) + chat_id = kwargs.get( + "chat_id", + g.telegram_chat.id + ) + path = extract_absolute_path( + message, + CommandName.UNPUBLISH.value, + kwargs.get("route_source") + ) + + if not path: + if stateful_chat_is_enabled(): + set_disposable_handler( + user_id, + chat_id, + CommandName.UNPUBLISH.value, + [ + DispatcherEvent.PLAIN_TEXT.value, + DispatcherEvent.BOT_COMMAND.value, + DispatcherEvent.EMAIL.value, + DispatcherEvent.HASHTAG.value, + DispatcherEvent.URL.value + ], + current_app.config["RUNTIME_DISPOSABLE_HANDLER_EXPIRE"] + ) + + return request_absolute_path(chat_id) + + user = g.db_user + access_token = user.yandex_disk_token.get_access_token() + + try: + unpublish_item( + access_token, + path + ) + except YandexAPIRequestError as error: + cancel_command(chat_id) + raise error + except YandexAPIUnpublishItemError as error: + send_yandex_disk_error(chat_id, str(error)) + + # it is expected error and should be + # logged only to user + return + + telegram.send_message( + chat_id=chat_id, + text="Unpublished" + ) diff --git a/src/blueprints/telegram_bot/webhook/commands/upload.py b/src/blueprints/telegram_bot/webhook/commands/upload.py index bc151c0..9858491 100644 --- a/src/blueprints/telegram_bot/webhook/commands/upload.py +++ b/src/blueprints/telegram_bot/webhook/commands/upload.py @@ -1,45 +1,96 @@ from abc import ABCMeta, abstractmethod -from typing import Union +from typing import Union, Set +from collections import deque +from urllib.parse import urlparse from flask import g, current_app from src.api import telegram -from src.blueprints.telegram_bot.webhook import telegram_interface -from .common.decorators import ( - yd_access_token_required, - get_db_data +from src.extensions import task_queue +from src.blueprints._common.utils import get_current_iso_datetime +from src.blueprints.telegram_bot._common import youtube_dl +from src.blueprints.telegram_bot._common.telegram_interface import ( + Message as TelegramMessage ) -from .common.responses import ( - abort_command, - cancel_command +from src.blueprints.telegram_bot._common.command_names import ( + CommandName ) -from .common.yandex_api import ( +from src.blueprints.telegram_bot._common.yandex_disk import ( upload_file_with_url, + get_element_info, + publish_item, YandexAPIRequestError, YandexAPICreateFolderError, YandexAPIUploadFileError, YandexAPIExceededNumberOfStatusChecksError ) +from src.blueprints.telegram_bot._common.stateful_chat import ( + stateful_chat_is_enabled, + set_disposable_handler +) +from src.blueprints.telegram_bot.webhook.dispatcher_events import ( + DispatcherEvent +) +from ._common.decorators import ( + yd_access_token_required, + get_db_data +) +from ._common.responses import ( + abort_command, + cancel_command, + send_yandex_disk_error, + AbortReason +) +from ._common.utils import ( + create_element_info_html_text +) + + +class MessageHealth: + """ + Health status of Telegram message. + """ + def __init__( + self, + ok: bool, + abort_reason: Union[AbortReason, None] = None + ) -> None: + """ + :param ok: + Message is valid for subsequent handling. + :param abort_reason: + Reason of abort. `None` if `ok = True`. + """ + self.ok = ok + self.abort_reason = None class AttachmentHandler(metaclass=ABCMeta): """ Handles uploading of attachment of Telegram message. + + - most of attachments will be treated as files. + - some of the not abstract class functions are common + for most attachments. If you need specific logic in some + function, then override it. """ def __init__(self) -> None: # Sended message to Telegram user. - # This message will be updated, rather - # than sending new message every time - self.sended_message: Union[ - telegram_interface.Message, - None - ] = None + # This message will be updated, instead + # than sending new message every time again + self.sended_message: Union[TelegramMessage, None] = None @staticmethod @abstractmethod - def handle() -> None: + def handle(*args, **kwargs) -> None: """ Starts uploading process. + + - `*args`, `**kwargs` - arguments from dispatcher. + + :raises: + Raises an error if any! So, this function should + be handled by top-function. """ pass @@ -47,87 +98,302 @@ def handle() -> None: @abstractmethod def telegram_action(self) -> str: """ - :returns: Action type from + :returns: + Action type from https://core.telegram.org/bots/api/#sendchataction """ pass + @property @abstractmethod - def message_is_valid( - self, - message: telegram_interface.Message - ) -> bool: + def telegram_command(self) -> str: + """ + - use `CommandName` enum. + + :returns: + With what Telegram command this handler + is associated. It is exact command name, + not enum value. """ - :param message: Incoming Telegram message. + pass - :returns: Message is valid and should be handled. + @property + @abstractmethod + def raw_data_key(self) -> str: + """ + :returns: + Key in message, under this key stored needed raw data. + Example: `audio`. + See https://core.telegram.org/bots/api#message """ pass + @property @abstractmethod + def raw_data_type(self) -> type: + """ + :returns: + Expected type of raw data. + Example: `dict`. + `None` never should be returned! + """ + pass + + @property + @abstractmethod + def dispatcher_events(self) -> Set[str]: + """ + - use `DispatcherEvent` enum. + + :returns: + With what dispatcher events this handler + is associated. These events will be used + to set disposable handler. It is exact event + names, not enum values. + """ + pass + + @abstractmethod + def create_help_message(self) -> str: + """ + - supports HTML markup. + + :returns: + Help message that will be sended to user + in case of some triggers (for example, when + there are no suitable data for handling). + """ + pass + + @property + def public_upload(self) -> bool: + """ + :returns: + Upload file and then publish it. + Defaults to `False`. + """ + return False + def get_attachment( self, - message: telegram_interface.Message + message: TelegramMessage ) -> Union[dict, str, None]: """ - :param message: Incoming Telegram message. + :param message: + Incoming Telegram message. + + :returns: + Attachment of message (photo object, file object, + audio object, etc.). If `None`, then uploading should + be aborted. If `dict`, it will have `file_id` and + `file_unique_id` properties. If `str`, it should be + assumed as direct file URL. + See https://core.telegram.org/bots/api/#available-types + """ + return message.raw_data.get(self.raw_data_key) - :returns: Attachment of message (photo object, - file object, audio object, etc.). If `None`, - uploading will be aborted. If `dict`, it must have `file_id` - and `file_unique_id` properties. If `str`, it is assumed - as direct file URL. See - https://core.telegram.org/bots/api/#available-types + def check_message_health( + self, + attachment: Union[dict, str, None] + ) -> MessageHealth: """ - pass + :param attachment: + Attachment of incoming Telegram message. + `None` means there is no attachment. + + :returns: + See `MessageHealth` documentation. + Message should be handled by next operators only + in case of `ok = true`. + """ + health = MessageHealth(True) + + if attachment is None: + health.ok = False + health.abort_reason = AbortReason.NO_SUITABLE_DATA + elif not isinstance(attachment, self.raw_data_type): + health.ok = False + health.abort_reason = AbortReason.NO_SUITABLE_DATA + elif ( + (type(attachment) in [str]) and + (len(attachment) == 0) + ): + health.ok = False + health.abort_reason = AbortReason.NO_SUITABLE_DATA + elif ( + isinstance(attachment, dict) and + self.is_too_big_file(attachment) + ): + health.ok = False + health.abort_reason = AbortReason.EXCEED_FILE_SIZE_LIMIT + + return health - @abstractmethod def create_file_name( self, attachment: Union[dict, str], file: Union[dict, None] ) -> str: """ - :param attachment: Not `None` value from `self.get_attachment()`. - :param file: Representation of this attachment as a file on - Telegram servers. If `attachment` is `str`, then this will - be equal `None`. See https://core.telegram.org/bots/api/#file + :param attachment: + Not `None` value from `self.get_attachment()`. + :param file: + Representation of this attachment as a file on + Telegram servers. If `attachment` is `str`, then + this will be equal `None`. + See https://core.telegram.org/bots/api/#file + + :returns: + Name of file which will be uploaded. + """ + if isinstance(attachment, str): + return attachment + + name = ( + attachment.get("file_name") or + file["file_unique_id"] + ) + extension = self.get_mime_type(attachment) + + if extension: + name = f"{name}.{extension}" + + return name + + def get_mime_type(self, attachment: dict) -> str: + """ + :param attachment: + `dict` result from `self.get_attachment()`. - :returns: Name of file which will be uploaded. + :returns: + Empty string in case if `attachment` doesn't have + required key. Otherwise mime type of this attachment. """ - pass + result = "" + + if "mime_type" in attachment: + types = attachment["mime_type"].split("/") + result = types[1] + + return result + + def is_too_big_file(self, file: dict) -> bool: + """ + Checks if size of file exceeds limit size of upload. + + :param file: + `dict` value from `self.get_attachment()`. + + :returns: + File size exceeds upload limit size. + Always `False` if file size is unknown. + """ + limit = current_app.config["TELEGRAM_API_MAX_FILE_SIZE"] + size = limit + + if "file_size" in file: + size = file["file_size"] + + return (size > limit) + + def set_disposable_handler( + self, + user_id: int, + chat_id: int + ) -> None: + """ + Sets disposable handler. + + It is means that next message with matched + `self.dispatcher_events` will be forwarded to + `self.telegram_command`. + + - will be used when user didn't sent any + suitable data for handling. + + :param user_id: + Telegram ID of current user. + :param chat_id: + Telegram ID of current chat. + """ + if not stateful_chat_is_enabled(): + return + + expire = current_app.config[ + "RUNTIME_DISPOSABLE_HANDLER_EXPIRE" + ] + + set_disposable_handler( + user_id, + chat_id, + self.telegram_command, + self.dispatcher_events, + expire + ) @yd_access_token_required @get_db_data - def upload(self) -> None: + def init_upload(self, *args, **kwargs) -> None: """ - Uploads an attachment. + Initializes uploading process of message attachment. + Attachment will be prepared for uploading, and if + everything is ok, then uploading will be automatically + started, otherwise error will be logged back to user. + + - it is expected entry point for dispatcher. + - `*args`, `**kwargs` - arguments from dispatcher. + + NOTE: + Depending on app configuration uploading can start + in same or separate process. If it is same process, + then this function will take a long time to complete, + if it is separate process, then this function will + be completed fast. """ - message = g.telegram_message - user = g.db_user - chat = g.db_chat + user_id = kwargs.get( + "chat_id", + g.db_user.telegram_id + ) + chat_id = kwargs.get( + "chat_id", + g.db_chat.telegram_id + ) + message = kwargs.get( + "message", + g.telegram_message + ) + attachment = self.get_attachment(message) + message_health = self.check_message_health(attachment) - if not (self.message_is_valid(message)): - return abort_command(chat.telegram_id) + if not message_health.ok: + reason = ( + message_health.abort_reason or + AbortReason.UNKNOWN + ) - attachment = self.get_attachment(message) + if (reason == AbortReason.NO_SUITABLE_DATA): + self.set_disposable_handler(user_id, chat_id) - if (attachment is None): - return abort_command(chat.telegram_id) + return self.send_html_message( + chat_id, + self.create_help_message() + ) + else: + return abort_command(chat_id, reason) try: telegram.send_chat_action( - chat_id=chat.telegram_id, + chat_id=chat_id, action=self.telegram_action ) except Exception as error: - print(error) - return cancel_command(chat.telegram_id) + cancel_command(chat_id) + raise error download_url = None file = None - if (isinstance(attachment, str)): + if isinstance(attachment, str): download_url = attachment else: result = None @@ -137,90 +403,310 @@ def upload(self) -> None: file_id=attachment["file_id"] ) except Exception as error: - print(error) - return cancel_command(chat.telegram_id) + cancel_command(chat_id) + raise error file = result["content"] download_url = telegram.create_file_download_url( file["file_path"] ) + message_id = message.message_id + user = g.db_user file_name = self.create_file_name(attachment, file) user_access_token = user.yandex_disk_token.get_access_token() folder_path = current_app.config[ "YANDEX_DISK_API_DEFAULT_UPLOAD_FOLDER" ] + arguments = ( + folder_path, + file_name, + download_url, + user_access_token, + chat_id, + message_id + ) - def long_task(): - try: - for status in upload_file_with_url( - user_access_token=user_access_token, - folder_path=folder_path, - file_name=file_name, - download_url=download_url - ): - self.send_message(f"Status: {status}") - except YandexAPICreateFolderError as error: - error_text = str(error) or ( - "I can't create default upload folder " - "due to an unknown Yandex error." - ) - - return self.send_message(error_text) - except YandexAPIUploadFileError as error: - error_text = str(error) or ( - "I can't upload this due " - "to an unknown Yandex error." - ) + # Everything is fine by this moment. + # Because task workers can be busy, + # it can take a while to start uploading. + # Let's indicate to user that uploading + # process is started and user shouldn't + # send any data again + self.reply_to_message( + message_id, + chat_id, + "Status: pending", + False + ) - return self.send_message(error_text) - except YandexAPIExceededNumberOfStatusChecksError: - error_text = ( - "I can't track operation status of " - "this anymore. Perform manual checking." - ) + if task_queue.is_enabled: + job_timeout = current_app.config[ + "RUNTIME_UPLOAD_WORKER_JOB_TIMEOUT" + ] + ttl = current_app.config[ + "RUNTIME_UPLOAD_WORKER_UPLOAD_TTL" + ] + result_ttl = current_app.config[ + "RUNTIME_UPLOAD_WORKER_RESULT_TTL" + ] + failure_ttl = current_app.config[ + "RUNTIME_UPLOAD_WORKER_FAILURE_TTL" + ] + + task_queue.enqueue( + self.start_upload, + args=arguments, + description=self.telegram_command, + job_timeout=job_timeout, + ttl=ttl, + result_ttl=result_ttl, + failure_ttl=failure_ttl + ) + else: + # NOTE: current thread will + # be blocked for a long time + self.start_upload(*arguments) - return self.send_message(error_text) - except (YandexAPIRequestError, Exception) as error: - print(error) + def start_upload( + self, + folder_path: str, + file_name: str, + download_url: str, + user_access_token: str, + chat_id: int, + message_id: int + ) -> None: + """ + Starts uploading of provided URL. + + It will send provided URL to Yandex.Disk API, + after that operation monitoring will be started. + See app configuration for monitoring config. + + NOTE: + This function requires long time to complete. + And because it is sync function, it will block + your thread. + + :param folder_path: + Yandex.Disk path where to put file. + :param file_name: + Name (with extension) of result file. + :param download_url: + Direct URL to file. Yandex.Disk will download it. + :param user_access_token: + Access token of user to access Yandex.Disk API. + :param chat_id: + ID of incoming Telegram chat. + :param message_id: + ID of incoming Telegram message. + This message will be reused to edit this message + with new status instead of sending it every time. + + :raises: + Raises error if occurs. + """ + full_path = f"{folder_path}/{file_name}" - if (self.sended_message is None): - return cancel_command( - chat.telegram_id, - reply_to_message=message.message_id - ) + try: + for status in upload_file_with_url( + user_access_token=user_access_token, + folder_path=folder_path, + file_name=file_name, + download_url=download_url + ): + success = status["success"] + text_content = deque() + is_html_text = False + + if success: + is_private_message = (not self.public_upload) + + if self.public_upload: + try: + publish_item( + user_access_token, + full_path + ) + except Exception as error: + print(error) + text_content.append( + "\n" + "Failed to publish. Type to do it:" + "\n" + f"{CommandName.PUBLISH.value} {full_path}" + ) + + info = None + + try: + info = get_element_info( + user_access_token, + full_path, + get_public_info=False + ) + except Exception as error: + print(error) + text_content.append( + "\n" + "Failed to get information. Type to do it:" + "\n" + f"{CommandName.ELEMENT_INFO.value} {full_path}" + ) + + if text_content: + text_content.append( + "It is successfully uploaded, " + "but i failed to perform some actions. " + "You need to execute them manually." + ) + text_content.reverse() + + if info: + # extra line before info + if text_content: + text_content.append("") + + is_html_text = True + info_text = create_element_info_html_text( + info, + include_private_info=is_private_message + ) + text_content.append(info_text) else: - return cancel_command( - chat.telegram_id, - edit_message=self.sended_message.message_id + # You shouldn't use HTML for this, + # because `upload_status` can be a same + upload_status = status["status"] + text_content.append( + f"Status: {upload_status}" ) - long_task() + text = "\n".join(text_content) + + self.reply_to_message( + message_id, + chat_id, + text, + is_html_text + ) + except YandexAPICreateFolderError as error: + error_text = str(error) or ( + "I can't create default upload folder " + "due to an unknown Yandex.Disk error." + ) + + return send_yandex_disk_error( + chat_id, + error_text, + message_id + ) + except YandexAPIUploadFileError as error: + error_text = str(error) or ( + "I can't upload this due " + "to an unknown Yandex.Disk error." + ) + + return send_yandex_disk_error( + chat_id, + error_text, + message_id + ) + except YandexAPIExceededNumberOfStatusChecksError: + error_text = ( + "I can't track operation status of " + "this anymore. It can be uploaded " + "after a while. Type to check:" + "\n" + f"{CommandName.ELEMENT_INFO.value} {full_path}" + ) - def send_message(self, text: str) -> None: + return self.reply_to_message( + message_id, + chat_id, + error_text + ) + except Exception as error: + if self.sended_message is None: + cancel_command( + chat_id, + reply_to_message=message_id + ) + else: + cancel_command( + chat_id, + edit_message=self.sended_message.message_id + ) + + raise error + + def send_html_message( + self, + chat_id: int, + html_text: str + ) -> None: """ - Sends message to Telegram user. + Sends HTML message to Telegram user. + """ + telegram.send_message( + chat_id=chat_id, + text=html_text, + parse_mode="HTML", + disable_web_page_preview=True + ) - - if message already was sent, then sent message - will be updated with new text. + def reply_to_message( + self, + incoming_message_id: int, + chat_id: int, + text: str, + html_text=False + ) -> None: + """ + Sends reply message to Telegram user. + + - if message already was sent, then sent + message will be updated with new text. + - NOTE: using HTML text may lead to error, + because text should be compared with already + sended text, but already sended text will not + contain HTML tags (even if they was before sending), + and `text` will, so, comparing already sended HTML + text and `text` always will results to `False`. """ - incoming_message = g.telegram_message - chat = g.db_chat + enabled_html = {} + + if html_text: + enabled_html["parse_mode"] = "HTML" + + result = None - if (self.sended_message is None): + if self.sended_message is None: result = telegram.send_message( - reply_to_message_id=incoming_message.message_id, - chat_id=chat.telegram_id, - text=text - ) - self.sended_message = telegram_interface.Message( - result["content"] + reply_to_message_id=incoming_message_id, + chat_id=chat_id, + text=text, + allow_sending_without_reply=True, + disable_web_page_preview=True, + **enabled_html ) elif (text != self.sended_message.get_text()): - telegram.edit_message_text( + result = telegram.edit_message_text( message_id=self.sended_message.message_id, - chat_id=chat.telegram_id, - text=text + chat_id=chat_id, + text=text, + disable_web_page_preview=True, + **enabled_html + ) + + new_message_sended = ( + (result is not None) and + result["ok"] + ) + + if new_message_sended: + self.sended_message = TelegramMessage( + result["content"] ) @@ -229,107 +715,168 @@ class PhotoHandler(AttachmentHandler): Handles uploading of photo. """ @staticmethod - def handle(): + def handle(*args, **kwargs): handler = PhotoHandler() - handler.upload() + handler.init_upload(*args, **kwargs) @property def telegram_action(self): return "upload_photo" - def message_is_valid(self, message: telegram_interface.Message): - raw_data = message.raw_data + @property + def telegram_command(self): + return CommandName.UPLOAD_PHOTO.value + + @property + def raw_data_key(self): + return "photo" + @property + def raw_data_type(self): + # dict, not list, because we will select biggest photo + return dict + + @property + def dispatcher_events(self): + return [ + DispatcherEvent.PHOTO.value + ] + + def create_help_message(self): return ( - isinstance( - raw_data.get("photo"), - list - ) and - len(raw_data["photo"]) > 0 + "Send a photos that you want to upload" + f"{' and publish' if self.public_upload else ''}." + "\n\n" + "Note:" + "\n" + "- original name will be not saved" + "\n" + "- original quality and size will be decreased" ) - def get_attachment(self, message: telegram_interface.Message): - raw_data = message.raw_data - photos = raw_data["photo"] - biggest_photo = photos[0] + def get_attachment(self, message: TelegramMessage): + photos = message.raw_data.get(self.raw_data_key, []) + biggest_photo = None + biggest_pixels_count = -1 - for photo in photos[1:]: - if (photo["height"] > biggest_photo["height"]): + for photo in photos: + if self.is_too_big_file(photo): + continue + + current_pixels_count = photo["width"] * photo["height"] + + if (current_pixels_count > biggest_pixels_count): biggest_photo = photo + biggest_pixels_count = current_pixels_count return biggest_photo - def create_file_name(self, attachment, file): - return file["file_unique_id"] - class FileHandler(AttachmentHandler): """ Handles uploading of file. """ @staticmethod - def handle(): + def handle(*args, **kwargs): handler = FileHandler() - handler.upload() + handler.init_upload(*args, **kwargs) @property def telegram_action(self): return "upload_document" - def message_is_valid(self, message: telegram_interface.Message): - return ( - isinstance( - message.raw_data.get("document"), - dict - ) - ) + @property + def telegram_command(self): + return CommandName.UPLOAD_FILE.value + + @property + def raw_data_key(self): + return "document" - def get_attachment(self, message: telegram_interface.Message): - return message.raw_data["document"] + @property + def raw_data_type(self): + return dict - def create_file_name(self, attachment, file): + @property + def dispatcher_events(self): + return [ + DispatcherEvent.FILE.value + ] + + def create_help_message(self): return ( - attachment.get("file_name") or - file["file_unique_id"] + "Send a files that you want to upload" + f"{' and publish' if self.public_upload else ''}." + "\n\n" + "Note:" + "\n" + "- original name will be saved" + "\n" + "- original quality and size will be saved" ) + def get_mime_type(self, attachment): + # file name already contains type + return "" + class AudioHandler(AttachmentHandler): """ Handles uploading of audio. """ @staticmethod - def handle(): + def handle(*args, **kwargs): handler = AudioHandler() - handler.upload() + handler.init_upload(*args, **kwargs) @property def telegram_action(self): return "upload_audio" - def message_is_valid(self, message: telegram_interface.Message): + @property + def telegram_command(self): + return CommandName.UPLOAD_AUDIO.value + + @property + def raw_data_key(self): + return "audio" + + @property + def raw_data_type(self): + return dict + + @property + def dispatcher_events(self): + return [ + DispatcherEvent.AUDIO.value + ] + + def create_help_message(self): return ( - isinstance( - message.raw_data.get("audio"), - dict - ) + "Send a music that you want to upload" + f"{' and publish' if self.public_upload else ''}." + "\n\n" + "Note:" + "\n" + "- original name will be saved" + "\n" + "- original quality and size will be saved" + "\n" + "- original type may be changed" ) - def get_attachment(self, message: telegram_interface.Message): - return message.raw_data["audio"] - def create_file_name(self, attachment, file): name = file["file_unique_id"] - if ("title" in attachment): + if "title" in attachment: name = attachment["title"] - if ("performer" in attachment): + if "performer" in attachment: name = f"{attachment['performer']} - {name}" - if ("mime_type" in attachment): - types = attachment["mime_type"].split("/") - extension = types[1] + extension = self.get_mime_type(attachment) + + if extension: name = f"{name}.{extension}" return name @@ -340,34 +887,45 @@ class VideoHandler(AttachmentHandler): Handles uploading of video. """ @staticmethod - def handle(): + def handle(*args, **kwargs): handler = VideoHandler() - handler.upload() + handler.init_upload(*args, **kwargs) @property def telegram_action(self): return "upload_video" - def message_is_valid(self, message: telegram_interface.Message): - return ( - isinstance( - message.raw_data.get("video"), - dict - ) - ) + @property + def telegram_command(self): + return CommandName.UPLOAD_VIDEO.value - def get_attachment(self, message: telegram_interface.Message): - return message.raw_data["video"] + @property + def raw_data_key(self): + return "video" - def create_file_name(self, attachment, file): - name = file["file_unique_id"] + @property + def raw_data_type(self): + return dict - if ("mime_type" in attachment): - types = attachment["mime_type"].split("/") - extension = types[1] - name = f"{name}.{extension}" + @property + def dispatcher_events(self): + return [ + DispatcherEvent.VIDEO.value + ] - return name + def create_help_message(self): + return ( + "Send a video that you want to upload" + f"{' and publish' if self.public_upload else ''}." + "\n\n" + "Note:" + "\n" + "- original name will be saved" + "\n" + "- original quality and size will be saved" + "\n" + "- original type may be changed" + ) class VoiceHandler(AttachmentHandler): @@ -375,62 +933,318 @@ class VoiceHandler(AttachmentHandler): Handles uploading of voice. """ @staticmethod - def handle(): + def handle(*args, **kwargs): handler = VoiceHandler() - handler.upload() + handler.init_upload(*args, **kwargs) @property def telegram_action(self): return "upload_audio" - def message_is_valid(self, message: telegram_interface.Message): - return ( - isinstance( - message.raw_data.get("voice"), - dict - ) - ) + @property + def telegram_command(self): + return CommandName.UPLOAD_VOICE.value - def get_attachment(self, message: telegram_interface.Message): - return message.raw_data["voice"] + @property + def raw_data_key(self): + return "voice" - def create_file_name(self, attachment, file): - name = file["file_unique_id"] + @property + def raw_data_type(self): + return dict - if ("mime_type" in attachment): - types = attachment["mime_type"].split("/") - extension = types[1] - name = f"{name}.{extension}" + @property + def dispatcher_events(self): + return [ + DispatcherEvent.VOICE.value + ] - return name + def create_help_message(self): + return ( + "Send a voice message that you want to upload" + f"{' and publish' if self.public_upload else ''}." + ) + + def create_file_name(self, attachment, file): + return get_current_iso_datetime(sep=" ") -class URLHandler(AttachmentHandler): +class DirectURLHandler(AttachmentHandler): """ Handles uploading of direct URL to file. """ @staticmethod - def handle(): - handler = URLHandler() - handler.upload() + def handle(*args, **kwargs): + handler = DirectURLHandler() + handler.init_upload(*args, **kwargs) @property def telegram_action(self): return "upload_document" - def message_is_valid(self, message: telegram_interface.Message): - value = self.get_attachment(message) + @property + def telegram_command(self): + return CommandName.UPLOAD_URL.value + + @property + def raw_data_key(self): + return "url" + + @property + def raw_data_type(self): + return str + @property + def dispatcher_events(self): + return [ + DispatcherEvent.URL.value + ] + + def create_help_message(self): return ( - isinstance(value, str) and - len(value) > 0 + "Send a direct URL to file that you want to upload" + f"{' and publish' if self.public_upload else ''}." + "\n\n" + "Note:" + "\n" + "- original name from URL will be saved" + "\n" + "- original quality and size will be saved" ) - def get_attachment(self, message: telegram_interface.Message): - return message.get_entity_value("url") + def get_attachment(self, message: TelegramMessage): + return message.get_entity_value(self.raw_data_key) def create_file_name(self, attachment, file): - return attachment.split("/")[-1] + parse_result = urlparse(attachment) + filename = parse_result.path.split("/")[-1] + + # for example, `https://ya.ru` leads to + # empty path, so, `filename` will be empty + # in that case. Then let it be `ya.ru` + if not filename: + filename = parse_result.netloc + + return filename + + +class IntellectualURLHandler(DirectURLHandler): + """ + Handles uploading of direct URL to file. + + But unlike `DirectURLHandler`, this handler will + try to guess which content user actually assumed. + For example, if user passed URL to YouTube video, + `DirectURLHandler` will download HTML page, but + `IntellectualURLHandler` will download a video. + + In short, `DirectURLHandler` makes minimum changes + to input content; `IntellectualURLHandler` makes + maximum changes, but these changes focused for + "that user actually wants" and "it should be right". + + What this handler does: + - using `youtube_dl` gets direct URL to + input video/music URL, and gets right filename + - in case if nothing works, then fallback to `DirectURLHandler` + """ + def __init__(self): + super().__init__() + + self.input_url = None + self.youtube_dl_info = None + + @staticmethod + def handle(*args, **kwargs): + handler = IntellectualURLHandler() + handler.init_upload(*args, **kwargs) + + def create_help_message(self): + return ( + "Send an URL to resource that you want to upload" + f"{' and publish' if self.public_upload else ''}." + "\n\n" + "Note:" + "\n" + "- for direct URL's to file original name, " + "quality and size will be saved" + "\n" + "- for URL's to some resource best name amd " + "best possible quality will be selected" + "\n" + "- i will try to guess what resource you actually assumed. " + "For example, you can send URL to YouTube video or " + "Twitch clip, and video from that URL will be uploaded" + "\n" + "- you can send URL to any resource: video, audio, image, " + "text, page, etc. Not everything will work as you expect, " + "but some URL's will" + "\n" + "- i'm using youtube-dl, if that means anything to you (:" + ) + + def get_attachment(self, message: TelegramMessage): + self.input_url = super().get_attachment(message) + + if not self.input_url: + return None + + best_url = self.input_url + + try: + self.youtube_dl_info = youtube_dl.extract_info( + self.input_url + ) + except youtube_dl.UnsupportedURLError: + # Unsupported URL's is expected here, + # let's treat them as direct URL's to files + pass + except youtube_dl.UnexpectedError as error: + # TODO: + # Something goes wrong in `youtube_dl`. + # It is better to log this error to user, + # because there can be restrictions or limits, + # but there also can be some internal info + # which shouldn't be printed to user. + # At the moment there is no best way for UX, so, + # let's just print this information in logs. + print( + "Unexpected youtube_dl error:", + error + ) + + if self.youtube_dl_info: + best_url = self.youtube_dl_info["direct_url"] + + return best_url + + def create_file_name(self, attachment, file): + input_filename = super().create_file_name( + self.input_url, + file + ) + youtube_dl_filename = None + + if self.youtube_dl_info: + youtube_dl_filename = self.youtube_dl_info.get( + "filename" + ) + + best_filename = ( + youtube_dl_filename or + input_filename + ) + + return best_filename + + +class PublicHandler: + """ + Handles public uploading. + """ + @property + def public_upload(self): + return True + + +class PublicPhotoHandler(PublicHandler, PhotoHandler): + """ + Handles public uploading of photo. + """ + @staticmethod + def handle(*args, **kwargs): + handler = PublicPhotoHandler() + handler.init_upload(*args, **kwargs) + + @property + def telegram_command(self): + return CommandName.PUBLIC_UPLOAD_PHOTO.value + + +class PublicFileHandler(PublicHandler, FileHandler): + """ + Handles public uploading of file. + """ + @staticmethod + def handle(*args, **kwargs): + handler = PublicFileHandler() + handler.init_upload(*args, **kwargs) + + @property + def telegram_command(self): + return CommandName.PUBLIC_UPLOAD_FILE.value + + +class PublicAudioHandler(PublicHandler, AudioHandler): + """ + Handles public uploading of audio. + """ + @staticmethod + def handle(*args, **kwargs): + handler = PublicAudioHandler() + handler.init_upload(*args, **kwargs) + + @property + def telegram_command(self): + return CommandName.PUBLIC_UPLOAD_AUDIO.value + + +class PublicVideoHandler(PublicHandler, VideoHandler): + """ + Handles public uploading of video. + """ + @staticmethod + def handle(*args, **kwargs): + handler = PublicVideoHandler() + handler.init_upload(*args, **kwargs) + + @property + def telegram_command(self): + return CommandName.PUBLIC_UPLOAD_VIDEO.value + + +class PublicVoiceHandler(PublicHandler, VoiceHandler): + """ + Handles public uploading of voice. + """ + @staticmethod + def handle(*args, **kwargs): + handler = PublicVoiceHandler() + handler.init_upload(*args, **kwargs) + + @property + def telegram_command(self): + return CommandName.PUBLIC_UPLOAD_VOICE.value + + +class PublicDirectURLHandler(PublicHandler, DirectURLHandler): + """ + Handles public uploading of direct URL to file. + """ + @staticmethod + def handle(*args, **kwargs): + handler = PublicDirectURLHandler() + handler.init_upload(*args, **kwargs) + + @property + def telegram_command(self): + return CommandName.PUBLIC_UPLOAD_URL.value + + +class PublicIntellectualURLHandler(PublicHandler, IntellectualURLHandler): + """ + Handles public uploading of direct URL to file. + + - see `IntellectualURLHandler` documentation. + """ + @staticmethod + def handle(*args, **kwargs): + handler = PublicIntellectualURLHandler() + handler.init_upload(*args, **kwargs) + + @property + def telegram_command(self): + return CommandName.PUBLIC_UPLOAD_URL.value handle_photo = PhotoHandler.handle @@ -438,4 +1252,10 @@ def create_file_name(self, attachment, file): handle_audio = AudioHandler.handle handle_video = VideoHandler.handle handle_voice = VoiceHandler.handle -handle_url = URLHandler.handle +handle_url = IntellectualURLHandler.handle +handle_public_photo = PublicPhotoHandler.handle +handle_public_file = PublicFileHandler.handle +handle_public_audio = PublicAudioHandler.handle +handle_public_video = PublicVideoHandler.handle +handle_public_voice = PublicVoiceHandler.handle +handle_public_url = PublicIntellectualURLHandler.handle diff --git a/src/blueprints/telegram_bot/webhook/commands/yd_auth.py b/src/blueprints/telegram_bot/webhook/commands/yd_auth.py index bcefc4d..1aa59b2 100644 --- a/src/blueprints/telegram_bot/webhook/commands/yd_auth.py +++ b/src/blueprints/telegram_bot/webhook/commands/yd_auth.py @@ -1,145 +1,306 @@ -import secrets +from flask import g, current_app -from flask import ( - g, - current_app -) -import jwt - -from src.database import ( - db, - YandexDiskToken -) -from src.api import telegram, yandex -from src.blueprints.utils import ( +from src.api import telegram +from src.configs.flask import YandexOAuthAPIMethod +from src.blueprints._common.utils import ( absolute_url_for, get_current_datetime ) -from .common.decorators import ( +from src.blueprints.telegram_bot._common import yandex_oauth +from src.blueprints.telegram_bot._common.stateful_chat import ( + stateful_chat_is_enabled, + set_disposable_handler, + set_user_chat_data, + get_user_chat_data, + delete_user_chat_data +) +from src.blueprints.telegram_bot.webhook.dispatcher_events import ( + DispatcherEvent, + RouteSource +) +from ._common.decorators import ( register_guest, get_db_data ) -from .common.responses import ( +from ._common.responses import ( request_private_chat, cancel_command ) -from . import CommandsNames +from src.blueprints.telegram_bot._common.command_names import CommandName @register_guest @get_db_data -def handle(): +def handle(*args, **kwargs): """ - Handles `/yandex_disk_authorization` command. + Handles `/grant_access` command. - Authorization of bot in user Yandex.Disk. + Allowing to bot to use user Yandex.Disk. """ - user = g.db_user - incoming_chat = g.db_chat private_chat = g.db_private_chat - yd_token = user.yandex_disk_token - - if (private_chat is None): - return request_private_chat(incoming_chat.telegram_id) - - if (yd_token is None): - try: - yd_token = create_empty_yd_token(user) - except Exception as error: - print(error) - return cancel_command(private_chat.telegram_id) - - refresh_needed = False - - if (yd_token.have_access_token()): - try: - yd_token.get_access_token() - - telegram.send_message( - chat_id=private_chat.telegram_id, - text=( - "You already grant me access to your Yandex.Disk." - "\n" - "You can revoke my access with " - f"{CommandsNames.YD_REVOKE.value}" - ) - ) - # `access_token` is valid - return - except Exception: - # `access_token` is expired (most probably) or - # data in DB is invalid - refresh_needed = True - - if (refresh_needed): - success = refresh_access_token(yd_token) - - if (success): - current_datetime = get_current_datetime() - date = current_datetime["date"] - time = current_datetime["time"] - timezone = current_datetime["timezone"] - - telegram.send_message( - chat_id=private_chat.telegram_id, - parse_mode="HTML", - text=( - "Access to Yandex.Disk Refreshed" - "\n\n" - "Your granted access was refreshed automatically by me " - f"on {date} at {time} {timezone}." - "\n\n" - "If it wasn't you, you can detach this access with " - f"{CommandsNames.YD_REVOKE.value}" - ) - ) + # we allow to use this command only in + # private chats for security reasons + if private_chat is None: + incoming_chat_id = kwargs.get( + "chat_id", + g.db_chat.telegram_id + ) - return + return request_private_chat(incoming_chat_id) - yd_token.clear_all_tokens() - yd_token.set_insert_token( - secrets.token_hex( - current_app.config[ - "YANDEX_DISK_API_INSERT_TOKEN_BYTES" - ] + user = g.db_user + chat_id = private_chat.telegram_id + route_source = kwargs.get("route_source") + + # console client is waiting for user code + if (route_source == RouteSource.DISPOSABLE_HANDLER): + return end_console_client( + user, + chat_id, + kwargs["message"].get_plain_text() ) + + oauth_method = current_app.config.get("YANDEX_OAUTH_API_METHOD") + + if (oauth_method == YandexOAuthAPIMethod.AUTO_CODE_CLIENT): + run_auto_code_client(user, chat_id) + elif (oauth_method == YandexOAuthAPIMethod.CONSOLE_CLIENT): + start_console_client(user, chat_id) + else: + cancel_command(chat_id) + raise Exception("Unknown Yandex.OAuth method") + + +def run_auto_code_client(db_user, chat_id: int) -> None: + client = yandex_oauth.YandexOAuthAutoCodeClient() + result = None + + try: + result = client.before_user_interaction(db_user) + except Exception as error: + cancel_command(chat_id) + raise error + + status = result["status"] + + if (status == yandex_oauth.OperationStatus.HAVE_ACCESS_TOKEN): + message_have_access_token(chat_id) + elif (status == yandex_oauth.OperationStatus.ACCESS_TOKEN_REFRESHED): + message_access_token_refreshed(chat_id) + elif (status == yandex_oauth.OperationStatus.CONTINUE_TO_URL): + # only in that case further user actions is needed + message_grant_access_redirect( + chat_id, + result["url"], + result["lifetime"] + ) + else: + cancel_command(chat_id) + raise Exception("Unknown operation status") + + +def start_console_client(db_user, chat_id: int) -> None: + if not stateful_chat_is_enabled(): + cancel_command(chat_id) + raise Exception("Stateful chat is required to be enabled") + + client = yandex_oauth.YandexOAuthConsoleClient() + result = None + + try: + result = client.before_user_interaction(db_user) + except Exception as error: + cancel_command(chat_id) + raise error + + status = result["status"] + + if (status == yandex_oauth.OperationStatus.HAVE_ACCESS_TOKEN): + message_have_access_token(chat_id) + elif (status == yandex_oauth.OperationStatus.ACCESS_TOKEN_REFRESHED): + message_access_token_refreshed(chat_id) + elif (status == yandex_oauth.OperationStatus.CONTINUE_TO_URL): + # only in that case further user actions is needed + message_grant_access_code( + chat_id, + result["url"], + result["lifetime"] + ) + # we will pass this state later + set_user_chat_data( + db_user.telegram_id, + chat_id, + "yandex_oauth_console_client_state", + result["state"], + result["lifetime"] + ) + # on next plain text message we will handle provided code + set_disposable_handler( + db_user.telegram_id, + chat_id, + CommandName.YD_AUTH.value, + [DispatcherEvent.PLAIN_TEXT.value], + result["lifetime"] + ) + else: + cancel_command(chat_id) + raise Exception("Unknown operation status") + + +def end_console_client(db_user, chat_id: int, code: str) -> None: + state = get_user_chat_data( + db_user.telegram_id, + chat_id, + "yandex_oauth_console_client_state" ) - yd_token.insert_token_expires_in = ( - current_app.config[ - "YANDEX_DISK_API_INSERT_TOKEN_LIFETIME" - ] + delete_user_chat_data( + db_user.telegram_id, + chat_id, + "yandex_oauth_console_client_state" ) - db.session.commit() - insert_token = None + if not state: + return telegram.send_message( + chat_id=chat_id, + text=( + "You waited too long. " + f"Send {CommandName.YD_AUTH} again." + ) + ) + + client = yandex_oauth.YandexOAuthConsoleClient() + result = None + message = None try: - insert_token = yd_token.get_insert_token() + result = client.handle_code(state, code) + except yandex_oauth.InvalidState: + message = ( + "Your credentials is not valid. " + f"Try send {CommandName.YD_AUTH} again." + ) + except yandex_oauth.ExpiredInsertToken: + message = ( + "You waited too long. " + f"Send {CommandName.YD_AUTH} again." + ) + except yandex_oauth.InvalidInsertToken: + message = ( + "You have too many authorization sessions. " + f"Send {CommandName.YD_AUTH} again and use only last link." + ) except Exception as error: - print(error) - return cancel_command(private_chat.telegram_id) - - if (insert_token is None): - print("Error: Insert Token is NULL") - return cancel_command(private_chat.telegram_id) - - state = jwt.encode( - { - "user_id": user.id, - "insert_token": insert_token - }, - current_app.secret_key.encode(), - algorithm="HS256" - ).decode() - yandex_oauth_url = yandex.create_user_oauth_url(state) - insert_token_lifetime = int( - yd_token.insert_token_expires_in / 60 + cancel_command(chat_id) + raise error + + if message: + return telegram.send_message( + chat_id=chat_id, + text=message + ) + + if not result["ok"]: + error = result["error"] + title = error.get( + "title", + "Unknown error" + ) + description = error.get( + "description", + "Can't tell you more" + ) + + return telegram.send_message( + chat_id=chat_id, + parse_mode="HTML", + text=( + "Yandex.OAuth Error" + "\n\n" + f"Error: {title}" + "\n" + f"Description: {description}" + "\n\n" + "I still don't have access. " + f"Start new session using {CommandName.YD_AUTH.value}" + ) + ) + + message_access_token_granted(chat_id) + + +# region Messages + + +def message_have_access_token(chat_id: int) -> None: + telegram.send_message( + chat_id=chat_id, + text=( + "You already grant me access to your Yandex.Disk." + "\n" + "You can revoke my access with " + f"{CommandName.YD_REVOKE.value}" + ) + ) + + +def message_access_token_refreshed(chat_id: int) -> None: + now = get_current_datetime() + date = now["date"] + time = now["time"] + timezone = now["timezone"] + + telegram.send_message( + chat_id=chat_id, + parse_mode="HTML", + text=( + "Access to Yandex.Disk Refreshed" + "\n\n" + "Your granted access was refreshed automatically by me " + f"on {date} at {time} {timezone}." + "\n\n" + "If it wasn't you, you can detach this access with " + f"{CommandName.YD_REVOKE.value}" + ) + ) + + +def message_access_token_granted(chat_id: int) -> None: + now = get_current_datetime() + date = now["date"] + time = now["time"] + timezone = now["timezone"] + + telegram.send_message( + chat_id=chat_id, + parse_mode="HTML", + text=( + "Access to Yandex.Disk Granted" + "\n\n" + "My access was attached to your Telegram account " + f"on {date} at {time} {timezone}." + "\n\n" + "If it wasn't you, then detach this access with " + f"{CommandName.YD_REVOKE.value}" + ) ) + + +def message_grant_access_redirect( + chat_id: int, + url: str, + lifetime_in_seconds: int +) -> None: open_link_button_text = "Grant access" + revoke_command = CommandName.YD_REVOKE.value + yandex_passport_url = "https://passport.yandex.ru/profile" + source_code_url = current_app.config["PROJECT_URL_FOR_CODE"] + privacy_policy_url = absolute_url_for("legal.privacy_policy") + terms_url = absolute_url_for("legal.terms_and_conditions") + lifetime_in_minutes = int(lifetime_in_seconds / 60) telegram.send_message( - chat_id=private_chat.telegram_id, + chat_id=chat_id, parse_mode="HTML", disable_web_page_preview=True, text=( @@ -149,90 +310,109 @@ def handle(): "IMPORTANT: don't give this link to anyone, " "because it contains your secret information." "\n\n" - f"This link will expire in {insert_token_lifetime} minutes." + f"This link will expire in {lifetime_in_minutes} minutes." "\n" "This link leads to Yandex page and redirects to bot page." "\n\n" "It is safe to give the access?" "\n" "Yes! I'm getting access only to your Yandex.Disk, " - "not to your account. You can revoke my access at any time with " - f"{CommandsNames.YD_REVOKE.value} or in your " - 'Yandex Profile. ' + "not to your account. You can revoke my access at any time " + f"with {revoke_command} or in your " + f'Yandex profile. ' "By the way, i'm " - f'open-source ' # noqa + f'open-source ' "and you can make sure that your data will be safe. " - "You can even create your own bot with my functionality if using " - "me makes you feel uncomfortable (:" + "You can even create your own bot with my functionality " + "if using me makes you feel uncomfortable (:" "\n\n" "By using me, you accept " - f'Privacy Policy and ' # noqa - f'Terms of service. ' # noqa + f'Privacy Policy and ' + f'Terms of service. ' ), - reply_markup={"inline_keyboard": [ - [ - { - "text": open_link_button_text, - "url": yandex_oauth_url - } + reply_markup={ + "inline_keyboard": [ + [ + { + "text": open_link_button_text, + "url": url + } + ] ] - ]} + } ) -def create_empty_yd_token(user) -> YandexDiskToken: - """ - Creates empty Yandex.Disk token and binds - this to provided user. - """ - new_yd_token = YandexDiskToken(user=user) - - db.session.add(new_yd_token) - db.session.commit() - - return new_yd_token - - -def refresh_access_token(yd_token: YandexDiskToken) -> bool: - """ - Tries to refresh user access token by using refresh token. - - :returns: `True` in case of success else `False`. - """ - refresh_token = yd_token.get_refresh_token() - - if (refresh_token is None): - return False - - result = None - - try: - result = yandex.get_access_token( - grant_type="refresh_token", - refresh_token=refresh_token - ) - except Exception as error: - print(error) - return False - - yandex_response = result["content"] - - if ("error" in yandex_response): - return False +def message_grant_access_code( + chat_id: int, + url: str, + lifetime_in_seconds: int +) -> None: + open_link_button_text = "Grant access" + revoke_command = CommandName.YD_REVOKE.value + yandex_passport_url = "https://passport.yandex.ru/profile" + source_code_url = current_app.config["PROJECT_URL_FOR_CODE"] + privacy_policy_url = absolute_url_for("legal.privacy_policy") + terms_url = absolute_url_for("legal.terms_and_conditions") + lifetime_in_minutes = int(lifetime_in_seconds / 60) - yd_token.clear_insert_token() - yd_token.set_access_token( - yandex_response["access_token"] - ) - yd_token.access_token_type = ( - yandex_response["token_type"] - ) - yd_token.access_token_expires_in = ( - yandex_response["expires_in"] + telegram.send_message( + chat_id=chat_id, + parse_mode="HTML", + disable_web_page_preview=True, + text=( + f'Open special link by pressing on "{open_link_button_text}" ' + "button and grant me access to your Yandex.Disk." + "\n\n" + "IMPORTANT: don't give this link to anyone, " + "because it contains your secret information." + "\n\n" + f"This link will expire in {lifetime_in_minutes} minutes." + "\n" + "This link leads to Yandex page. After granting access, " + "you will need to send me the issued code." + "\n\n" + "It is safe to give the access?" + "\n" + "Yes! I'm getting access only to your Yandex.Disk, " + "not to your account. You can revoke my access at any time " + f"with {revoke_command} or in your " + f'Yandex profile. ' + "By the way, i'm " + f'open-source ' + "and you can make sure that your data will be safe. " + "You can even create your own bot with my functionality " + "if using me makes you feel uncomfortable (:" + "\n\n" + "By using me, you accept " + f'Privacy Policy and ' + f'Terms of service. ' + ), + reply_markup={ + "inline_keyboard": [ + [ + { + "text": open_link_button_text, + "url": url + } + ] + ] + } ) - yd_token.set_refresh_token( - yandex_response["refresh_token"] + # TODO: + # React on press of inline keyboard button + # (https://core.telegram.org/bots/api#callbackquery), + # not send separate message immediately. + # But it requires refactoring of dispatcher and others. + # At the moment let it be implemented as it is, + # because "Console Client" is mostly for developers, not users. + telegram.send_message( + chat_id=chat_id, + text=( + "Open this link, grant me an access " + "and then send me a code" + ) ) - db.session.commit() - return True + +# endregion diff --git a/src/blueprints/telegram_bot/webhook/commands/yd_revoke.py b/src/blueprints/telegram_bot/webhook/commands/yd_revoke.py index ff113b0..20ad565 100644 --- a/src/blueprints/telegram_bot/webhook/commands/yd_revoke.py +++ b/src/blueprints/telegram_bot/webhook/commands/yd_revoke.py @@ -1,53 +1,85 @@ from flask import g -from src.database import db +from src.extensions import db from src.api import telegram -from src.blueprints.utils import get_current_datetime -from .common.decorators import get_db_data -from .common.responses import request_private_chat -from . import CommandsNames +from src.blueprints._common.utils import get_current_datetime +from src.blueprints.telegram_bot._common.yandex_oauth import YandexOAuthClient +from ._common.decorators import ( + get_db_data, + register_guest +) +from ._common.responses import ( + request_private_chat, + cancel_command +) +from src.blueprints.telegram_bot._common.command_names import CommandName +class YandexOAuthRemoveClient(YandexOAuthClient): + def clear_access_token(self, db_user) -> None: + super().clear_access_token(db_user) + db.session.commit() + + +@register_guest @get_db_data -def handle(): +def handle(*args, **kwargs): """ - Handles `/yandex_disk_revoke` command. + Handles `/revoke_access` command. Revokes bot access to user Yandex.Disk. """ - user = g.db_user - incoming_chat = g.db_chat private_chat = g.db_private_chat - if (private_chat is None): - return request_private_chat(incoming_chat.telegram_id) + if private_chat is None: + incoming_chat_id = kwargs.get( + "chat_id", + g.db_chat.telegram_id + ) + + return request_private_chat(incoming_chat_id) + + user = g.db_user + chat_id = private_chat.telegram_id + client = YandexOAuthRemoveClient() if ( (user is None) or - (user.yandex_disk_token is None) or - (not user.yandex_disk_token.have_access_token()) + not client.have_valid_access_token(user) ): - telegram.send_message( - chat_id=private_chat.telegram_id, - text=( - "You don't granted me access to your Yandex.Disk." - "\n" - f"You can do that with {CommandsNames.YD_AUTH.value}" - ) - ) + return message_dont_have_access_token(chat_id) + + try: + client.clear_access_token(user) + except Exception as error: + cancel_command(chat_id) + raise error + + message_access_token_removed(chat_id) - return - user.yandex_disk_token.clear_all_tokens() - db.session.commit() +# region Messages - current_datetime = get_current_datetime() - date = current_datetime["date"] - time = current_datetime["time"] - timezone = current_datetime["timezone"] +def message_dont_have_access_token(chat_id: int) -> None: telegram.send_message( - chat_id=private_chat.telegram_id, + chat_id=chat_id, + text=( + "You don't granted me access to your Yandex.Disk." + "\n" + f"You can do that with {CommandName.YD_AUTH.value}" + ) + ) + + +def message_access_token_removed(chat_id: int) -> None: + now = get_current_datetime() + date = now["date"] + time = now["time"] + timezone = now["timezone"] + + telegram.send_message( + chat_id=chat_id, parse_mode="HTML", disable_web_page_preview=True, text=( @@ -58,5 +90,10 @@ def handle(): "\n\n" "Don't forget to do that in your " 'Yandex Profile.' + "\n" + f"To grant access again use {CommandName.YD_AUTH.value}" ) ) + + +# endregion diff --git a/src/blueprints/telegram_bot/webhook/dispatcher.py b/src/blueprints/telegram_bot/webhook/dispatcher.py new file mode 100644 index 0000000..48e81d0 --- /dev/null +++ b/src/blueprints/telegram_bot/webhook/dispatcher.py @@ -0,0 +1,406 @@ +from typing import ( + Union, + Callable, + Set +) +from collections import deque +import traceback + +from flask import current_app +from src.blueprints.telegram_bot._common.stateful_chat import ( + stateful_chat_is_enabled, + get_disposable_handler, + delete_disposable_handler, + get_subscribed_handlers, + set_user_chat_data, + get_user_chat_data +) +from src.blueprints.telegram_bot._common.telegram_interface import ( + Message as TelegramMessage +) +from src.blueprints.telegram_bot._common.command_names import CommandName +from . import commands +from .dispatcher_events import ( + DispatcherEvent, + RouteSource +) + + +def intellectual_dispatch( + message: TelegramMessage +) -> Callable: + """ + Intellectual dispatch to handlers of a message. + Provides support for stateful chat (if Redis is enabled). + + Priority of handlers: + 1) if disposable handler exists and message events matches, + then only that handler will be called and after removed. + That disposable handler will be associated with message date. + 2) if subscribed handlers exists, then only ones with events + matched to message events will be called. If nothing is matched, + then forwarding to № 3 + 3) attempt to get first `bot_command` entity from message. + It will be treated as direct command. That direct command will be + associated with message date. If nothing found, then forwarding to № 4 + 4) attempt to get command based on message date. + For example, when `/upload_photo` was used for message that was + sent on `1607677734` date, and after that separate message was + sent on same `1607677734` date, then it is the case. + If nothing found, then forwarding to № 5 + 5) guessing of command that user assumed based on + content of message + + Events matching: + - if at least one event matched, then that handler will be + marked as "matched". + + If stateful chat not enabled, then № 1 and № 2 will be skipped. + + Note: there can be multiple handlers picked for single message. + Order of execution not determined. + + :param message: + Incoming Telegram message. + + :returns: + It is guaranteed that most appropriate callable handler that + not raises an error will be returned. Function arguments already + configured, but you can also provided your own through `*args` + and `**kwargs`. You should call this function in order to run + handlers (there can be multiple handlers in one return function). + """ + user_id = message.get_user().id + chat_id = message.get_chat().id + message_date = message.get_date() + is_stateful_chat = stateful_chat_is_enabled() + disposable_handler = None + subscribed_handlers = None + + if is_stateful_chat: + disposable_handler = get_disposable_handler(user_id, chat_id) + subscribed_handlers = get_subscribed_handlers(user_id, chat_id) + + message_events = ( + detect_events(message) if ( + disposable_handler or + subscribed_handlers + ) else None + ) + handler_names = deque() + route_source = None + + if disposable_handler: + match = match_events( + message_events, + disposable_handler["events"] + ) + + if match: + route_source = RouteSource.DISPOSABLE_HANDLER + handler_names.append(disposable_handler["name"]) + delete_disposable_handler(user_id, chat_id) + + if ( + subscribed_handlers and + not handler_names + ): + for handler in subscribed_handlers: + match = match_events( + message_events, + handler["events"] + ) + + if match: + route_source = RouteSource.SUBSCRIBED_HANDLER + handler_names.append(handler["name"]) + + if not handler_names: + command = message.get_entity_value("bot_command") + + if command: + route_source = RouteSource.DIRECT_COMMAND + handler_names.append(command) + + should_bind_command_to_date = ( + is_stateful_chat and + ( + route_source in + ( + RouteSource.DISPOSABLE_HANDLER, + RouteSource.DIRECT_COMMAND + ) + ) + ) + should_get_command_by_date = ( + is_stateful_chat and + (not handler_names) + ) + + if should_bind_command_to_date: + # we expect only one active command + command = handler_names[0] + + # we need to handle cases when user forwards + # many separate messages (one with direct command and + # others without any command but with some attachments). + # These messages will be sended by Telegram one by one + # (it is means we got separate direct command and + # separate attachments without that any commands). + # We also using `RouteSource.DISPOSABLE_HANDLER` + # because user can start command without any attachments, + # but forward multiple attachments at once or send + # media group (media group messages have same date). + bind_command_to_date( + user_id, + chat_id, + message_date, + command + ) + elif should_get_command_by_date: + command = get_command_by_date( + user_id, + chat_id, + message_date + ) + + if command: + route_source = RouteSource.SAME_DATE_COMMAND + handler_names.append(command) + + if not handler_names: + route_source = RouteSource.GUESSED_COMMAND + handler_names.append(guess_bot_command(message)) + + def method(*args, **kwargs): + for handler_name in handler_names: + handler_method = direct_dispatch(handler_name) + + try: + handler_method( + *args, + **kwargs, + user_id=user_id, + chat_id=chat_id, + message=message, + route_source=route_source, + message_events=message_events + ) + except Exception as error: + print( + f"{handler_name}: {error}", + "\n", + traceback.format_exc() + ) + + return method + + +def direct_dispatch( + command: Union[CommandName, str], + fallback: Callable = commands.unknown_handler +) -> Callable: + """ + Direct dispatch to handler of the command. + i.e., it doesn't uses any guessing or stateful chats, + it is just direct route (command_name -> command_handler). + + :param command: + Name of command to dispatch to. + :param fallback: + Fallback handler that will be used in case + if command is unknown. + + :returns: + It is guaranteed that some callable handler will be returned. + It is handler for incoming command and you should call this. + """ + if isinstance(command, CommandName): + command = command.value + + routes = { + CommandName.START.value: commands.help_handler, + CommandName.HELP.value: commands.help_handler, + CommandName.ABOUT.value: commands.about_handler, + CommandName.SETTINGS.value: commands.settings_handler, + CommandName.YD_AUTH.value: commands.yd_auth_handler, + CommandName.YD_REVOKE.value: commands.yd_revoke_handler, + CommandName.UPLOAD_PHOTO.value: commands.upload_photo_handler, + CommandName.UPLOAD_FILE.value: commands.upload_file_handler, + CommandName.UPLOAD_AUDIO.value: commands.upload_audio_handler, + CommandName.UPLOAD_VIDEO.value: commands.upload_video_handler, + CommandName.UPLOAD_VOICE.value: commands.upload_voice_handler, + CommandName.UPLOAD_URL.value: commands.upload_url_handler, + CommandName.PUBLIC_UPLOAD_PHOTO.value: commands.public_upload_photo_handler, # noqa + CommandName.PUBLIC_UPLOAD_FILE.value: commands.public_upload_file_handler, # noqa + CommandName.PUBLIC_UPLOAD_AUDIO.value: commands.public_upload_audio_handler, # noqa + CommandName.PUBLIC_UPLOAD_VIDEO.value: commands.public_upload_video_handler, # noqa + CommandName.PUBLIC_UPLOAD_VOICE.value: commands.public_upload_voice_handler, # noqa + CommandName.PUBLIC_UPLOAD_URL.value: commands.public_upload_url_handler, # noqa + CommandName.CREATE_FOLDER.value: commands.create_folder_handler, + CommandName.PUBLISH.value: commands.publish_handler, + CommandName.UNPUBLISH.value: commands.unpublish_handler, + CommandName.SPACE_INFO.value: commands.space_info_handler, + CommandName.ELEMENT_INFO.value: commands.element_info_handler, + CommandName.DISK_INFO.value: commands.disk_info_handler, + CommandName.COMMANDS_LIST.value: commands.commands_list_handler + } + handler = routes.get(command, fallback) + + def method(*args, **kwargs): + handler(*args, **kwargs) + + return method + + +def guess_bot_command( + message: TelegramMessage, + fallback: CommandName = CommandName.HELP +) -> str: + """ + Tries to guess which bot command user + assumed based on content of a message. + + :param fallback: + Fallback command which will be returned if unable to guess. + + :returns: + Guessed bot command name based on a message. + """ + command = fallback + raw_data = message.raw_data + + if ("photo" in raw_data): + command = CommandName.UPLOAD_PHOTO + elif ("document" in raw_data): + command = CommandName.UPLOAD_FILE + elif ("audio" in raw_data): + command = CommandName.UPLOAD_AUDIO + elif ("video" in raw_data): + command = CommandName.UPLOAD_VIDEO + elif ("voice" in raw_data): + command = CommandName.UPLOAD_VOICE + elif (message.get_entity_value("url") is not None): + command = CommandName.UPLOAD_URL + + return command.value + + +def detect_events( + message: TelegramMessage +) -> Set[str]: + """ + :returns: + Detected dispatcher events. + See `DispatcherEvent` documentation for more. + Note: it is strings values, because these values + should be compared with Redis values, which is + also strings. + """ + events = set() + entities = message.get_entities() + photo, document, audio, video, voice = map( + lambda x: x in message.raw_data, + ("photo", "document", "audio", "video", "voice") + ) + url, hashtag, email, bot_command = map( + lambda x: any(e.type == x for e in entities), + ("url", "hashtag", "email", "bot_command") + ) + plain_text = message.get_plain_text() + + if photo: + events.add(DispatcherEvent.PHOTO.value) + + if document: + events.add(DispatcherEvent.FILE.value) + + if audio: + events.add(DispatcherEvent.AUDIO.value) + + if video: + events.add(DispatcherEvent.VIDEO.value) + + if voice: + events.add(DispatcherEvent.VOICE.value) + + if url: + events.add(DispatcherEvent.URL.value) + + if hashtag: + events.add(DispatcherEvent.HASHTAG.value) + + if email: + events.add(DispatcherEvent.EMAIL.value) + + if bot_command: + events.add(DispatcherEvent.BOT_COMMAND.value) + + if plain_text: + events.add(DispatcherEvent.PLAIN_TEXT.value) + + if not len(events): + events.add(DispatcherEvent.NONE.value) + + return events + + +def match_events( + a: Set[str], + b: Set[str] +) -> bool: + """ + Checks if two groups of events are matched. + + :returns: + `True` - match found, `False` otherwise. + """ + return any(x in b for x in a) + + +def bind_command_to_date( + user_id: int, + chat_id: int, + date: int, + command: str +) -> None: + """ + Binds command to date. + + You can use it to detect right command for messages + with same date but without specific command. + + - stateful chat should be enabled. + """ + key = f"dispatcher:date:{date}:command" + expire = current_app.config[ + "RUNTIME_SAME_DATE_COMMAND_EXPIRE" + ] + + set_user_chat_data( + user_id, + chat_id, + key, + command, + expire + ) + + +def get_command_by_date( + user_id: int, + chat_id: int, + date: int +) -> Union[str, None]: + """ + - stateful chat should be enabled. + + :returns: + Value that was set using `bind_command_to_date()`. + """ + key = f"dispatcher:date:{date}:command" + + return get_user_chat_data( + user_id, + chat_id, + key + ) diff --git a/src/blueprints/telegram_bot/webhook/dispatcher_events.py b/src/blueprints/telegram_bot/webhook/dispatcher_events.py new file mode 100644 index 0000000..08c656e --- /dev/null +++ b/src/blueprints/telegram_bot/webhook/dispatcher_events.py @@ -0,0 +1,68 @@ +""" +This code directly related to dispatcher (`dispatcher.py`). +However, it was extracted into separate file to +avoid circular imports. + +Many handlers using `DispatcherEvent`. Howerver, dispatcher +itself imports these handlers. So, circular import occurs. +Handlers may import entire dispatcher module, not only enum +(`import dispatcher`). But i don't like this approach, so, +dispatcher events were extracted into separate file. +Same logic for another enums. +""" + + +from enum import Enum, auto + + +class StringAutoName(Enum): + """ + `auto()` will return strings, not ints. + """ + @staticmethod + def _generate_next_value_(name, start, count, last_values): + return str(count) + + +class DispatcherEvent(StringAutoName): + """ + An event that was detected and fired by dispatcher. + + These events is primarily for Telegram messages. + Note: there can be multiple events in one message, + so, you always should operate with list of events, + not single event (instead you can use list with one element). + + Note: values of that enums are strings, not ints. + It is because it is expected that these values will be + compared with Redis values in future. Redis by default + returns strings. + """ + # Nothing of known events is detected + NONE = auto() + # Message contains plain text (non empty) + PLAIN_TEXT = auto() + PHOTO = auto() + VIDEO = auto() + FILE = auto() + AUDIO = auto() + VOICE = auto() + BOT_COMMAND = auto() + URL = auto() + HASHTAG = auto() + EMAIL = auto() + + +class RouteSource(Enum): + """ + Who initiated the route to a handler. + + For example, `DISPOSABLE_HANDLER` means a handler was + called because of `set_disposable_handler()` from + stateful chat module. + """ + DISPOSABLE_HANDLER = auto() + SUBSCRIBED_HANDLER = auto() + SAME_DATE_COMMAND = auto() + DIRECT_COMMAND = auto() + GUESSED_COMMAND = auto() diff --git a/src/blueprints/telegram_bot/webhook/views.py b/src/blueprints/telegram_bot/webhook/views.py index 004a89f..449e473 100644 --- a/src/blueprints/telegram_bot/webhook/views.py +++ b/src/blueprints/telegram_bot/webhook/views.py @@ -5,7 +5,8 @@ ) from src.blueprints.telegram_bot import telegram_bot_blueprint as bp -from . import commands, telegram_interface +from src.blueprints.telegram_bot._common import telegram_interface +from .dispatcher import intellectual_dispatch, direct_dispatch @bp.route("/webhook", methods=["POST"]) @@ -39,47 +40,21 @@ def webhook(): g.telegram_message = message g.telegram_user = message.get_user() g.telegram_chat = message.get_chat() - g.route_to = route_command - - command = message.get_entity_value("bot_command") - - if (command is None): - command = message.guess_bot_command() - - route_command(command) + g.direct_dispatch = direct_dispatch + + # We call this handler and do not handle any errors. + # We assume that all errors already was handeld by + # handlers, loggers, etc. + # WARNING: in case of any exceptions there will be + # 500 from a server. Telegram will send user message + # again and again until it get 200 from a server. + # So, it is important to always return 200 or return + # 500 and expect same message again + intellectual_dispatch(message)() return make_success_response() -def route_command(command: str) -> None: - """ - Routes command to specific handler. - """ - CommandNames = commands.CommandsNames - - if (isinstance(command, CommandNames)): - command = command.value - - routes = { - CommandNames.START.value: commands.help_handler, - CommandNames.HELP.value: commands.help_handler, - CommandNames.ABOUT.value: commands.about_handler, - CommandNames.SETTINGS.value: commands.settings_handler, - CommandNames.YD_AUTH.value: commands.yd_auth_handler, - CommandNames.YD_REVOKE.value: commands.yd_revoke_handler, - CommandNames.UPLOAD_PHOTO.value: commands.upload_photo_handler, - CommandNames.UPLOAD_FILE.value: commands.upload_file_handler, - CommandNames.UPLOAD_AUDIO.value: commands.upload_audio_handler, - CommandNames.UPLOAD_VIDEO.value: commands.upload_video_handler, - CommandNames.UPLOAD_VOICE.value: commands.upload_voice_handler, - CommandNames.UPLOAD_URL.value: commands.upload_url_handler, - CommandNames.CREATE_FOLDER.value: commands.create_folder_handler - } - method = routes.get(command, commands.unknown_handler) - - method() - - def make_error_response(): """ Creates error response for Telegram Webhook. diff --git a/src/blueprints/telegram_bot/yd_auth/exceptions.py b/src/blueprints/telegram_bot/yd_auth/exceptions.py deleted file mode 100644 index a9eda46..0000000 --- a/src/blueprints/telegram_bot/yd_auth/exceptions.py +++ /dev/null @@ -1,20 +0,0 @@ -class InvalidCredentials(Exception): - """ - Provided credentials is not valid. - """ - pass - - -class LinkExpired(Exception): - """ - Link is expired and not valid anymore. - """ - pass - - -class InvalidInsertToken(Exception): - """ - Provided `insert_token` is not valid with - `insert_token` from DB. - """ - pass diff --git a/src/blueprints/telegram_bot/yd_auth/views.py b/src/blueprints/telegram_bot/yd_auth/views.py index a5803c5..49121ca 100644 --- a/src/blueprints/telegram_bot/yd_auth/views.py +++ b/src/blueprints/telegram_bot/yd_auth/views.py @@ -1,146 +1,95 @@ -import base64 - from flask import ( request, abort, - render_template, - current_app + render_template ) -import jwt -from src.database import ( - db, - UserQuery, - ChatQuery -) -from src.api import yandex, telegram +from src.api import telegram +from src.database import ChatQuery +from src.blueprints._common.utils import get_current_datetime from src.blueprints.telegram_bot import telegram_bot_blueprint as bp -from src.blueprints.utils import ( - get_current_datetime -) -from src.blueprints.telegram_bot.webhook.commands import CommandsNames -from .exceptions import ( - InvalidCredentials, - LinkExpired, - InvalidInsertToken -) +from src.blueprints.telegram_bot._common import yandex_oauth +from src.blueprints.telegram_bot._common.command_names import CommandName @bp.route("/yandex_disk_authorization") def yd_auth(): """ - Handles user redirect from Yandex OAuth page. + Handles user redirect from Yandex.OAuth page + (Auto Code method). """ - if (is_error_request()): - return handle_error() - elif (is_success_request()): + if is_success_request(): return handle_success() + elif is_error_request(): + return handle_error() else: abort(400) -def is_error_request() -> bool: +def is_success_request() -> bool: """ - :returns: Incoming request is a failed user authorization. + :returns: + Incoming request is a successful user authorization. """ state = request.args.get("state", "") - error = request.args.get("error", "") + code = request.args.get("code", "") return ( len(state) > 0 and - len(error) > 0 + len(code) > 0 ) -def is_success_request() -> bool: +def is_error_request() -> bool: """ - :returns: Incoming request is a successful user authorization. + :returns: + Incoming request is a failed user authorization. """ state = request.args.get("state", "") - code = request.args.get("code", "") + error = request.args.get("error", "") return ( len(state) > 0 and - len(code) > 0 + len(error) > 0 ) -def handle_error(): - """ - Handles failed user authorization. - """ - try: - db_user = get_db_user() - db_user.yandex_disk_token.clear_insert_token() - db.session.commit() - except Exception: - pass - - return create_error_response() - - def handle_success(): """ Handles success user authorization. """ - db_user = None + state = request.args["state"] + code = request.args["code"] + client = yandex_oauth.YandexOAuthAutoCodeClient() + result = None try: - db_user = get_db_user() - except InvalidCredentials: + result = client.after_success_redirect(state, code) + except yandex_oauth.InvalidState: return create_error_response("invalid_credentials") - except LinkExpired: + except yandex_oauth.ExpiredInsertToken: return create_error_response("link_expired") - except InvalidInsertToken: + except yandex_oauth.InvalidInsertToken: return create_error_response("invalid_insert_token") except Exception as error: print(error) return create_error_response("internal_server_error") - code = request.args["code"] - yandex_response = None - - try: - yandex_response = yandex.get_access_token( - grant_type="authorization_code", - code=code - )["content"] - except Exception as error: - print(error) - return create_error_response("internal_server_error") - - if ("error" in yandex_response): - db_user.yandex_disk_token.clear_all_tokens() - db.session.commit() - + if not result["ok"]: return create_error_response( error_code="internal_server_error", - raw_error_title=yandex_response["error"], - raw_error_description=yandex_response.get("error_description") + raw_error_title=result["error"], + raw_error_description=result.get("error_description") ) - db_user.yandex_disk_token.clear_insert_token() - db_user.yandex_disk_token.set_access_token( - yandex_response["access_token"] - ) - db_user.yandex_disk_token.access_token_type = ( - yandex_response["token_type"] - ) - db_user.yandex_disk_token.access_token_expires_in = ( - yandex_response["expires_in"] - ) - db_user.yandex_disk_token.set_refresh_token( - yandex_response["refresh_token"] - ) - db.session.commit() + user = result["user"] + private_chat = ChatQuery.get_private_chat(user.id) - private_chat = ChatQuery.get_private_chat(db_user.id) - - if (private_chat): - current_datetime = get_current_datetime() - date = current_datetime["date"] - time = current_datetime["time"] - timezone = current_datetime["timezone"] + if private_chat: + now = get_current_datetime() + date = now["date"] + time = now["time"] + timezone = now["timezone"] telegram.send_message( chat_id=private_chat.telegram_id, @@ -152,30 +101,60 @@ def handle_success(): f"on {date} at {time} {timezone}." "\n\n" "If it wasn't you, then detach this access with " - f"{CommandsNames.YD_REVOKE.value}" + f"{CommandName.YD_REVOKE.value}" ) ) return create_success_response() +def handle_error(): + """ + Handles failed user authorization. + """ + state = request.args["state"] + client = yandex_oauth.YandexOAuthAutoCodeClient() + + try: + client.after_error_redirect(state) + except Exception: + # we do not care about any errors at that stage + pass + + return create_error_response() + + +def create_success_response(): + """ + :returns: + Rendered template for success page. + """ + return render_template( + "telegram_bot/yd_auth/success.html" + ) + + def create_error_response( error_code: str = None, raw_error_title: str = None, raw_error_description: str = None ): """ - :param error_code: Name of error for user friendly - information. If not specified, then defaults to + :param error_code: + Name of error for user friendly information. + If not specified, then defaults to `error` argument from request. - :param raw_error_title: Raw error title for - debugging purposes. If not specified, then defaults to + :param raw_error_title: + Raw error title for debugging purposes. + If not specified, then defaults to `error_code` argument. - :param raw_error_description: Raw error description - for debugging purposes. If not specified, then defaults to + :param raw_error_description: + Raw error description for debugging purposes. + If not specified, then defaults to `error_description` argument from request. - :returns: Rendered template for error page. + :returns: + Rendered template for error page. """ possible_errors = { "access_denied": { @@ -188,7 +167,7 @@ def create_error_response( "unauthorized_client": { "title": "Application is unavailable", "description": ( - "There is a problems with the me. " + "There is a problems with me. " "Try later please." ) }, @@ -223,7 +202,7 @@ def create_error_response( error = request.args.get("error") error_description = request.args.get("error_description") - if (error_code is None): + if error_code is None: error_code = error error_info = possible_errors.get(error_code, {}) @@ -237,84 +216,3 @@ def create_error_response( raw_error_description=(raw_error_description or error_description), raw_state=state ) - - -def create_success_response(): - """ - :returns: Rendered template for success page. - """ - return render_template( - "telegram_bot/yd_auth/success.html" - ) - - -def get_db_user(): - """ - - `insert_token` will be checked. If it is not valid, - an error will be thrown. You shouldn't clear any tokens - in case of error, because provided tokens is not known - to attacker (potential). - - you shouldn't try to avoid checking logic! It is really - unsafe to access DB user without `insert_token`. - - :returns: User from DB based on incoming `state` from request. - This user have `yandex_disk_token` property which is - not `None`. - - :raises InvalidCredentials: If `state` have invalid - data or user not found in DB. - :raises LinkExpired: Requested link is expired and - not valid anymore. - :raises InvalidInsertToken: Provided `insert_token` - is not valid. - """ - base64_state = request.args["state"] - encoded_state = None - decoded_state = None - - try: - encoded_state = base64.urlsafe_b64decode( - base64_state.encode() - ).decode() - except Exception: - raise InvalidCredentials() - - try: - decoded_state = jwt.decode( - encoded_state, - current_app.secret_key.encode(), - algorithm="HS256" - ) - except Exception: - raise InvalidCredentials() - - incoming_user_id = decoded_state.get("user_id") - incoming_insert_token = decoded_state.get("insert_token") - - if ( - incoming_user_id is None or - incoming_insert_token is None - ): - raise InvalidCredentials() - - db_user = UserQuery.get_user_by_id(int(incoming_user_id)) - - if ( - db_user is None or - # for some reason `yandex_disk_token` not created, - # it is not intended behavior. - db_user.yandex_disk_token is None - ): - raise InvalidCredentials() - - db_insert_token = None - - try: - db_insert_token = db_user.yandex_disk_token.get_insert_token() - except Exception: - raise LinkExpired() - - if (incoming_insert_token != db_insert_token): - raise InvalidInsertToken() - - return db_user diff --git a/src/blueprints/utils.py b/src/blueprints/utils.py deleted file mode 100644 index 60d4ca8..0000000 --- a/src/blueprints/utils.py +++ /dev/null @@ -1,41 +0,0 @@ -from datetime import datetime, timezone - -from flask import ( - current_app, - url_for -) - - -def absolute_url_for(endpoint: str, **kwargs) -> str: - """ - Implements Flask `url_for`, but by default - creates absolute URL (`_external` and `_scheme`) with - `PREFERRED_URL_SCHEME` scheme. - - - you can specify these parameters to change behavior. - - https://flask.palletsprojects.com/en/1.1.x/api/#flask.url_for - """ - if ("_external" not in kwargs): - kwargs["_external"] = True - - if ("_scheme" not in kwargs): - kwargs["_scheme"] = current_app.config["PREFERRED_URL_SCHEME"] - - return url_for(endpoint, **kwargs) - - -def get_current_datetime() -> dict: - """ - :returns: Information about current date and time. - """ - current_datetime = datetime.now(timezone.utc) - current_date = current_datetime.strftime("%d.%m.%Y") - current_time = current_datetime.strftime("%H:%M:%S") - current_timezone = current_datetime.strftime("%Z") - - return { - "date": current_date, - "time": current_time, - "timezone": current_timezone - } diff --git a/src/configs/flask.py b/src/configs/flask.py index 26802ab..e383c63 100644 --- a/src/configs/flask.py +++ b/src/configs/flask.py @@ -3,21 +3,70 @@ """ import os +from enum import Enum, auto from dotenv import load_dotenv -load_dotenv() +def load_env(): + config_name = os.getenv("CONFIG_NAME") + file_names = { + "production": ".env.production", + "development": ".env.development", + "testing": ".env.testing" + } + file_name = file_names.get(config_name) + + if file_name is None: + raise Exception( + "Unable to map configuration name and .env.* files. " + "Did you forget to set env variables?" + ) + + load_dotenv(file_name) + + +load_env() + + +class YandexOAuthAPIMethod(Enum): + """ + Which method to use for Yandex OAuth API. + """ + # When user give access, he will be redirected + # to the app site, and app will extract code + # automatically. + # https://yandex.ru/dev/oauth/doc/dg/reference/auto-code-client.html + AUTO_CODE_CLIENT = auto() + + # When user give access, he will see code, and + # that code user should manually send to the Telegram bot. + # This method intended for cases when you don't have + # permanent domain name (for example, when testing with `ngrok`) + # or when you want to hide it. + # `AUTO_CODE_CLIENT` provides better UX. + # https://yandex.ru/dev/oauth/doc/dg/reference/console-client.html/ + CONSOLE_CLIENT = auto() class Config: """ Notes: - - keep in mind that Heroku have 30 seconds request timeout. - So, if your configuration value can exceed 30 seconds, then - request will be terminated by Heroku. + - don't remove any keys from this configuration, because code + logic can depend on this. Instead, set "off" value (if code logic + supports it); or set empty value and edit code logic to handle + such values. + - keep in mind that Telegram, Heroku, etc. have request timeout. + It is about 30 seconds, but actual value can be different. + If you don't end current request in a long time, then it will + be force closed. Telegram will send new request in that case. + Try to always use background task queue, not block current thread. + If you have no opportunity to use background task queue, then + change current configuration in order request with blocked thread + will be not able to take long time to complete. """ - # Project + # region Project + # name of app that will be used in HTML and so on PROJECT_APP_NAME = "Yandex.Disk Telegram Bot" PROJECT_AUTHOR = "Sergey Kuznetsov" @@ -25,54 +74,178 @@ class Config: PROJECT_URL_FOR_ISSUE = "https://github.com/Amaimersion/yandex-disk-telegram-bot/issues/new?template=bug_report.md" # noqa PROJECT_URL_FOR_REQUEST = "https://github.com/Amaimersion/yandex-disk-telegram-bot/issues/new?template=feature_request.md" # noqa PROJECT_URL_FOR_QUESTION = "https://github.com/Amaimersion/yandex-disk-telegram-bot/issues/new?template=question.md" # noqa + PROJECT_URL_FOR_BOT = "https://t.me/Ya_Disk_Bot" + + # endregion + + # region Runtime (interaction of bot with user, behavior of bot, etc.) + + # Default value (in seconds) when setted but unused + # disposable handler should expire and be removed. + # Example: user send `/create_folder` but didn't send + # any data for that command; bot will handle next message + # as needed data for that command; if user don't send any + # data in 10 minutes, then this handler will be removed from queue. + # Keep in mind that it is only recommended default value, + # specific handler can use it own expiration time and ignore + # this value at all. + # Set to 0 to disable expiration + RUNTIME_DISPOSABLE_HANDLER_EXPIRE = 60 * 10 + + # Dispatcher will bind command to message date. + # How long this data should be stored. In seconds. + # We don't need to memorize it for a long, because + # bot expects messages with exact same date to be sent + # in a short period of time (for example, "Forward" sents + # messages one by one as fast as server process them; + # or user uploads all files as media group within 1 minute). + RUNTIME_SAME_DATE_COMMAND_EXPIRE = 60 * 2 + + # RQ (background tasks queue) is enabled. + # Also depends on `REDIS_URL` + RUNTIME_RQ_ENABLED = True + + # Maximum runtime of uploading process in `/upload` + # before it’s interrupted. In seconds. + # This value shouldn't be less than + # `YANDEX_DISK_API_CHECK_OPERATION_STATUS_MAX_ATTEMPTS` * + # `YANDEX_DISK_API_CHECK_OPERATION_STATUS_INTERVAL`. + # This timeout should be used only for force quit if + # uploading function start behave incorrectly. + # Use `MAX_ATTEMPTS` and `INTERVAL` for expected quit. + # Applied only if task queue (RQ, for example) is enabled + RUNTIME_UPLOAD_WORKER_JOB_TIMEOUT = 30 + + # Maximum queued time of upload function before it's discarded. + # "Queued" means function awaits execution. + # In seconds. `None` for infinite awaiting. + # Applied only if task queue (RQ, for example) is enabled + RUNTIME_UPLOAD_WORKER_UPLOAD_TTL = None + + # How long successful result of uploading is kept. + # In seconds. + # Applied only if task queue (RQ, for example) is enabled + RUNTIME_UPLOAD_WORKER_RESULT_TTL = 0 + + # How long failed result of uploading is kept. + # "Failed result" means function raises an error, + # not any logical error returns from function. + # In seconds. + # Applied only if task queue (RQ, for example) is enabled + RUNTIME_UPLOAD_WORKER_FAILURE_TTL = 0 + + # See `RUNTIME_UPLOAD_WORKER_JOB_TIMEOUT` documentation. + # This value is for `/element_info` worker. + RUNTIME_ELEMENT_INFO_WORKER_JOB_TIMEOUT = 5 + + # See `RUNTIME_UPLOAD_WORKER_UPLOAD_TTL` documentation. + # This value is for `/element_info` worker. + # Because worker is used only to send preview, + # we will use small TTL, because if all workers + # are busy, most likely it can take a long time + # before preview will be sended. There is no point + # to send preview after 5 minutes - preview should + # be sended either now or never. + RUNTIME_ELEMENT_INFO_WORKER_TTL = 10 + + # See `RUNTIME_UPLOAD_WORKER_JOB_TIMEOUT` documentation. + # This value is for `/space_info` worker. + RUNTIME_SPACE_INFO_WORKER_TIMEOUT = 5 + + # endregion + + # region Flask - # Flask DEBUG = False TESTING = False SECRET_KEY = os.getenv("FLASK_SECRET_KEY") - # Flask SQLAlchemy - SQLALCHEMY_DATABASE_URI = os.getenv("DATABASE_URL") + # endregion + + # region Flask SQLAlchemy + + SQLALCHEMY_DATABASE_URI = os.getenv( + "DATABASE_URL", + "sqlite:///temp.sqlite" + ) SQLALCHEMY_TRACK_MODIFICATIONS = False - # Telegram API + # endregion + + # region Redis + + REDIS_URL = os.getenv("REDIS_URL") + + # endregion + + # region Telegram API + # stop waiting for a Telegram response # after a given number of seconds TELEGRAM_API_TIMEOUT = 5 - # Yandex OAuth API + # maximum file size in bytes that bot + # can handle by itself. + # It is Telegram limit, not bot. + # Binary system should be used, not decimal. + # For example, MebiBytes (M = 1024 * 1024), + # not MegaBytes (MB = 1000 * 1000). + # In Linux you can use `truncate -s 20480K test.txt` + # to create exactly 20M file + TELEGRAM_API_MAX_FILE_SIZE = 20 * 1024 * 1024 + + # endregion + + # region Yandex OAuth API + # stop waiting for a Yandex response # after a given number of seconds YANDEX_OAUTH_API_TIMEOUT = 15 - # Yandex.Disk API - # stop waiting for a Yandex response - # after a given number of seconds - YANDEX_DISK_API_TIMEOUT = 5 + # see `YandexOAuthAPIMethod` for more + YANDEX_OAUTH_API_METHOD = YandexOAuthAPIMethod.AUTO_CODE_CLIENT + # `insert_token` (controls `INSERT` operation) - # will contain n random bytes. Each byte will be + # will contain N random bytes. Each byte will be # converted to two hex digits - YANDEX_DISK_API_INSERT_TOKEN_BYTES = 8 + YANDEX_OAUTH_API_INSERT_TOKEN_BYTES = 8 + # lifetime of `insert_token` in seconds starting # from date of issue. It is better to find # best combination between `bytes` and `lifetime` - YANDEX_DISK_API_INSERT_TOKEN_LIFETIME = 60 * 10 + YANDEX_OAUTH_API_INSERT_TOKEN_LIFETIME = 60 * 10 + + # endregion + + # region Yandex.Disk API + + # stop waiting for a Yandex response + # after a given number of seconds + YANDEX_DISK_API_TIMEOUT = 5 + # maximum number of checks of operation status # (for example, if file is downloaded by Yandex.Disk). # It is blocks request until check ending! YANDEX_DISK_API_CHECK_OPERATION_STATUS_MAX_ATTEMPTS = 5 + # interval in seconds between checks of operation status. # It is blocks request until check ending! # For example, if max. attempts is 5 and interval is 2, # then request will be blocked maximum for (5 * 2) seconds. YANDEX_DISK_API_CHECK_OPERATION_STATUS_INTERVAL = 2 + # in this folder files will be uploaded by default # if user not specified custom folder. YANDEX_DISK_API_DEFAULT_UPLOAD_FOLDER = "Telegram Bot" - # Google Analytics + # endregion + + # region Google Analytics + GOOGLE_ANALYTICS_UA = os.getenv("GOOGLE_ANALYTICS_UA") + # endregion + class ProductionConfig(Config): DEBUG = False @@ -85,16 +258,13 @@ class ProductionConfig(Config): class DevelopmentConfig(Config): DEBUG = True TESTING = False - SECRET_KEY = "q8bjscr0sLmAf50gXRFaIghoS7BvDd4Afxo2RjT3r3E=" - SQLALCHEMY_DATABASE_URI = "sqlite:///development.sqlite" SQLALCHEMY_ECHO = "debug" + YANDEX_OAUTH_API_METHOD = YandexOAuthAPIMethod.CONSOLE_CLIENT class TestingConfig(Config): DEBUG = False TESTING = True - SECRET_KEY = "ReHdIY8zGRQUJRTgxo_zeKiv3MjIU-OYBD66GlW9ZKw=" - SQLALCHEMY_DATABASE_URI = "sqlite:///testing.sqlite" config = { diff --git a/src/configs/nginx.conf b/src/configs/nginx.conf index 2e64c37..ebff794 100644 --- a/src/configs/nginx.conf +++ b/src/configs/nginx.conf @@ -33,6 +33,10 @@ http { alias src/static/robots/robots.txt; } + location /favicon.ico { + alias src/static/favicons/favicon.ico; + } + location /static/ { root src; autoindex off; diff --git a/src/database/__init__.py b/src/database/__init__.py index ee22204..54e85fb 100644 --- a/src/database/__init__.py +++ b/src/database/__init__.py @@ -1,5 +1,3 @@ -from .database import db -from .migrate import migrate from .models import ( User, Chat, diff --git a/src/database/database.py b/src/database/database.py deleted file mode 100644 index 6d947de..0000000 --- a/src/database/database.py +++ /dev/null @@ -1,13 +0,0 @@ -from sqlalchemy.pool import NullPool - -from flask_sqlalchemy import SQLAlchemy - - -db = SQLAlchemy( - engine_options={ - # pooling will be disabled until that - # https://stackoverflow.com/questions/61197228/flask-gunicorn-gevent-sqlalchemy-postgresql-too-many-connections - # will be resolved - "poolclass": NullPool - } -) diff --git a/src/database/migrate.py b/src/database/migrate.py deleted file mode 100644 index 111e719..0000000 --- a/src/database/migrate.py +++ /dev/null @@ -1,10 +0,0 @@ -from flask_migrate import Migrate - -# we need to import every model in order Migrate knows them -from .models import * # noqa - - -migrate = Migrate( - compare_type=True, - render_as_batch=True -) diff --git a/src/database/models/chat.py b/src/database/models/chat.py index cc8350c..8891cef 100644 --- a/src/database/models/chat.py +++ b/src/database/models/chat.py @@ -1,6 +1,6 @@ from enum import IntEnum, unique -from src.database import db +from src.extensions import db @unique @@ -45,6 +45,7 @@ class Chat(db.Model): telegram_id = db.Column( db.BigInteger, unique=True, + index=True, nullable=False, comment="Unique ID to identificate chat in Telegram" ) diff --git a/src/database/models/user.py b/src/database/models/user.py index e1f7e8f..9aa8028 100644 --- a/src/database/models/user.py +++ b/src/database/models/user.py @@ -2,8 +2,8 @@ from sqlalchemy.sql import func -from src.database import db -from src.localization import SupportedLanguages +from src.extensions import db +from src.localization import SupportedLanguage @unique @@ -46,6 +46,7 @@ class User(db.Model): telegram_id = db.Column( db.Integer, unique=True, + index=True, nullable=False, comment="Unique ID to identificate user in Telegram" ) @@ -55,8 +56,8 @@ class User(db.Model): comment="User is bot in Telegram" ) language = db.Column( - db.Enum(SupportedLanguages), - default=SupportedLanguages.EN, + db.Enum(SupportedLanguage), + default=SupportedLanguage.EN, nullable=False, comment="Preferred language of user" ) @@ -112,7 +113,7 @@ def create_fake(): step=1 ) result.is_bot = (fake.pyint() % 121 == 0) - result.language = fake.random_element(list(SupportedLanguages)) + result.language = fake.random_element(list(SupportedLanguage)) result.group = ( fake.random_element(list(UserGroup)) if ( random_number % 20 == 0 diff --git a/src/database/models/yandex_disk_token.py b/src/database/models/yandex_disk_token.py index 8f4cc43..03245b1 100644 --- a/src/database/models/yandex_disk_token.py +++ b/src/database/models/yandex_disk_token.py @@ -8,7 +8,7 @@ InvalidToken as InvalidTokenFernetError ) -from src.database import db +from src.extensions import db class YandexDiskToken(db.Model): @@ -69,6 +69,7 @@ class YandexDiskToken(db.Model): db.Integer, db.ForeignKey("users.id"), unique=True, + index=True, nullable=False, comment="Tokens belongs to this user" ) diff --git a/src/extensions.py b/src/extensions.py new file mode 100644 index 0000000..5eb5cb0 --- /dev/null +++ b/src/extensions.py @@ -0,0 +1,124 @@ +""" +Flask extensions that used by the app. +They already preconfigured and should be just +initialized by the app. Optionally, at initialization +stage you can provide additional configuration. +These extensions extracted in separate module in order +to avoid circular imports. +""" + +from typing import ( + Union +) + +from flask import Flask +from flask_sqlalchemy import SQLAlchemy +from flask_migrate import Migrate +from sqlalchemy.pool import NullPool +import redis +from rq import Queue as RQ + + +# Database + +db = SQLAlchemy( + engine_options={ + # pooling will be disabled until that + # https://stackoverflow.com/questions/61197228/flask-gunicorn-gevent-sqlalchemy-postgresql-too-many-connections + # will be resolved + "poolclass": NullPool + } +) + + +# Migration + +migrate = Migrate( + compare_type=True, + render_as_batch=True +) + + +# Redis + +class FlaskRedis: + def __init__(self): + self._redis_client = None + + def __getattr__(self, name): + return getattr(self._redis_client, name) + + def __getitem__(self, name): + return self._redis_client[name] + + def __setitem__(self, name, value): + self._redis_client[name] = value + + def __delitem__(self, name): + del self._redis_client[name] + + @property + def connection(self) -> redis.Redis: + return self._redis_client + + @property + def is_enabled(self) -> bool: + return (self.connection is not None) + + def init_app(self, app: Flask, **kwargs) -> None: + self._redis_client = None + redis_server_url = app.config.get("REDIS_URL") + + if not redis_server_url: + return + + self._redis_client = redis.from_url( + redis_server_url, + decode_responses=True, + **kwargs + ) + + +redis_client: Union[redis.Redis, FlaskRedis] = FlaskRedis() + + +# Redis Queue + +class RedisQueue: + def __init__(self): + self._queue = None + + def __getattr__(self, name): + return getattr(self._queue, name) + + def __getitem__(self, name): + return self._queue[name] + + def __setitem__(self, name, value): + self._queue[name] = value + + def __delitem__(self, name): + del self._queue[name] + + @property + def is_enabled(self) -> bool: + return (self._queue is not None) + + def init_app( + self, + app: Flask, + redis_connection: redis.Redis, + **kwargs + ) -> None: + enabled = app.config.get("RUNTIME_RQ_ENABLED") + + if not enabled: + return + + self._queue = RQ( + connection=redis_connection, + name="default" + ) + + +task_queue: Union[RQ, RedisQueue] = RedisQueue() diff --git a/src/localization/__init__.py b/src/localization/__init__.py index 309ef19..c8230fb 100644 --- a/src/localization/__init__.py +++ b/src/localization/__init__.py @@ -1 +1 @@ -from .languages import SupportedLanguages +from .languages import SupportedLanguage diff --git a/src/localization/languages.py b/src/localization/languages.py index d7f3bd9..3b3ed06 100644 --- a/src/localization/languages.py +++ b/src/localization/languages.py @@ -2,9 +2,9 @@ @unique -class SupportedLanguages(IntEnum): +class SupportedLanguage(IntEnum): """ - Languages supported by app. + Language supported by the app. """ EN = 1 @@ -18,9 +18,9 @@ def get(ietf_tag: str) -> int: """ ietf_tag = ietf_tag.lower() languages = { - "en": SupportedLanguages.EN, - "en-us": SupportedLanguages.EN, - "en-gb": SupportedLanguages.EN + "en": SupportedLanguage.EN, + "en-us": SupportedLanguage.EN, + "en-gb": SupportedLanguage.EN } - return languages.get(ietf_tag, SupportedLanguages.EN) + return languages.get(ietf_tag, SupportedLanguage.EN) diff --git a/worker.py b/worker.py new file mode 100644 index 0000000..bd4642f --- /dev/null +++ b/worker.py @@ -0,0 +1,37 @@ +""" +Runs RQ worker. +""" + +import redis +from rq import Worker, Queue, Connection + +from src.app import create_app + + +def main(): + app = create_app() + redis_url = app.config.get("REDIS_URL") + + if not redis_url: + raise Exception("Redis URL is not specified") + + connection = redis.from_url(redis_url) + listen = ["default"] + + with Connection(connection): + # we should bind Flask app context + # to worker context in order worker + # have access to valid `current_app`. + # It will be not actual app that is + # currently being running, but app + # with same ENV configuration + with app.app_context(): + worker = Worker( + map(Queue, listen) + ) + + worker.work() + + +if __name__ == "__main__": + main()