diff --git a/.gitignore b/.gitignore index 533854a..decff03 100644 --- a/.gitignore +++ b/.gitignore @@ -1,8 +1,7 @@ # Created by .ignore support plugin (hsz.mobi) ### Project files # auth data -/tests/auth_data.py -/tests/captcha.jpg +.pytest_cache/ ### Python template # Byte-compiled / optimized / DLL files diff --git a/README.rst b/README.rst index 30b18f2..89cce4f 100644 --- a/README.rst +++ b/README.rst @@ -8,13 +8,13 @@ Features * support python 3.5+ versions * have only one dependency - ``aiohttp 3+`` * support two-factor authentication -* support socks proxy with ``aiosocksy`` +* support socks proxy with ``aiohttp-socks`` * support rate limit of requests * support Long Poll connection TODO ---- -* replace ``aiosocksy`` to ``aiohttp-socks`` +* need refactoring tests for ``AsyncVkExecuteRequestPool`` Install ------- @@ -109,9 +109,9 @@ Drivers .. code-block:: python - >>> driver = Socks5Driver(PROXY_ADDRESS, PORT) # 1234 is port - >>> driver = Socks5Driver(PROXY_ADDRESS, PORT, timeout=10) - >>> driver = Socks5Driver(PROXY_ADDRESS, PORT, PROXY_LOGIN, PROXY_PASSWORD, timeout=10) + >>> driver = ProxyDriver(PROXY_ADDRESS, PORT) # 1234 is port + >>> driver = ProxyDriver(PROXY_ADDRESS, PORT, timeout=10) + >>> driver = ProxyDriver(PROXY_ADDRESS, PORT, PROXY_LOGIN, PROXY_PASSWORD, timeout=10) How to use custom driver with session: @@ -125,7 +125,7 @@ How to use driver with own loop: >>> loop = asyncio.get_event_loop() >>> asyncio.set_event_loop(None) - >>> session = TokenSession(driver=HttpDriver(loop=loop)) # or Socks5Driver + >>> session = TokenSession(driver=HttpDriver(loop=loop)) # or ProxyDriver How to use driver with custom http session object: @@ -211,6 +211,14 @@ Use Session object >>> await lp.get_pts(need_ts=True) # return pts, ts 191231223, 1820350345 +You can iterate over events + +.. code-block:: python + + >>> async for event in lp.iter(): + ... print(event) + {"type":..., "object": {...}} + Notice that ``wait`` value only for long pool connection. Real pause could be more ``wait`` time because of need time @@ -225,7 +233,7 @@ Use exist API object .. code-block:: python >>> api = API(session) - >>> lp = BotsLongPoll(api, mode=2, group_id=1) # default wait=25 + >>> lp = BotsLongPoll(api, group_id=1) # default wait=25 >>> await lp.wait() {"ts":345,"updates":[...]} >>> await lp.wait() @@ -235,7 +243,7 @@ Use Session object .. code-block:: python - >>> lp = BotsLongPoll(session, mode=2, group_id=1) # default wait=25 + >>> lp = BotsLongPoll(session, group_id=1) # default wait=25 >>> await lp.wait() {"ts":78455,"updates":[...]} >>> await lp.get_pts() # return pts @@ -243,7 +251,56 @@ Use Session object >>> await lp.get_pts(need_ts=True) # return pts, ts 191231223, 1820350345 +BotsLongPoll supports iterating too + +.. code-block:: python + + >>> async for event in lp.iter(): + ... print(event) + {"type":..., "object": {...}} + Notice that ``wait`` value only for long pool connection. Real pause could be more ``wait`` time because of need time for authorization (if needed), reconnect and etc. + +Async execute request pool +-------------- +For documentation, see: https://vk.com/dev/execute + +.. code-block:: python + + from aiovk.pools import AsyncVkExecuteRequestPool + + async with AsyncVkExecuteRequestPool() as pool: + response = pool.add_call('users.get', 'YOUR_TOKEN', {'user_ids': 1}) + response2 = pool.add_call('users.get', 'YOUR_TOKEN', {'user_ids': 2}) + response3 = pool.add_call('users.get', 'ANOTHER_TOKEN', {'user_ids': 1}) + response4 = pool.add_call('users.get', 'ANOTHER_TOKEN', {'user_ids': -1}) + + >>> print(response.ok) + True + >>> print(response.result) + [{'id': 1, 'first_name': 'Павел', 'last_name': 'Дуров'}] + >>> print(response2.result) + [{'id': 2, 'first_name': 'Александра', 'last_name': 'Владимирова'}] + >>> print(response3.result) + [{'id': 1, 'first_name': 'Павел', 'last_name': 'Дуров'}] + >>> print(response4.ok) + False + >>> print(response4.error) + {'method': 'users.get', 'error_code': 113, 'error_msg': 'Invalid user id'} + +or + +.. code-block:: python + + from aiovk.pools import AsyncVkExecuteRequestPool + + pool = AsyncVkExecuteRequestPool() + response = pool.add_call('users.get', 'YOUR_TOKEN', {'user_ids': 1}) + response2 = pool.add_call('users.get', 'YOUR_TOKEN', {'user_ids': 2}) + response3 = pool.add_call('users.get', 'ANOTHER_TOKEN', {'user_ids': 1}) + response4 = pool.add_call('users.get', 'ANOTHER_TOKEN', {'user_ids': -1}) + await pool.execute() + ... diff --git a/aiovk/__init__.py b/aiovk/__init__.py index 377c7a3..04c28b1 100644 --- a/aiovk/__init__.py +++ b/aiovk/__init__.py @@ -1,4 +1,4 @@ -__version__ = '3.0.0' +__version__ = '4.0.0' from aiovk.sessions import ImplicitSession, TokenSession, AuthorizationCodeSession from aiovk.api import API diff --git a/aiovk/api.py b/aiovk/api.py index 92f1801..28d6929 100644 --- a/aiovk/api.py +++ b/aiovk/api.py @@ -24,12 +24,9 @@ def __getattr__(self, method_name): async def __call__(self, **method_args): timeout = method_args.pop('timeout', None) + need_raw_response = method_args.pop('raw_response', False) self._method_args = method_args - return await self._api._session.send_api_request( - self._method_name, - method_args, - timeout, - ) + return await self._api._session.send_api_request(self._method_name, method_args, timeout, need_raw_response) class LazyAPI: diff --git a/aiovk/drivers.py b/aiovk/drivers.py index 25b186b..a1c5032 100644 --- a/aiovk/drivers.py +++ b/aiovk/drivers.py @@ -1,8 +1,7 @@ import aiohttp try: - import aiosocksy - from aiosocksy.connector import ProxyConnector + from aiohttp_socks.connector import ProxyConnector except ImportError as e: ProxyConnector = None @@ -12,32 +11,32 @@ def __init__(self, timeout=10, loop=None): self.timeout = timeout self._loop = loop - async def json(self, url, params, timeout=None): - ''' + async def post_json(self, url, params, timeout=None): + """ :param params: dict of query params - :return: dict from json response - ''' + :return: http status code, dict from json response + """ raise NotImplementedError - async def get_text(self, url, params, timeout=None): - ''' + async def get_bin(self, url, params, timeout=None): + """ :param params: dict of query params - :return: http status code, text body of response - ''' + :return: http status code, binary body of response + """ raise NotImplementedError - async def get_bin(self, url, params, timeout=None): - ''' + async def get_text(self, url, params, timeout=None): + """ :param params: dict of query params - :return: http status code, binary body of response - ''' + :return: http status code, text body of response and redirect_url + """ raise NotImplementedError async def post_text(self, url, data, timeout=None): - ''' + """ :param data: dict pr string - :return: redirect url and text body of response - ''' + :return: http status code, text body of response and redirect url + """ raise NotImplementedError async def close(self): @@ -52,37 +51,36 @@ def __init__(self, timeout=10, loop=None, session=None): else: self.session = session - async def json(self, url, params, timeout=None): - # timeouts - https://docs.aiohttp.org/en/v3.0.0/client_quickstart.html#timeouts - async with self.session.get(url, params=params, timeout=timeout or self.timeout) as response: - return await response.json() + async def post_json(self, url, params, timeout=None): + async with self.session.post(url, data=params, timeout=timeout or self.timeout) as response: + return response.status, await response.json() - async def get_text(self, url, params, timeout=None): + async def get_bin(self, url, params, timeout=None): async with self.session.get(url, params=params, timeout=timeout or self.timeout) as response: - return response.status, await response.text() + return response.status, await response.read() - async def get_bin(self, url, params, timeout=None): + async def get_text(self, url, params, timeout=None): async with self.session.get(url, params=params, timeout=timeout or self.timeout) as response: - return await response.read() + return response.status, await response.text(), response.real_url async def post_text(self, url, data, timeout=None): async with self.session.post(url, data=data, timeout=timeout or self.timeout) as response: - return response._real_url, await response.text() + return response.status, await response.text(), response.real_url async def close(self): await self.session.close() -if ProxyConnector: - class Socks5Driver(HttpDriver): - connector = ProxyConnector +class ProxyDriver(HttpDriver): + connector = ProxyConnector - def __init__(self, address, port, login=None, password=None, timeout=10, loop=None): - addr = aiosocksy.Socks5Addr(address, port) - if login and password: - auth = aiosocksy.Socks5Auth(login, password=password) - else: - auth = None - conn = self.connector(proxy=addr, proxy_auth=auth, loop=loop) - session = aiohttp.ClientSession(connector=conn) - super().__init__(timeout, loop, session) + def __init__(self, address, port, login=None, password=None, timeout=10, **kwargs): + connector = ProxyConnector( + host=address, + port=port, + username=login, + password=password, + **kwargs + ) + session = aiohttp.ClientSession(connector=connector) + super().__init__(timeout, kwargs.get('loop'), session) diff --git a/aiovk/exceptions.py b/aiovk/exceptions.py index c56585a..c553de8 100644 --- a/aiovk/exceptions.py +++ b/aiovk/exceptions.py @@ -1,5 +1,4 @@ -import urllib.parse -from aiovk.utils import get_request_params +from urllib.parse import urlencode CAPTCHA_IS_NEEDED = 14 @@ -14,7 +13,7 @@ class VkAuthError(VkException): def __init__(self, error, description, url='', params=''): self.error = error self.description = description - self.url = "{}?{}".format(url, urllib.parse.urlencode(params)) + self.url = "{}?{}".format(url, urlencode(params)) def __str__(self): return self.description @@ -39,7 +38,7 @@ class VkAPIError(VkException): def __init__(self, error, url): self.error_code = error.get('error_code') self.error_msg = error.get('error_msg') - self.params = get_request_params(error.get('request_params')) + self.params = {param['key']: param['value'] for param in error.get('request_params', [])} self.url = url @@ -47,7 +46,7 @@ class VkLongPollError(VkException): def __init__(self, error, description, url='', params=''): self.error = error self.description = description - self.url = "{}?{}".format(url, urllib.parse.urlencode(params)) + self.url = "{}?{}".format(url, urlencode(params)) def __str__(self): return self.description diff --git a/aiovk/longpoll.py b/aiovk/longpoll.py index df55a17..f2df9f3 100644 --- a/aiovk/longpoll.py +++ b/aiovk/longpoll.py @@ -1,5 +1,6 @@ import json from abc import ABC, abstractmethod +from typing import Union, Optional from aiovk import API from aiovk.api import LazyAPI @@ -8,7 +9,8 @@ class BaseLongPoll(ABC): """Interface for all types of Longpoll API""" - def __init__(self, session_or_api, mode: int or list, wait: int=25, version: int=2, timeout: int=None): + def __init__(self, session_or_api, mode: Optional[Union[int, list]], + wait: int = 25, version: int = 2, timeout: int = None): """ :param session_or_api: session object or data for creating a new session :type session_or_api: BaseSession or API or LazyAPI @@ -30,16 +32,19 @@ def __init__(self, session_or_api, mode: int or list, wait: int=25, version: int self.base_params = { 'version': version, 'wait': wait, - 'mode': mode, 'act': 'a_check' } + + if mode is not None: + self.base_params['mode'] = mode + self.pts = None self.ts = None self.key = None self.base_url = None @abstractmethod - async def _get_long_poll_server(self, need_pts: bool=False) -> None: + async def _get_long_poll_server(self, need_pts: bool = False) -> None: """Send *.getLongPollServer request and update internal data :param need_pts: need return the pts field @@ -59,12 +64,12 @@ async def wait(self, need_pts=False) -> dict: } params.update(self.base_params) # invalid mimetype from server - code, response = await self.api._session.driver.get_text( + status, response, _ = await self.api._session.driver.get_text( self.base_url, params, timeout=2 * self.base_params['wait'] ) - if code == 403: + if status == 403: raise VkLongPollError(403, 'smth weth wrong', self.base_url + '/', params) response = json.loads(response) @@ -87,6 +92,12 @@ async def wait(self, need_pts=False) -> dict: self.base_url = None return await self.wait() + + async def iter(self): + while True: + response = await self.wait() + for event in response['updates']: + yield event async def get_pts(self, need_ts=False): if not self.base_url or not self.pts: @@ -99,6 +110,8 @@ async def get_pts(self, need_ts=False): class UserLongPoll(BaseLongPoll): """Implements https://vk.com/dev/using_longpoll""" + # False for testing + use_https = True async def _get_long_poll_server(self, need_pts=False): response = await self.api('messages.getLongPollServer', need_pts=int(need_pts), timeout=self.timeout) @@ -106,7 +119,7 @@ async def _get_long_poll_server(self, need_pts=False): self.ts = response['ts'] self.key = response['key'] # fucking differences between long poll methods in vk api! - self.base_url = 'https://{}'.format(response['server']) + self.base_url = f'http{"s" if self.use_https else ""}://{response["server"]}' class LongPoll(UserLongPoll): @@ -118,8 +131,8 @@ class LongPoll(UserLongPoll): class BotsLongPoll(BaseLongPoll): """Implements https://vk.com/dev/bots_longpoll""" - def __init__(self, session_or_api, mode, group_id, wait=25, version=1, timeout=None): - super().__init__(session_or_api, mode, wait, version, timeout) + def __init__(self, session_or_api, group_id, wait=25, version=1, timeout=None): + super().__init__(session_or_api, None, wait, version, timeout) self.group_id = group_id async def _get_long_poll_server(self, need_pts=False): diff --git a/aiovk/mixins.py b/aiovk/mixins.py index 3eb6638..5f17d38 100644 --- a/aiovk/mixins.py +++ b/aiovk/mixins.py @@ -1,33 +1,31 @@ -from aiovk.utils import TaskQueue, wait_free_slot +from aiovk.drivers import BaseDriver +from aiovk.shaping import TaskQueue, wait_free_slot -class LimitRateDriverMixin: - requests_per_period = 3 - period = 1 # seconds - - def __init__(self, *args, **kwargs): +class LimitRateDriverMixin(BaseDriver): + def __init__(self, *args, requests_per_period=3, period=1, **kwargs): super().__init__(*args, **kwargs) - self._queue = TaskQueue(self.requests_per_period, self.period) - - @wait_free_slot - async def json(self, *args, **kwargs): - return await super().json(*args, **kwargs) + self._queue = TaskQueue(requests_per_period, period) @wait_free_slot - async def get_text(self, *args, **kwargs): - return await super().get_text(*args, **kwargs) + async def post_json(self, *args, **kwargs): + return await super().post_json(*args, **kwargs) @wait_free_slot async def get_bin(self, *args, **kwargs): return await super().get_bin(*args, **kwargs) + @wait_free_slot + async def get_text(self, *args, **kwargs): + return await super().get_text(*args, **kwargs) + @wait_free_slot async def post_text(self, *args, **kwargs): return await super().post_text(*args, **kwargs) async def close(self): await super().close() - self._queue.canel() + self._queue.cancel() class SimpleImplicitSessionMixin: diff --git a/aiovk/pools.py b/aiovk/pools.py new file mode 100644 index 0000000..47aab1a --- /dev/null +++ b/aiovk/pools.py @@ -0,0 +1,143 @@ +import asyncio +import json +from collections import defaultdict +from dataclasses import dataclass +from typing import List, Dict, Optional + +from aiovk import TokenSession, API +from aiovk.exceptions import VkAuthError + + +class AsyncResult: + def __init__(self): + self._result = None + self.ready = False + self.error = None + + @property + def result(self): + return self._result + + @result.setter + def result(self, val): + self._result = val + self.ready = True + + @property + def ok(self): + return self.ready and not self.error + + +@dataclass +class VkCall: + method: str + method_args: dict + result: AsyncResult + + def get_execute_representation(self) -> str: + return f"API.{self.method}({json.dumps(self.method_args, ensure_ascii=False)})" + + +class AsyncVkExecuteRequestPool: + """ + Allows concatenation of api calls using one token into groups and execute each group of hits in + one request using `execute` method + """ + + def __init__(self, call_number_per_request=25, token_session_class=TokenSession): + self.token_session_class = token_session_class + self.call_number_per_request = call_number_per_request + self.pool: Dict[str, List[VkCall]] = defaultdict(list) + self.sessions = [] + + async def __aenter__(self): + return self + + async def __aexit__(self, *args, **kwargs): + await self.execute() + + async def execute(self): + try: + await self._execute() + await asyncio.gather(*[session.close() for session in self.sessions]) + finally: + self.pool.clear() + self.sessions.clear() + + async def _execute(self): + """ + Groups hits and executes them using the execute method, after execution the pool is cleared + """ + executed_pools = [] + for token, calls in self.pool.items(): + session = self.token_session_class(token) + self.sessions.append(session) + api = API(session) + + for methods_pool in chunks(calls, self.call_number_per_request): + executed_pools.append(VkExecuteMethodsPool(methods_pool).execute(api)) + await asyncio.gather(*executed_pools) + + def add_call(self, method, token, method_args=None) -> AsyncResult: + """ + Adds an any api method call to the execute pool + + :param method: api vk method name + :param token: session token + :param method_args: params + :return: object that will contain the result after the pool is closed + """ + if method_args is None: + method_args = {} + + result = None + # searching already added calls with equal token, method and values + for call in self.pool[token]: + if call.method == method and call.method_args == method_args: + result = call.result + break + if result: + return result + result = AsyncResult() + self.pool[token].append(VkCall(method=method, method_args=method_args, result=result)) + return result + + +class VkExecuteMethodsPool: + def __init__(self, pool: Optional[VkCall] = None): + if not pool: + pool = [] + self.pool: List[VkCall] = pool + + async def execute(self, api: API): + """ + Executes calls to the pool using the execute method and stores the results for each call + + :param api: API object to make the request + """ + methods = [call.get_execute_representation() for call in self.pool] + code = f"return [{','.join(methods)}];" + try: + response = await api.execute(code=code, raw_response=True) + except VkAuthError as e: + for call in self.pool: + call.result.error = { + 'method': call.method, + 'error_code': 5, + 'error_msg': e.description + } + return + errors = response.pop('execute_errors', [])[::-1] + response = response['response'] + + for call, result in zip(self.pool, response): + if result is False: + call.result.error = errors.pop() + else: + call.result.result = result + + +def chunks(lst, n): + """Yield successive n-sized chunks from lst.""" + for i in range(0, len(lst), n): + yield lst[i: i + n] diff --git a/aiovk/sessions.py b/aiovk/sessions.py index e06a22e..2f80798 100644 --- a/aiovk/sessions.py +++ b/aiovk/sessions.py @@ -1,5 +1,6 @@ import json from abc import ABC, abstractmethod +from typing import Tuple from urllib.parse import parse_qsl from yarl import URL @@ -25,13 +26,15 @@ async def close(self) -> None: """Perform the actions associated with the completion of the current session""" @abstractmethod - async def send_api_request(self, method_name: str, params: dict = None, timeout: int = None) -> dict: + async def send_api_request(self, method_name: str, params: dict = None, timeout: int = None, + raw_response: bool = False) -> dict: """Method that use API instance for sending request to vk server :param method_name: any value from the left column of the methods table from `https://vk.com/dev/methods` :param params: dict of params that available for current method. For example see `Parameters` block from: `https://vk.com/dev/account.getInfo` :param timeout: timeout for response from the server + :param raw_response: return full response :return: dict that contain data from `Result` block. Example see here: `https://vk.com/dev/account.getInfo` """ @@ -59,7 +62,8 @@ async def __aenter__(self) -> BaseSession: async def __aexit__(self, exc_type, exc_val, exc_tb): return await self.close() - async def send_api_request(self, method_name: str, params: dict = None, timeout: int = None) -> dict: + async def send_api_request(self, method_name: str, params: dict = None, timeout: int = None, + raw_response: bool = False) -> dict: # Prepare request if not timeout: timeout = self.timeout @@ -67,10 +71,11 @@ async def send_api_request(self, method_name: str, params: dict = None, timeout: params = {} if self.access_token: params['access_token'] = self.access_token - params['v'] = self.API_VERSION + if 'v' not in params: + params['v'] = self.API_VERSION # Send request - response = await self.driver.json(self.REQUEST_URL + method_name, params, timeout) + _, response = await self.driver.post_json(self.REQUEST_URL + method_name, params, timeout) # Process response # Checking the section with errors @@ -85,16 +90,18 @@ async def send_api_request(self, method_name: str, params: dict = None, timeout: params['captcha_sid'] = captcha_sid # Send request again # Provide one attempt to repeat the request - return await self.send_api_request(method_name, params, timeout) + return await self.send_api_request(method_name, params, timeout, raw_response) elif err_code == AUTHORIZATION_FAILED: await self.authorize() # Send request again # Provide one attempt to repeat the request - return await self.send_api_request(method_name, params, timeout) + return await self.send_api_request(method_name, params, timeout, raw_response) else: # Other errors is not related with security raise VkAPIError(error, self.REQUEST_URL + method_name) - # Must return only useful data + if raw_response: + return response + # Return only useful data return response['response'] async def authorize(self) -> None: @@ -146,12 +153,15 @@ def __init__(self, login: str, password: str, app_id: int, scope: str or int or async def authorize(self) -> None: """Getting a new token from server""" - html = await self._get_auth_page() - url = URL('/authorize?email') + url, html = await self._get_auth_page() for step in range(self.num_of_attempts): - if url.path == '/authorize' and 'email' in url.query: - # Invalid login or password and 'email' in q.query - url, html = await self._process_auth_form(html) + if url.path == '/authorize': + if '__q_hash' in url.query: + # Give rights for app + url, html = await self._process_access_form(html) + else: + # Invalid login or password and 'email' in q.query + url, html = await self._process_auth_form(html) if url.path == '/login' and url.query.get('act', '') == 'authcheck': # Entering 2auth code url, html = await self._process_2auth_form(html) @@ -168,10 +178,10 @@ async def authorize(self) -> None: return raise VkAuthError('Something went wrong', 'Exceeded the number of attempts to log in') - async def _get_auth_page(self) -> str: + async def _get_auth_page(self) -> Tuple[str, str]: """ Get authorization mobile page without js - :return: html page + :return: redirect_url, html page """ # Prepare request params = { @@ -185,13 +195,13 @@ async def _get_auth_page(self) -> str: params['scope'] = self.scope # Send request - status, response = await self.driver.get_text(self.AUTH_URL, params) + status, response, redirect_url = await self.driver.get_text(self.AUTH_URL, params) # Process response if status != 200: error_dict = json.loads(response) raise VkAuthError(error_dict['error'], error_dict['error_description'], self.AUTH_URL, params) - return response + return redirect_url, response async def _process_auth_form(self, html: str) -> (str, str): """ @@ -221,8 +231,8 @@ async def _process_auth_form(self, html: str) -> (str, str): form_url = "https://m.vk.com{}".format(form_url) # Send request - url, html = await self.driver.post_text(form_url, form_data) - return url, html + _, html, redirect_url = await self.driver.post_text(form_url, form_data) + return redirect_url, html async def _process_2auth_form(self, html: str) -> (str, str): """ @@ -245,8 +255,8 @@ async def _process_2auth_form(self, html: str) -> (str, str): form_data['code'] = await self.enter_confirmation_code() # Send request - url, html = await self.driver.post_text(form_url, form_data) - return url, html + _, html, redirect_url = await self.driver.post_text(form_url, form_data) + return redirect_url, html async def _process_access_form(self, html: str) -> (str, str): """ @@ -264,8 +274,8 @@ async def _process_access_form(self, html: str) -> (str, str): form_data = dict(p.inputs) # Send request - url, html = await self.driver.post_text(form_url, form_data) - return url, html + _, html, redirect_url = await self.driver.post_text(form_url, form_data) + return redirect_url, html async def enter_confirmation_code(self) -> str: """ @@ -306,7 +316,7 @@ async def authorize(self, code: str = None) -> None: 'redirect_uri': self.redirect_uri, 'code': code } - response = await self.driver.json(self.CODE_URL, params, self.timeout) + _, response = await self.driver.post_json(self.CODE_URL, params, self.timeout) if 'error' in response: raise VkAuthError(response['error'], response['error_description'], self.CODE_URL, params) self.access_token = response['access_token'] diff --git a/aiovk/utils.py b/aiovk/shaping.py similarity index 85% rename from aiovk/utils.py rename to aiovk/shaping.py index 064f847..713c520 100644 --- a/aiovk/utils.py +++ b/aiovk/shaping.py @@ -18,7 +18,7 @@ async def dispatcher(self, maxsize): for i in range(maxsize - self.qsize()): self.put_nowait(1) - def canel(self): + def cancel(self): self.task.cancel() @@ -27,7 +27,3 @@ async def wrapper(self, *args, **kwargs): await self._queue.get() return await func(self, *args, **kwargs) return wrapper - - -def get_request_params(request_params): - return {param['key']: param['value'] for param in request_params} diff --git a/requirements-dev.txt b/requirements-dev.txt index 7dc1c58..28bc534 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -1,3 +1,8 @@ -r requirements.txt -jinja2==2.10.1 -pyotp==2.2.7 +aiohttp-jinja2==1.4.2 +pyotp==2.4.1 +proxy.py==2.2.0 +pytest==6.2.1 +pytest-asyncio==0.14.0 +pytest-aiohttp==0.3.0 +python-dotenv \ No newline at end of file diff --git a/requirements.txt b/requirements.txt index 09f137d..1ff3fe7 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,12 +1,2 @@ -aiohttp>=3.2.0 -aiosocksy==0.1.2 -appdirs==1.4.3 -async-timeout==2.0.1 -chardet==3.0.4 -idna==2.6 -idna-ssl==1.0.1 -multidict==4.1.0 -packaging==16.8 -pyparsing==2.2.0 -six==1.10.0 -yarl==1.1.1 +aiohttp>=3.7.3 +aiohttp-socks>=0.5.5 diff --git a/setup.py b/setup.py index d83a8af..726e291 100644 --- a/setup.py +++ b/setup.py @@ -7,6 +7,9 @@ with open('README.rst', 'r') as f: readme = f.read() +with open('requirements.txt') as f: + requirements = list(map(lambda x: x.strip(), f.readlines())) + with codecs.open(os.path.join(os.path.abspath(os.path.dirname( __file__)), 'aiovk', '__init__.py'), 'r', 'latin1') as fp: try: @@ -27,16 +30,19 @@ long_description=readme, packages=find_packages(), - install_requires='aiohttp>=3.2.0', + install_requires=requirements, license='MIT License', classifiers=[ 'Environment :: Web Environment', 'Intended Audience :: Developers', 'License :: OSI Approved :: MIT License', - 'Programming Language :: Python :: 3.5', 'Programming Language :: Python :: 3.6', + 'Programming Language :: Python :: 3.7', + 'Programming Language :: Python :: 3.8', 'Topic :: Software Development :: Libraries :: Python Modules', ], keywords='vk.com api vk wrappper asyncio', + test_suite="tests", + python_requires='>=3.6' ) diff --git a/tests/certs/cert.pem b/tests/certs/cert.pem deleted file mode 100644 index f126db7..0000000 --- a/tests/certs/cert.pem +++ /dev/null @@ -1,17 +0,0 @@ ------BEGIN CERTIFICATE----- -MIICpTCCAg6gAwIBAgIJALIUVtKHFoiqMA0GCSqGSIb3DQEBCwUAMGoxCzAJBgNV -BAYTAlVTMQ8wDQYDVQQIDAZPcmVnb24xETAPBgNVBAcMCFBvcnRsYW5kMRUwEwYD -VQQKDAxDb21wYW55IE5hbWUxDDAKBgNVBAsMA09yZzESMBAGA1UEAwwJbG9jYWxo -b3N0MB4XDTE4MTIxNjEyMDAyNFoXDTI0MDYwNzEyMDAyNFowajELMAkGA1UEBhMC -VVMxDzANBgNVBAgMBk9yZWdvbjERMA8GA1UEBwwIUG9ydGxhbmQxFTATBgNVBAoM -DENvbXBhbnkgTmFtZTEMMAoGA1UECwwDT3JnMRIwEAYDVQQDDAlsb2NhbGhvc3Qw -gZ8wDQYJKoZIhvcNAQEBBQADgY0AMIGJAoGBAOPT1EkOPwRtscX3WCI1Qd8dwoOL -nh0LvQSi1OBDuHFvlcfoEtAZgAFSoXXK+rofhYEcB2fwCSXVmRUzVqA8CINNMFCJ -ue/G+e2XUTKzJNZyOOTR5FNH3Uh/m3DVtmDK6aV39rQYxk0AvW9SDsfUXil3lCSy -bJvIW5knYU4g53/zAgMBAAGjUzBRMB0GA1UdDgQWBBSukcOPn66h0WXEOPaSVJ41 -lH6ZbzAfBgNVHSMEGDAWgBSukcOPn66h0WXEOPaSVJ41lH6ZbzAPBgNVHRMBAf8E -BTADAQH/MA0GCSqGSIb3DQEBCwUAA4GBAIbO7sMaOVnbEaNe19EUSuzhEmTMdSgk -llyqDTus7skH5ZDMI9rutGqfmZPcjym2/swC0stScPJ/iQym7rZvPd0tWKPYzLhw -h/Ltj0rxXCd/3Qn3Te3QGMHXlHkq65hF4YcqXaqiufZKaXIGn+V5yU7vt6osiMiT -auv4U+l0Bgas ------END CERTIFICATE----- diff --git a/tests/certs/generate_cert.sh b/tests/certs/generate_cert.sh deleted file mode 100755 index 8f3f123..0000000 --- a/tests/certs/generate_cert.sh +++ /dev/null @@ -1,2 +0,0 @@ -#!/bin/sh -openssl req -x509 -nodes -newkey rsa:1024 -keyout key.pem -out cert.pem -days 2000 -subj "/C=US/ST=Oregon/L=Portland/O=Company Name/OU=Org/CN=localhost" diff --git a/tests/certs/key.pem b/tests/certs/key.pem deleted file mode 100644 index e4e9ba3..0000000 --- a/tests/certs/key.pem +++ /dev/null @@ -1,16 +0,0 @@ ------BEGIN PRIVATE KEY----- -MIICdwIBADANBgkqhkiG9w0BAQEFAASCAmEwggJdAgEAAoGBAOPT1EkOPwRtscX3 -WCI1Qd8dwoOLnh0LvQSi1OBDuHFvlcfoEtAZgAFSoXXK+rofhYEcB2fwCSXVmRUz -VqA8CINNMFCJue/G+e2XUTKzJNZyOOTR5FNH3Uh/m3DVtmDK6aV39rQYxk0AvW9S -DsfUXil3lCSybJvIW5knYU4g53/zAgMBAAECgYAVLX+yOszI0JmR2Wgr5d91qgKG -z9emw4ySfcBkCGeAw+qcZoo5JeWsOc4gcPnVi1CRq9VUaA+xoKVWPytyQaJYxVAW -LtIMerzVMquqaY8U+6ejxQF1TCACIeamUpBnYm+jvjmMWsayM20gw/U18pVYsbWO -FFEXYgCHlqD1LYb+YQJBAPX9R0GcysNrJbi1kmx2s2/oX9ygH9OAA91hZ70jqn1M -yV8QZzwKsAj7rx4XpeNliB5Qmz77ehX+2DuC9AV4jw8CQQDtGVb8ILQxZBa5bIP8 -UmvYRBrd2bQk+Kfk7N+G5Khmh+zT7cGAqOoIFvbJUE7UbPp5x1PrfhZOBEWURWYa -7ADdAkBhnaPkRqs0B1YNyYgUoLouQ4GfFK/sh1WBSYEYTon+dTVIE0NUUU1wEyh4 -AZxj88ujdAtXYAYfqmT2oM3jSedDAkEA0ZfK60rFv7uo0vV4n38E0sMxtNgUhXJC -iP5UgtxzeV/DHX1ZxzCK4efa9Q9HAEXuDeUE7HIjqYfhMjc/EaF7CQJBANb/WGsK -4tqn27wjvqdnSk/mCEAEhPpwZUqP+TUYKSL1+sPmQtgjHEuT55oEpC/T9Y+NMpmG -8bqwPfzzpvDVmhw= ------END PRIVATE KEY----- diff --git a/tests/functional/__init__.py b/tests/functional/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/functional/conftest.py b/tests/functional/conftest.py new file mode 100644 index 0000000..30031c8 --- /dev/null +++ b/tests/functional/conftest.py @@ -0,0 +1,81 @@ +from pathlib import Path + +import aiohttp_jinja2 +import jinja2 +import pytest +from aiohttp import web + + +@pytest.fixture() +def user_1_data(): + return [{'id': 1, 'first_name': 'Павел', 'last_name': 'Дуров'}] + + +@pytest.fixture() +def valid_token(): + return 'token' + + +@pytest.fixture() +async def vk_server(aiohttp_server, user_1_data, valid_token): + @aiohttp_jinja2.template('authorize_page.jinja2') + async def authorize(request): + if 'client_id' in request.query: + return {'base_url': request.host} + else: + raise web.HTTPNotImplemented + + @aiohttp_jinja2.template('blank.jinja2') + async def blank(request): + return + + async def longpoolwait(request): + return web.json_response({'ts': 1857669127, 'updates': []}) + + async def user_get_error(request): + return web.json_response({'error': {'error_code': 5, + 'error_msg': 'User authorization failed: invalid access_token (4).', + 'request_params': [{'key': 'oauth', 'value': '1'}, + {'key': 'method', 'value': 'users.get'}, + {'key': 'user_ids', 'value': '1'}, + {'key': 'v', 'value': '5.74'}]}}) + + async def user_get(request): + return web.json_response({'response': user_1_data}) + + async def get_long_poll_server(request): + data = await request.post() + if data.get('access_token') == valid_token: + return web.json_response( + {'response': {'key': 'long_pool_key', 'server': f'{request.host}/im1774', 'ts': 1857669095}}) + else: + return web.json_response({'error': {'error_code': 5, + 'error_msg': 'User authorization failed: no access_token passed.', + 'request_params': [{'key': 'oauth', 'value': '1'}, {'key': 'method', + 'value': 'messages.getLongPollServer'}, + {'key': 'need_pts', 'value': '0'}, + {'key': 'v', 'value': '5.74'}]}}) + + async def root(request): + data = await request.post() + if data.get('email') == 'login': + location = request.app.router['blank'].url_for() + raise web.HTTPFound(location=f'{location}#access_token={valid_token}') + else: + response = aiohttp_jinja2.render_template('blank.jinja2', request, None) + return response + + app = web.Application() + template_dir = Path(__file__).parent.parent / 'responses' + aiohttp_jinja2.setup(app, loader=jinja2.FileSystemLoader(template_dir)) + app.add_routes([ + web.get('/authorize', authorize), + web.get('/blank.html', blank, name='blank'), + web.get('/im1774', longpoolwait), + web.post('/method/users.get.error', user_get_error), + web.post('/method/users.get', user_get), + web.post('/method/messages.getLongPollServer', get_long_poll_server), + web.post('/', root), + ]) + server = await aiohttp_server(app) + yield server diff --git a/tests/functional/test_drivers.py b/tests/functional/test_drivers.py new file mode 100644 index 0000000..512deae --- /dev/null +++ b/tests/functional/test_drivers.py @@ -0,0 +1,128 @@ +import json + +import proxy +import pytest +from aiohttp import web +from aiohttp.test_utils import unused_port +from python_socks import ProxyType +from yarl import URL + +from aiovk.drivers import HttpDriver, ProxyDriver + +pytestmark = pytest.mark.asyncio + + +@pytest.fixture() +def simple_response_data(): + return {'a': 1} + + +@pytest.fixture() +async def vk_server(aiohttp_server, simple_response_data): + async def simple_response_data_handler(request): + return web.json_response(simple_response_data) + + app = web.Application() + app.add_routes([web.route('*', r'/{name:.*}', simple_response_data_handler)]) + server = await aiohttp_server(app) + yield server + + +@pytest.fixture(scope='module') +def proxy_server(): + with proxy.start(['--hostname', '127.0.0.1', '--num-workers', '1', '--port', str(unused_port())]) as p: + yield p + + +@pytest.mark.parametrize( + 'driver_class, use_proxy', + [ + (HttpDriver, False), + (ProxyDriver, True), + ] +) +async def test_post_json(simple_response_data, vk_server, proxy_server, driver_class, use_proxy): + url = f'http://{vk_server.host}:{vk_server.port}/' + params = {} + if use_proxy: + params['address'] = str(proxy_server.flags.hostname) + params['port'] = proxy_server.flags.port + params['proxy_type'] = ProxyType.HTTP + driver = driver_class(**params) + status, jsn = await driver.post_json(url, {}) + await driver.close() + + assert status == 200 + assert jsn == simple_response_data + + +@pytest.mark.parametrize( + 'driver_class, use_proxy', + [ + (HttpDriver, False), + (ProxyDriver, True), + ] +) +async def test_get_bin(simple_response_data, vk_server, proxy_server, driver_class, use_proxy): + url = f'http://{vk_server.host}:{vk_server.port}/' + params = {} + if use_proxy: + params['address'] = str(proxy_server.flags.hostname) + params['port'] = proxy_server.flags.port + params['proxy_type'] = ProxyType.HTTP + driver = driver_class(**params) + status, text = await driver.get_bin(url, {}) + await driver.close() + + assert status == 200 + assert text == json.dumps(simple_response_data).encode() + + +@pytest.mark.parametrize( + 'driver_class, use_proxy', + [ + (HttpDriver, False), + (ProxyDriver, True), + ] +) +async def test_get_text(simple_response_data, vk_server, proxy_server, driver_class, use_proxy): + url = f'http://{vk_server.host}:{vk_server.port}/' + params = {} + if use_proxy: + params['address'] = str(proxy_server.flags.hostname) + params['port'] = proxy_server.flags.port + params['proxy_type'] = ProxyType.HTTP + driver = driver_class(**params) + status, text, redirect_url = await driver.get_text(url, {}) + await driver.close() + + assert status == 200 + assert text == json.dumps(simple_response_data) + assert redirect_url == URL(url) + + +@pytest.mark.parametrize( + 'driver_class, use_proxy', + [ + (HttpDriver, False), + (ProxyDriver, True), + ] +) +async def test_post_text(simple_response_data, vk_server, proxy_server, driver_class, use_proxy): + data = { + 'login': 'test', + 'password': 'test' + } + url = f'http://{vk_server.host}:{vk_server.port}/' + params = {} + if use_proxy: + params['address'] = str(proxy_server.flags.hostname) + params['port'] = proxy_server.flags.port + params['proxy_type'] = ProxyType.HTTP + driver = driver_class(**params) + status, text, redirect_url = await driver.post_text(url, data=data) + await driver.close() + + assert status == 200 + assert text == json.dumps(simple_response_data) + assert redirect_url == URL(url) diff --git a/tests/functional/test_longpool.py b/tests/functional/test_longpool.py new file mode 100644 index 0000000..c0f4fdb --- /dev/null +++ b/tests/functional/test_longpool.py @@ -0,0 +1,52 @@ +import pytest + +from aiovk import ImplicitSession, API, LongPoll, TokenSession + +pytestmark = pytest.mark.asyncio + + +async def test_wait_valid_with_token_session(vk_server, valid_token): + url = f'http://{vk_server.host}:{vk_server.port}' + t = TokenSession(valid_token, timeout=1000) + t.BASE_URL = url + t.REQUEST_URL = f'{url}/method/' + api = API(t) + lp = LongPoll(api, mode=2, wait=2) + lp.use_https = False + + response = await lp.wait() + await t.close() + assert 'ts' in response + assert 'updates' in response + + +async def test_wait_valid_with_session_authorised(vk_server): + url = f'http://{vk_server.host}:{vk_server.port}' + s = ImplicitSession(login='login', password='pass', app_id='123', scope='messages') + s.REQUEST_URL = f'{url}/method/' + s.AUTH_URL = f'{url}/authorize' + await s.authorize() + + lp = LongPoll(s, mode=2, wait=2) + lp.use_https = False + + response = await lp.wait() + await s.close() + assert 'ts' in response + assert 'updates' in response + + +async def test_wait_valid_with_session_auto_auth(vk_server): + url = f'http://{vk_server.host}:{vk_server.port}' + s = ImplicitSession(login='login', password='pass', app_id='123', scope='messages') + s.REQUEST_URL = f'{url}/method/' + s.AUTH_URL = f'{url}/authorize' + + api = API(s) + lp = LongPoll(api, mode=2, wait=2) + lp.use_https = False + + response = await lp.wait() + await s.close() + assert 'ts' in response + assert 'updates' in response diff --git a/tests/functional/test_sessions.py b/tests/functional/test_sessions.py new file mode 100644 index 0000000..eaa71ee --- /dev/null +++ b/tests/functional/test_sessions.py @@ -0,0 +1,70 @@ +import pytest + +from aiovk import ImplicitSession, TokenSession +from aiovk.exceptions import VkAuthError + +pytestmark = pytest.mark.asyncio + + +async def test_token_session_auth_with_empty_token(vk_server): + s = TokenSession() + with pytest.raises(VkAuthError): + await s.authorize() + await s.close() + + +async def test_token_session_auth_with_token(vk_server): + s = TokenSession('token') + with pytest.raises(VkAuthError): + await s.authorize() + await s.close() + + +async def test_token_session_auth_token_free_request_without_token(vk_server, user_1_data): + url = f'http://{vk_server.host}:{vk_server.port}' + s = TokenSession() + s.REQUEST_URL = f'{url}/method/' + result = await s.send_api_request('users.get', {'user_ids': 1}) + await s.close() + assert result == user_1_data + + +async def test_token_session_auth_token_request_without_token(vk_server): + url = f'http://{vk_server.host}:{vk_server.port}' + s = TokenSession('token') + s.REQUEST_URL = f'{url}/method/' + with pytest.raises(VkAuthError): + await s.send_api_request('users.get.error', {'user_ids': 1}) + await s.close() + + +async def test_implicit_session_auth_with_empty_data(vk_server): + url = f'http://{vk_server.host}:{vk_server.port}' + s = ImplicitSession(login='', password='', app_id='') + s.REQUEST_URL = f'{url}/method/' + s.AUTH_URL = f'{url}/authorize' + + with pytest.raises(VkAuthError): + await s.authorize() + await s.close() + + +async def test_implicit_session_auth_with_2factor(vk_server): + url = f'http://{vk_server.host}:{vk_server.port}' + s = ImplicitSession(login='login', password='pass', app_id='123') + s.REQUEST_URL = f'{url}/method/' + s.AUTH_URL = f'{url}/authorize' + + await s.authorize() + await s.close() + + +@pytest.mark.skip('TODO add captcha test') +async def test_implicit_session_auth_process_captcha_without(vk_server): + url = f'http://{vk_server.host}:{vk_server.port}' + s = ImplicitSession(login='login', password='pass', app_id='123') + s.REQUEST_URL = f'{url}/method/' + s.AUTH_URL = f'{url}/authorize' + + await s.authorize() + await s.close() diff --git a/tests/responses/2auth_form.html b/tests/responses/2auth_form.html deleted file mode 100644 index 9412ab8..0000000 --- a/tests/responses/2auth_form.html +++ /dev/null @@ -1,141 +0,0 @@ - - - -
- - - - - - - -